From 0f8fa8394be658279a455d268194cd2d5a3ee700 Mon Sep 17 00:00:00 2001 From: Chenglong Wang Date: Thu, 28 May 2026 22:03:13 -0700 Subject: [PATCH 01/40] some fixes --- .../datalake/workspace_manager.py | 11 +- src/app/dfSlice.tsx | 28 ++ src/views/DataFormulator.tsx | 34 ++- src/views/DataLoadingChat.tsx | 263 +++++++++--------- src/views/DataThread.tsx | 16 +- src/views/UnifiedDataUploadDialog.tsx | 140 ++++------ src/views/dataLoadingSuggestions.ts | 82 ++++-- tests/backend/data/test_workspace_manager.py | 64 ++++- 8 files changed, 369 insertions(+), 269 deletions(-) diff --git a/py-src/data_formulator/datalake/workspace_manager.py b/py-src/data_formulator/datalake/workspace_manager.py index 679452ca..23e37176 100644 --- a/py-src/data_formulator/datalake/workspace_manager.py +++ b/py-src/data_formulator/datalake/workspace_manager.py @@ -169,6 +169,10 @@ def list_workspaces(self) -> list[dict]: workspace. If a workspace directory lacks this file (legacy), it is auto-repaired via :meth:`_ensure_meta`. + Every workspace directory is listed, including empty + "Untitled Session" entries from data-loading chats. Users + manage (rename/delete) these themselves via the sidebar. + Returns list of {"id": str, "display_name": str, "updated_at": str}. """ workspaces = [] @@ -184,13 +188,16 @@ def list_workspaces(self) -> list[dict]: except Exception: continue + tc = meta.get("tableCount") + cc = meta.get("chartCount") + workspaces.append({ "id": child.name, "display_name": meta.get("displayName", child.name), "created_at": meta.get("createdAt") or meta.get("updatedAt"), "updated_at": meta.get("updatedAt"), - "table_count": meta.get("tableCount"), - "chart_count": meta.get("chartCount"), + "table_count": tc, + "chart_count": cc, }) workspaces.sort(key=lambda w: w.get("updated_at") or "", reverse=True) diff --git a/src/app/dfSlice.tsx b/src/app/dfSlice.tsx index a3fe5add..89b075ab 100644 --- a/src/app/dfSlice.tsx +++ b/src/app/dfSlice.tsx @@ -210,6 +210,17 @@ export interface DataFormulatorState { * Transient — not persisted. */ dataLoadingChatResetCounter: number; + /** + * Pending submission queued for the data-loading chat. Set by any + * surface that wants to hand a prompt off to the chat (the menu + * agent input box, suggestion auto-run, external dialog callers). + * `DataLoadingChat` consumes it on render: it clears the slot and + * sends the carried payload as a fresh user message. Using a single + * redux slot (instead of props + a reset counter) eliminates the + * cross-tick race where the parent's pre-clear would otherwise + * cancel the auto-send for the new prompt. Transient — not persisted. + */ + dataLoadingChatPending: { text: string; images: string[]; attachments: string[] } | null; /** * Pending hand-off from the Data Agent to a peer agent. Set by the * Data Agent's `delegate` action card; consumed by `DataFormulator` @@ -299,6 +310,7 @@ const initialState: DataFormulatorState = { dataLoadingChatMessages: [], dataLoadingChatInProgress: false, dataLoadingChatResetCounter: 0, + dataLoadingChatPending: null, agentHandoffRequest: null, generatedReports: [], @@ -720,6 +732,7 @@ export const dataFormulatorSlice = createSlice({ state.dataLoadingChatMessages = []; state.dataLoadingChatInProgress = false; state.dataLoadingChatResetCounter = (state.dataLoadingChatResetCounter ?? 0) + 1; + state.dataLoadingChatPending = null; state.generatedReports = []; @@ -837,6 +850,7 @@ export const dataFormulatorSlice = createSlice({ config: { ...initialState.config, ...(saved.config || {}) }, dataCleanBlocks: saved.dataCleanBlocks || [], dataLoadingChatMessages: saved.dataLoadingChatMessages || [], + dataLoadingChatPending: null, generatedReports: saved.generatedReports || [], // Reset transient fields @@ -1665,6 +1679,20 @@ export const dataFormulatorSlice = createSlice({ state.dataLoadingChatMessages = []; state.dataLoadingChatInProgress = false; state.dataLoadingChatResetCounter = (state.dataLoadingChatResetCounter ?? 0) + 1; + // Note: `dataLoadingChatPending` is intentionally left + // alone. Callers that want "fresh slate + auto-send the + // new prompt" dispatch `clearChatMessages` followed by + // `setDataLoadingChatPending` in the same tick — clearing + // pending here would race with that ordering. + }, + setDataLoadingChatPending: ( + state, + action: PayloadAction<{ text: string; images: string[]; attachments: string[] }>, + ) => { + state.dataLoadingChatPending = action.payload; + }, + clearDataLoadingChatPending: (state) => { + state.dataLoadingChatPending = null; }, confirmTableLoad: (state, action: PayloadAction<{messageId: string, tableName: string}>) => { const msg = state.dataLoadingChatMessages.find(m => m.id === action.payload.messageId); diff --git a/src/views/DataFormulator.tsx b/src/views/DataFormulator.tsx index 2c21c15b..00e8086e 100644 --- a/src/views/DataFormulator.tsx +++ b/src/views/DataFormulator.tsx @@ -301,24 +301,37 @@ export const DataFormulatorFC = ({ }) => { // State for unified data upload dialog const [uploadDialogOpen, setUploadDialogOpen] = useState(false); const [uploadDialogInitialTab, setUploadDialogInitialTab] = useState('menu'); - const [uploadDialogInitialChatPrompt, setUploadDialogInitialChatPrompt] = useState(undefined); - const [uploadDialogInitialChatImages, setUploadDialogInitialChatImages] = useState(undefined); // Loading state for sessions (from Redux, shared with App.tsx) const sessionLoading = useSelector((state: DataFormulatorState) => state.sessionLoading); const sessionLoadingLabel = useSelector((state: DataFormulatorState) => state.sessionLoadingLabel); - const openUploadDialog = (tab: UploadTabType, initialChatPrompt?: string, initialChatImages?: string[]) => { + const openUploadDialog = (tab: UploadTabType) => { // If no workspace is active, generate an ID (backend creates folder lazily on first data op) if (!activeWorkspace) { dispatch(dfActions.setActiveWorkspace({ id: generateSessionId(), displayName: 'Untitled Session' })); } setUploadDialogInitialTab(tab); - setUploadDialogInitialChatPrompt(initialChatPrompt); - setUploadDialogInitialChatImages(initialChatImages); setUploadDialogOpen(true); }; + // Seed the Data Loading chat through the single redux `pending` slot, + // then navigate to the extract tab. This is the one channel that + // carries text, images, AND file attachments as first-class fields — + // replacing the older `initialChatPrompt/Images` props that silently + // dropped file attachments (they had no dedicated field and only + // survived if their name was baked into the prompt text). + const startDataLoadingChat = (text: string, images: string[] = [], attachments: string[] = []) => { + if (text.trim().length > 0 || images.length > 0 || attachments.length > 0) { + // Fresh query replaces any prior conversation. + if (dataLoadingChatMessages.length > 0) { + dispatch(dfActions.clearChatMessages()); + } + dispatch(dfActions.setDataLoadingChatPending({ text, images, attachments })); + } + openUploadDialog('extract'); + }; + // Honor cross-component requests to hand off to the Data Loading // chat seeded with a prompt (e.g. Data Agent's `delegate` card with // target='data_loading'). Hand-offs targeting other agents (e.g. @@ -326,7 +339,7 @@ export const DataFormulatorFC = ({ }) => { const agentHandoffRequest = useSelector((state: DataFormulatorState) => state.agentHandoffRequest); useEffect(() => { if (agentHandoffRequest && agentHandoffRequest.target === 'data_loading') { - openUploadDialog('extract', agentHandoffRequest.prompt, agentHandoffRequest.images); + startDataLoadingChat(agentHandoffRequest.prompt, agentHandoffRequest.images ?? [], []); dispatch(dfActions.clearAgentHandoffRequest()); } // openUploadDialog is stable enough for this purpose; we only react @@ -730,7 +743,7 @@ export const DataFormulatorFC = ({ }) => { openUploadDialog(`connector:${conn.id}` as UploadTabType); } }} - onStartChat={(prompt, images) => openUploadDialog('extract', prompt, images)} + onStartChat={(prompt, images, attachments) => startDataLoadingChat(prompt, images, attachments)} hasPriorConversation={dataLoadingChatMessages.length > 0} onResumeChat={() => openUploadDialog('extract')} serverConfig={serverConfig} @@ -933,16 +946,9 @@ export const DataFormulatorFC = ({ }) => { open={uploadDialogOpen} onClose={() => { setUploadDialogOpen(false); - // Clear one-shot seed values so the next dialog - // open (e.g. via the upload button) doesn't - // re-fire the agent with a stale prompt/image. - setUploadDialogInitialChatPrompt(undefined); - setUploadDialogInitialChatImages(undefined); refreshPageConnectors(); }} initialTab={uploadDialogInitialTab} - initialChatPrompt={uploadDialogInitialChatPrompt} - initialChatImages={uploadDialogInitialChatImages} onConnectorsChanged={handleConnectorsChanged} /> {/* Loading overlay for session loading */} diff --git a/src/views/DataLoadingChat.tsx b/src/views/DataLoadingChat.tsx index 379e29fb..9fe59000 100644 --- a/src/views/DataLoadingChat.tsx +++ b/src/views/DataLoadingChat.tsx @@ -60,7 +60,11 @@ const getUniqueTableName = (baseName: string, existingNames: Set): strin // Modern monospace font stack for code blocks const CODE_FONT = '"SF Mono", "Cascadia Code", "Fira Code", Menlo, Consolas, "Liberation Mono", monospace'; -const MarkdownContent: React.FC<{ content: string }> = ({ content }) => { +// Memoized so typing in the chat input (which re-renders the parent +// `DataLoadingChat` on every keystroke) doesn't re-parse every assistant +// message through react-markdown. `content` is a stable string per +// committed message, so the default shallow equality is sufficient. +const MarkdownContent = React.memo(({ content }: { content: string }) => { return ( = ({ content }) => { ); -}; +}); // --------------------------------------------------------------------------- // Inline table preview — compact notebook-style @@ -317,10 +321,16 @@ const CodeBlockView: React.FC<{ block: CodeExecution }> = ({ block }) => { // Single chat message bubble // --------------------------------------------------------------------------- -const ChatBubble: React.FC<{ +// Memoized so typing in the chat input doesn't re-render every prior +// bubble (each one renders MarkdownContent + potentially code blocks / +// table previews, which is expensive on long threads). The parent +// stabilises `existingNames` via useMemo so memo equality holds across +// keystrokes. +const ChatBubble = React.memo<{ message: ChatMessage; existingNames: Set; -}> = ({ message, existingNames }) => { + onTableLoaded?: () => void; +}>(({ message, existingNames, onTableLoaded }) => { const theme = useTheme(); const { t } = useTranslation(); const dispatch = useDispatch(); @@ -340,6 +350,9 @@ const ChatBubble: React.FC<{ if (table) { dispatch(loadTable({ table: { ...table, source: { type: 'extract' as const } } })); dispatch(dfActions.confirmTableLoad({ messageId: message.id, tableName: pending.name })); + // Loading data is a deliberate commit — return the + // user to the canvas (the dialog closes via this hook). + onTableLoaded?.(); } } } catch (err) { @@ -468,6 +481,11 @@ const ChatBubble: React.FC<{ })); } dispatch(dfActions.markLoadPlanConfirmed({ messageId: message.id })); + if (selected.length > 0) { + // Return the user to the canvas after a + // deliberate batch load. + onTableLoaded?.(); + } }} /> )} @@ -493,7 +511,7 @@ const ChatBubble: React.FC<{ ); -}; +}); // --------------------------------------------------------------------------- // Tool call label mapping @@ -517,7 +535,10 @@ interface ToolStep { label: string; } -const StreamingIndicator: React.FC<{ content: string; toolSteps: ToolStep[] }> = ({ content, toolSteps }) => { +// Memoized so an unrelated parent re-render (e.g. typing) doesn't +// reflow the shimmer animation. Props are state values that only change +// during an active stream. +const StreamingIndicator = React.memo<{ content: string; toolSteps: ToolStep[] }>(({ content, toolSteps }) => { const theme = useTheme(); return ( @@ -579,55 +600,56 @@ const StreamingIndicator: React.FC<{ content: string; toolSteps: ToolStep[] }> = )} ); -}; +}); // --------------------------------------------------------------------------- // Main chat component // --------------------------------------------------------------------------- -export interface DataLoadingChatProps { - /** - * Optional initial text to pre-fill the chat input when the component - * mounts (or when the value changes). Used by external entry points - * (e.g. landing page quick-chat box) that want to hand off a prompt - * to the agent. - */ - initialPrompt?: string; - /** - * Optional images (data URLs) to seed alongside `initialPrompt` — - * used when an external surface (e.g. landing-page agent box) has - * already collected pasted/attached images and is handing them off. - */ - initialImages?: string[]; - /** - * If true, automatically send the `initialPrompt` once on mount/change. - * Otherwise the prompt is only pre-filled and the user presses Enter. - */ - autoSendInitialPrompt?: boolean; +interface DataLoadingChatProps { + /** Called after a table is successfully loaded into the app. The + * upload dialog wires this to its close handler so loading data + * returns the user to the canvas. */ + onTableLoaded?: () => void; } -export const DataLoadingChat: React.FC = ({ - initialPrompt, - initialImages, - autoSendInitialPrompt, -}) => { +export const DataLoadingChat: React.FC = ({ onTableLoaded }) => { const theme = useTheme(); const { t } = useTranslation(); const dispatch = useDispatch(); + // Keep the latest callback in a ref so the stable `handleTableLoaded` + // identity below doesn't bust `ChatBubble`'s memoization even when the + // parent passes a fresh closure each render. + const onTableLoadedRef = useRef(onTableLoaded); + onTableLoadedRef.current = onTableLoaded; + const handleTableLoaded = useCallback(() => { + onTableLoadedRef.current?.(); + }, []); + const chatMessages = useSelector((state: DataFormulatorState) => state.dataLoadingChatMessages); const chatInProgress = useSelector((state: DataFormulatorState) => state.dataLoadingChatInProgress); - // External reset signal — bumped by `clearChatMessages` (manual reset - // button, new menu-level query, full session reset). When it changes - // we abort any in-flight stream, drop partial UI state, and re-seed - // from props if the parent provided a new prompt/images. Without - // this, an in-flight stream's eventual dispatches would leak into - // the freshly-cleared thread. + // External reset signal — bumped by `clearChatMessages` (manual + // reset button, fresh menu submission, full session reset). Used + // here only to abort an in-flight stream and invalidate any + // late-arriving dispatches from that stream via `sessionRef`. const chatResetCounter = useSelector((state: DataFormulatorState) => state.dataLoadingChatResetCounter ?? 0); + // Pending submission queued by an external surface (menu agent + // box, suggestion auto-run, external dialog caller). When set, we + // consume it in a useEffect: clear the slot first, then send the + // carried payload as a fresh user message via `sendMessage`. + // Single redux signal = no prop race. + const pendingSubmission = useSelector((state: DataFormulatorState) => state.dataLoadingChatPending); const existingTables = useSelector((state: DataFormulatorState) => state.tables); const activeModel = useSelector(dfSelectors.getActiveModel); const frontendRowLimit = useSelector((state: DataFormulatorState) => state.config?.frontendRowLimit ?? 2_000_000); - const existingNames = new Set(existingTables.map(tbl => tbl.id)); + // Stable reference across renders that don't actually change the + // table list — without this, every keystroke in the chat input + // would rebuild the Set and bust `ChatBubble`'s memo equality. + const existingNames = React.useMemo( + () => new Set(existingTables.map(tbl => tbl.id)), + [existingTables], + ); const [prompt, setPrompt] = useState(''); const [userImages, setUserImages] = useState([]); @@ -654,95 +676,44 @@ export const DataLoadingChat: React.FC = ({ // Auto-focus input useEffect(() => { inputRef.current?.focus(); }, []); - // ---- External initial prompt handling ------------------------------- - // Pre-fill the input (and optionally auto-send) when `initialPrompt` - // is provided. Used by external surfaces (e.g. landing-page quick chat - // box) to hand off text to the agent. Auto-send only fires for a - // fresh conversation — we never auto-resend on remount mid-chat. - const hasExistingMessages = chatMessages.length > 0; - const [pendingAutoSend, setPendingAutoSend] = useState(false); + // ---- Reset handling ------------------------------------------------- + // On external reset (counter bump from `clearChatMessages`): abort + // any in-flight stream, invalidate the current session token, and + // clear local input/streaming UI state. We deliberately do NOT + // re-seed anything here — a reset means "clean slate"; any new + // submission arrives separately via `pendingSubmission`. useEffect(() => { - // Detect external reset: abort, invalidate in-flight session, - // and clear all local UI state before re-seeding. Including - // `chatResetCounter` in the dep list also guarantees that an - // identical-prompt re-submission (same `initialPrompt` string) - // still triggers a fresh auto-send — otherwise the deps would - // be unchanged and the effect would skip. - const isReset = chatResetCounter !== lastResetRef.current; - if (isReset) { - lastResetRef.current = chatResetCounter; - sessionRef.current += 1; - abortControllerRef.current?.abort(); - abortControllerRef.current = null; - setStreamingContent(''); - setStreamingToolSteps([]); - setPrompt(''); - setUserImages([]); - setUserAttachments([]); - setPendingAutoSend(false); - } - - // Extract `[Uploaded: name]` mentions from the seeded prompt and - // surface them as chips. The mention template is locale-aware, - // so we build the regex from the current i18n value rather than - // hard-coding the English form. - const mentionTemplate = t('dataLoading.uploaded', { name: '__DF_NAME__' }); - const mentionPattern = mentionTemplate - .replace(/[.*+?^${}()|[\]\\]/g, '\\$&') - .replace('__DF_NAME__', '(.+?)'); - const mentionRegex = new RegExp(mentionPattern, 'g'); - let seededPrompt = initialPrompt || ''; - const extractedNames: string[] = []; - if (seededPrompt) { - let match: RegExpExecArray | null; - while ((match = mentionRegex.exec(seededPrompt)) !== null) { - extractedNames.push(match[1]); - } - if (extractedNames.length > 0) { - seededPrompt = seededPrompt - .replace(new RegExp(`\\n?${mentionPattern}`, 'g'), '') - .trim(); - } - } - - const hasText = seededPrompt.trim().length > 0; - const hasImages = !!initialImages && initialImages.length > 0; - const hasAttachments = extractedNames.length > 0; - // Skip re-seeding the input on a user-initiated reset — the - // reset is meant to restore a clean slate, not re-populate the - // input with the prompt the user just cleared. - if (!isReset) { - if (hasText) setPrompt(seededPrompt); - if (hasAttachments) setUserAttachments(extractedNames); - if (hasImages) { - // Always replace, never append. The prop is a "seed" — each - // change represents a fresh handoff from the parent, not an - // additive update. Appending caused the same image to stack - // up every time the parent re-rendered with a new array ref. - setUserImages([...initialImages!]); - } - } - // Auto-send only on a genuinely fresh open (no prior messages, - // and not a user-initiated reset). Resetting means the user wants - // a clean slate — re-running the seeded prompt against their will - // would defeat the purpose of the reset button. - if (autoSendInitialPrompt && !isReset && (hasText || hasImages || hasAttachments) && !hasExistingMessages) { - setPendingAutoSend(true); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [initialPrompt, initialImages, autoSendInitialPrompt, chatResetCounter]); + if (chatResetCounter === lastResetRef.current) return; + lastResetRef.current = chatResetCounter; + sessionRef.current += 1; + abortControllerRef.current?.abort(); + abortControllerRef.current = null; + setStreamingContent(''); + setStreamingToolSteps([]); + setPrompt(''); + setUserImages([]); + setUserAttachments([]); + }, [chatResetCounter]); const stopGeneration = () => { abortControllerRef.current?.abort(); }; // ---- Send message ---- - const sendMessage = useCallback(() => { - const text = prompt.trim(); - if (!text && userImages.length === 0 && userAttachments.length === 0) return; + // Accepts an optional explicit payload so callers (suggestion + // auto-run, pending-submission consume) can submit the exact + // values they just chose without waiting for React state to flush. + // Reading via the `prompt`/`userImages`/`userAttachments` closures + // alone would be racy with batching and could submit the previous + // round's values on a fresh handoff. + const sendMessage = useCallback((explicit?: { text: string; images: string[]; attachments: string[] }) => { + const text = (explicit?.text ?? prompt).trim(); + const imgs = explicit?.images ?? userImages; + const atts = explicit?.attachments ?? userAttachments; + if (!text && imgs.length === 0 && atts.length === 0) return; if (chatInProgress) return; - const imageAttachments: ChatAttachment[] = userImages.map((url, i) => ({ + const imageAttachments: ChatAttachment[] = imgs.map((url, i) => ({ type: 'image' as const, name: `image-${i + 1}`, url, })); - const fileAttachments: ChatAttachment[] = userAttachments.map(name => ({ + const fileAttachments: ChatAttachment[] = atts.map(name => ({ type: 'file' as const, name, })); const attachments: ChatAttachment[] = [...imageAttachments, ...fileAttachments]; @@ -751,7 +722,7 @@ export const DataLoadingChat: React.FC = ({ // chips (rendered from `attachments`). The agent payload below // re-injects `[Uploaded: name]` mentions so the backend still // sees the file references inline. - const displayText = text || (userImages.length > 0 ? t('dataLoading.defaultImageMessage') : ''); + const displayText = text || (imgs.length > 0 ? t('dataLoading.defaultImageMessage') : ''); const userMsg: ChatMessage = { id: `msg-${Date.now()}-user`, role: 'user', @@ -967,25 +938,48 @@ export const DataLoadingChat: React.FC = ({ } } })(); - }, [prompt, userImages, chatInProgress, chatMessages, activeModel, existingTables, dispatch, streamingContent, t]); + }, [prompt, userImages, userAttachments, chatInProgress, chatMessages, activeModel, existingTables, dispatch, streamingContent, t]); - // Auto-send the initial prompt once it has been applied to state. + // Consume a queued submission from any external surface (menu + // agent input, suggestion auto-run, or a cross-component handoff + // routed through `startDataLoadingChat`). Single redux signal, + // single consumer — no prop race. + // + // Idempotency note: under React.StrictMode (dev), effects are + // intentionally double-invoked on mount with the *same* closure, + // so the `clearDataLoadingChatPending` dispatch in the first run + // isn't visible to the second run. `lastConsumedRef` tracks the + // exact payload object we've already sent, so the second + // invocation short-circuits before calling `sendMessage` again. + const lastConsumedRef = useRef(null); useEffect(() => { - if (!pendingAutoSend) return; + if (!pendingSubmission) return; + if (pendingSubmission === lastConsumedRef.current) return; if (chatInProgress) return; - if (prompt.trim().length === 0 && userImages.length === 0) return; - setPendingAutoSend(false); - sendMessage(); - }, [pendingAutoSend, prompt, userImages, chatInProgress, sendMessage]); + lastConsumedRef.current = pendingSubmission; + const payload = pendingSubmission; + dispatch(dfActions.clearDataLoadingChatPending()); + sendMessage(payload); + }, [pendingSubmission, chatInProgress, sendMessage, dispatch]); // Reuse the shared sample-task list so this in-session panel stays in // sync with the upload-dialog entry point (`UnifiedDataUploadDialog`). + // Auto-run is wired through the redux pending slot so the click — + // even on a chat with prior history — atomically clears the thread + // and queues the new submission. const focusSuggestions = React.useMemo(() => buildDataLoadingSuggestions({ t, setInput: setPrompt, setImages: setUserImages, setAttachments: setUserAttachments, - }), [t]); + requestAutoSend: (payload) => { + if (chatMessages.length > 0) { + dispatch(dfActions.clearChatMessages()); + } + dispatch(dfActions.setDataLoadingChatPending(payload)); + }, + // eslint-disable-next-line react-hooks/exhaustive-deps + }), [t, dispatch]); const isEmpty = chatMessages.length === 0 && !streamingContent; @@ -1047,7 +1041,7 @@ export const DataLoadingChat: React.FC = ({ ) : ( <> {chatMessages.map((msg) => ( - + ))} {streamingContent !== '' && } {chatInProgress && !streamingContent && } @@ -1065,7 +1059,7 @@ export const DataLoadingChat: React.FC = ({ onChange={setPrompt} images={userImages} onImagesChange={setUserImages} - onSend={sendMessage} + onSend={() => sendMessage()} onStop={stopGeneration} inProgress={chatInProgress} placeholder={t('dataLoading.placeholder')} @@ -1076,8 +1070,13 @@ export const DataLoadingChat: React.FC = ({ formData.append('file', file); apiRequest(getUrls().SCRATCH_UPLOAD_URL, { method: 'POST', body: formData, - }).then(() => { - setUserAttachments(prev => [...prev, file.name]); + }).then(({ data }) => { + // The backend hash-suffixes the filename + // (e.g. `name_a1b2c3d4.xlsx`). Store the + // server-assigned name so the `[Uploaded:]` + // mention points to the real scratch file. + const scratchName = (data?.path || `scratch/${file.name}`).replace(/^scratch\//, ''); + setUserAttachments(prev => [...prev, scratchName]); }).catch(err => console.error('Upload failed:', err)); }} attachments={userAttachments} diff --git a/src/views/DataThread.tsx b/src/views/DataThread.tsx index ae0cc9f1..c69a71a5 100644 --- a/src/views/DataThread.tsx +++ b/src/views/DataThread.tsx @@ -1751,6 +1751,9 @@ let SingleThreadGroupView: FC<{ const TIMELINE_GAP = '4px'; // gap between timeline and card content const DOT_SIZE = 6; const CARD_PY = '6px'; // vertical padding for each timeline row + // Mirror the left timeline gutter on the right so cards sit visually + // centred in their column instead of hugging the right edge. + const CARD_CONTENT_PR = `${TIMELINE_WIDTH}px`; // CSS `border-style: dashed` stretches dashes to fit each element's // height, so stacked segments end up with mismatched dash lengths. A @@ -1907,7 +1910,7 @@ let SingleThreadGroupView: FC<{ {isLast && hasContinuationBelow && } {isLast && !hasContinuationBelow && } - + {item.element} @@ -1983,7 +1986,7 @@ let SingleThreadGroupView: FC<{ {isLast && hasContinuationBelow && } {isLast && !hasContinuationBelow && } - + {item.element} @@ -2006,7 +2009,7 @@ let SingleThreadGroupView: FC<{ {isLast && hasContinuationBelow && } {isLast && !hasContinuationBelow && } - + {item.element} @@ -2054,7 +2057,7 @@ let SingleThreadGroupView: FC<{ )} {isLast && !hasContinuationBelow && } - {item.element} @@ -3119,7 +3122,10 @@ export const DataThread: FC<{sx?: SxProps}> = function ({ sx }) { // benefit, since the segments would just stack in the same single column. const CARD_GAP = 12; // padding + spacing between cards in a column const PANEL_PADDING = 16; - const CARD_WIDTH = 220; + // 220 visual card width + 14px right gutter (CARD_CONTENT_PR) so cards + // keep their original size while gaining a right margin that balances + // the left timeline gutter. + const CARD_WIDTH = 234; const COLUMN_WIDTH = CARD_WIDTH + CARD_GAP; // n columns need: n*CARD_WIDTH + (n-1)*CARD_GAP + PANEL_PADDING // Solving for n: n <= (containerWidth - PANEL_PADDING + CARD_GAP) / COLUMN_WIDTH diff --git a/src/views/UnifiedDataUploadDialog.tsx b/src/views/UnifiedDataUploadDialog.tsx index bd7167f8..73d325e1 100644 --- a/src/views/UnifiedDataUploadDialog.tsx +++ b/src/views/UnifiedDataUploadDialog.tsx @@ -448,12 +448,14 @@ export interface DataLoadMenuProps { onSelectConnector?: (connector: ConnectorInstance) => void; /** * Called when the user submits a prompt from the top-level Data Loading - * Agent chat box. Implementations should open the agent chat surface - * with the prompt (and optional pasted/attached images) pre-filled — - * typically auto-sent. If not provided, the chat box falls back to - * `onSelectTab('extract')`. + * Agent chat box. Implementations should hand the payload off to the + * agent chat surface, which will auto-send it as a fresh user + * message. Attachments are file names (already uploaded to the + * session scratch space) — the chat surface re-injects them as + * `[Uploaded: name]` mentions when building the backend payload. + * If not provided, the chat box falls back to `onSelectTab('extract')`. */ - onStartChat?: (prompt: string, images?: string[]) => void; + onStartChat?: (prompt: string, images: string[], attachments: string[]) => void; /** * True when a prior data-loading agent conversation exists in * state. When set together with `onResumeChat`, the menu renders @@ -605,22 +607,17 @@ export const DataLoadMenu: React.FC = ({ const submitAgentChat = () => { const text = agentInput.trim(); if (text.length === 0 && agentImages.length === 0 && agentAttachments.length === 0) { - // Empty submission — just open the chat surface. - if (onStartChat) onStartChat('', []); + // Empty submission — just surface the chat. + if (onStartChat) onStartChat('', [], []); else onSelectTab('extract'); return; } - // Augment the outgoing prompt with `[Uploaded: name]` lines so the - // agent sees attachments as text references, without polluting - // the editable input the user sees. - const mentions = agentAttachments - .map(name => t('dataLoading.uploaded', { name })) - .join('\n'); - const finalText = mentions - ? (text ? `${text}\n${mentions}` : mentions) - : text; + // Pass payload pieces unchanged — the chat surface builds the + // backend mentions itself. We deliberately do NOT pre-inject + // `[Uploaded: name]` into `text` here, so the visible message + // bubble stays clean and the file chips render uniformly. if (onStartChat) { - onStartChat(finalText, agentImages); + onStartChat(text, agentImages, agentAttachments); } else { onSelectTab('extract'); } @@ -631,14 +628,26 @@ export const DataLoadMenu: React.FC = ({ // Suggestions surfaced as a focus-time dropdown — sourced from a shared // factory so the in-session `DataLoadingChat` panel renders the exact - // same list. See `dataLoadingSuggestions.ts`. + // same list. See `dataLoadingSuggestions.ts`. Auto-run is routed + // through `onStartChat` so the parent dialog can dispatch its + // `clearChatMessages` + `setDataLoadingChatPending` sequence + // atomically — same path as a manual submit. const agentChatSuggestions = useMemo(() => buildDataLoadingSuggestions({ t, setInput: setAgentInput, setImages: setAgentImages, setAttachments: setAgentAttachments, ensureActiveWorkspace, - }), [t]); + requestAutoSend: onStartChat + ? (payload) => { + onStartChat(payload.text, payload.images, payload.attachments); + setAgentInput(''); + setAgentImages([]); + setAgentAttachments([]); + } + : undefined, + // eslint-disable-next-line react-hooks/exhaustive-deps + }), [t, onStartChat]); const agentChatBox = ( = ({ formData.append('file', file); apiRequest(getUrls().SCRATCH_UPLOAD_URL, { method: 'POST', body: formData, - }).then(() => { - setAgentAttachments(prev => [...prev, file.name]); + }).then(({ data }) => { + // The backend hash-suffixes the filename; store the + // server-assigned name so the `[Uploaded:]` mention + // resolves to the real scratch file. + const scratchName = (data?.path || `scratch/${file.name}`).replace(/^scratch\//, ''); + setAgentAttachments(prev => [...prev, scratchName]); }).catch(err => console.error('Upload failed:', err)); }} attachments={agentAttachments} @@ -1112,14 +1125,6 @@ export interface UnifiedDataUploadDialogProps { open: boolean; onClose: () => void; initialTab?: UploadTabType; - /** - * Optional initial prompt to hand off to the Data Loading Agent. When - * non-empty and `initialTab === 'extract'`, the prompt is pre-filled - * and auto-sent in the chat panel. - */ - initialChatPrompt?: string; - /** Optional images (data URLs) to seed the chat alongside `initialChatPrompt`. */ - initialChatImages?: string[]; onConnectorsChanged?: () => void; } @@ -1127,8 +1132,6 @@ export const UnifiedDataUploadDialog: React.FC = ( open, onClose, initialTab = 'menu', - initialChatPrompt, - initialChatImages, onConnectorsChanged, }) => { const theme = useTheme(); @@ -1143,21 +1146,6 @@ export const UnifiedDataUploadDialog: React.FC = ( const existingNames = new Set(existingTables.map(t => t.id)); const [activeTab, setActiveTab] = useState(initialTab === 'menu' ? 'menu' : initialTab); - // Prompt to seed the agent chat with. Sourced from the `initialChatPrompt` - // prop when the dialog opens directly on 'extract', or set internally - // when the user submits the in-menu agent chat box. - const [seededChatPrompt, setSeededChatPrompt] = useState( - initialTab === 'extract' ? initialChatPrompt : undefined, - ); - const [seededChatImages, setSeededChatImages] = useState( - initialTab === 'extract' ? initialChatImages : undefined, - ); - const [autoSendSeededPrompt, setAutoSendSeededPrompt] = useState( - initialTab === 'extract' && ( - (!!initialChatPrompt && initialChatPrompt.trim().length > 0) - || (!!initialChatImages && initialChatImages.length > 0) - ), - ); const fileInputRef = useRef(null); const urlInputRef = useRef(null); @@ -1175,27 +1163,8 @@ export const UnifiedDataUploadDialog: React.FC = ( if (open) { setConnectorInstances([]); refreshConnectors(); - // Re-seed chat prompt/images from props each time the dialog opens. - if (initialTab === 'extract') { - setSeededChatPrompt(initialChatPrompt); - setSeededChatImages(initialChatImages); - const hasText = !!initialChatPrompt && initialChatPrompt.trim().length > 0; - const hasImages = !!initialChatImages && initialChatImages.length > 0; - setAutoSendSeededPrompt(hasText || hasImages); - // Opening the dialog with a fresh prompt/images means the - // user wants a new data-loading conversation; clear any - // stale messages from a previous session so the new query - // isn't appended to an unrelated thread. - if ((hasText || hasImages) && dataLoadingChatMessages.length > 0) { - dispatch(dfActions.clearChatMessages()); - } - } else { - setSeededChatPrompt(undefined); - setSeededChatImages(undefined); - setAutoSendSeededPrompt(false); - } } - }, [open, refreshConnectors, identityKey, initialTab, initialChatPrompt, initialChatImages]); + }, [open, refreshConnectors, identityKey]); // Storage is determined by backend config — no user toggle const isEphemeral = serverConfig.WORKSPACE_BACKEND === 'ephemeral'; @@ -1848,29 +1817,32 @@ export const UnifiedDataUploadDialog: React.FC = ( setActiveTab(`connector:${conn.id}` as UploadTabType); } }} - onStartChat={(prompt, images) => { + onStartChat={(prompt, images, attachments) => { const hasText = prompt.trim().length > 0; - const hasImages = !!images && images.length > 0; - // If a prior conversation exists, treat a - // new query from the menu as a fresh data - // reload and reset the chat. Without this - // the new prompt would be appended onto an - // unrelated thread, confusing the agent. - if ((hasText || hasImages) && dataLoadingChatMessages.length > 0) { - dispatch(dfActions.clearChatMessages()); + const hasImages = images.length > 0; + const hasAttachments = attachments.length > 0; + // Always surface the chat. If the user + // is starting a fresh query, clear any + // prior conversation and enqueue the new + // submission as a redux `pending` slot + // — `DataLoadingChat` consumes it on + // render and auto-sends. Doing both + // dispatches in the same tick keeps the + // handoff atomic; there's no prop race. + if (hasText || hasImages || hasAttachments) { + if (dataLoadingChatMessages.length > 0) { + dispatch(dfActions.clearChatMessages()); + } + dispatch(dfActions.setDataLoadingChatPending({ + text: prompt, images, attachments, + })); } - setSeededChatPrompt(prompt); - setSeededChatImages(images); - setAutoSendSeededPrompt(hasText || hasImages); setActiveTab('extract'); }} hasPriorConversation={dataLoadingChatMessages.length > 0} onResumeChat={() => { // Reopen the existing thread without // clearing messages or auto-sending. - setSeededChatPrompt(undefined); - setSeededChatImages(undefined); - setAutoSendSeededPrompt(false); setActiveTab('extract'); }} serverConfig={serverConfig} @@ -2403,11 +2375,7 @@ export const UnifiedDataUploadDialog: React.FC = ( {/* Extract Data Tab */} - + {/* Local Folder Tab */} diff --git a/src/views/dataLoadingSuggestions.ts b/src/views/dataLoadingSuggestions.ts index 8d91b92b..f37e04f0 100644 --- a/src/views/dataLoadingSuggestions.ts +++ b/src/views/dataLoadingSuggestions.ts @@ -22,6 +22,12 @@ export interface DataLoadingSuggestion { onClick: () => void; } +export interface SuggestionPayload { + text: string; + images: string[]; + attachments: string[]; +} + export interface BuildSuggestionsArgs { t: TFunction; setInput: (value: string) => void; @@ -29,12 +35,22 @@ export interface BuildSuggestionsArgs { setAttachments: (names: string[]) => void; /** Optional hook that workspaces use to make sure a session exists before uploading. */ ensureActiveWorkspace?: () => void; + /** + * Optional auto-run hook. When provided, suggestions submit the + * complete payload immediately (after any required async upload / + * data-URL prep) instead of just pre-filling the input. Callers + * typically wire this to a redux pending-submission dispatch so the + * payload survives the parent→child handoff without prop races. + * When absent, the suggestion behaves like a paste: it only fills + * the input fields via the `set*` callbacks. + */ + requestAutoSend?: (payload: SuggestionPayload) => void; } const EXCEL_SAMPLE_NAME = 'climate-gas-indicator.xlsx'; export function buildDataLoadingSuggestions( - { t, setInput, setImages, setAttachments, ensureActiveWorkspace }: BuildSuggestionsArgs, + { t, setInput, setImages, setAttachments, ensureActiveWorkspace, requestAutoSend }: BuildSuggestionsArgs, ): DataLoadingSuggestion[] { const kindAsk = t('upload.agentChatSuggestion.kind.ask', { defaultValue: 'ask' }); const kindFind = t('upload.agentChatSuggestion.kind.find', { defaultValue: 'find' }); @@ -61,37 +77,38 @@ export function buildDataLoadingSuggestions( const iconSx = { fontSize: 14 }; + // Common: fill the input fields AND (if auto-run is enabled) submit + // the payload. Centralising the dual behaviour keeps every + // suggestion below short and consistent. + const fillAndMaybeSend = (payload: SuggestionPayload) => { + setImages(payload.images); + setAttachments(payload.attachments); + setInput(payload.text); + requestAutoSend?.(payload); + }; + return [ { kind: kindAsk, label: askLabel, icon: React.createElement(QuestionAnswerOutlinedIcon, { sx: iconSx }), - onClick: () => { - setImages([]); - setAttachments([]); - setInput(askLabel); - }, + onClick: () => fillAndMaybeSend({ text: askLabel, images: [], attachments: [] }), }, { kind: kindFind, label: findLabel, icon: React.createElement(SearchIcon, { sx: iconSx }), - onClick: () => { - setImages([]); - setAttachments([]); - setInput(findLabel); - }, + onClick: () => fillAndMaybeSend({ text: findLabel, images: [], attachments: [] }), }, { kind: kindExtract, label: extractExcelLabel, icon: React.createElement(TableChartOutlinedIcon, { sx: iconSx }), onClick: () => { - // Surface the attachment chip synchronously so it is - // always present when the user hits send, even if the - // upload below is still mid-flight. The chip is what - // gets serialised into the outgoing `[Uploaded: name]` - // mention and ultimately the chat bubble. + // Surface the attachment chip / input synchronously so + // it is visible during the async upload. The auto-send + // (if enabled) waits until the upload completes so the + // backend can actually find the scratch file. setImages([]); setAttachments([EXCEL_SAMPLE_NAME]); setInput(extractExcelLabel); @@ -108,6 +125,18 @@ export function buildDataLoadingSuggestions( method: 'POST', body: formData, }); }) + .then(({ data }) => { + // The backend hash-suffixes the filename, so use the + // server-assigned name for the chip and the mention + // — otherwise the agent looks for a file that the + // upload renamed and reports it missing. + const scratchName = (data?.path || `scratch/${EXCEL_SAMPLE_NAME}`).replace(/^scratch\//, ''); + setAttachments([scratchName]); + requestAutoSend?.({ + text: extractExcelLabel, images: [], + attachments: [scratchName], + }); + }) .catch(err => console.error('Sample Excel upload failed:', err)); }, }, @@ -116,16 +145,21 @@ export function buildDataLoadingSuggestions( label: extractImageLabel, icon: React.createElement(ImageOutlinedIcon, { sx: iconSx }), onClick: () => { + // Image needs to be read into a data URL before we can + // surface it as a chip or send it. Defer auto-send until + // the FileReader resolves. fetch(exampleImageTable) .then(res => res.blob()) .then(blob => { const reader = new FileReader(); reader.onload = () => { - if (reader.result) { - setImages([reader.result as string]); - setAttachments([]); - setInput(extractImageLabel); - } + if (!reader.result) return; + const dataUrl = reader.result as string; + fillAndMaybeSend({ + text: extractImageLabel, + images: [dataUrl], + attachments: [], + }); }; reader.readAsDataURL(blob); }); @@ -135,11 +169,7 @@ export function buildDataLoadingSuggestions( kind: kindExtract, label: extractTextLabel, icon: React.createElement(DescriptionOutlinedIcon, { sx: iconSx }), - onClick: () => { - setImages([]); - setAttachments([]); - setInput(extractTextPrompt); - }, + onClick: () => fillAndMaybeSend({ text: extractTextPrompt, images: [], attachments: [] }), }, ]; } diff --git a/tests/backend/data/test_workspace_manager.py b/tests/backend/data/test_workspace_manager.py index e2a00eb7..d82766c6 100644 --- a/tests/backend/data/test_workspace_manager.py +++ b/tests/backend/data/test_workspace_manager.py @@ -376,6 +376,13 @@ def test_legacy_workspace_with_only_yaml_appears_in_list(self, manager): yaml.safe_dump({"version": "1.1", "tables": {}}), encoding="utf-8", ) + # Pretend the legacy workspace had session state with tables. + (ws_dir / "session_state.json").write_text( + json.dumps({"tables": [{"id": "t1"}]}), + encoding="utf-8", + ) + # Trigger meta repair with a non-empty table count. + manager.save_session_state("legacy_ws", {"tables": [{"id": "t1"}]}) ws_list = manager.list_workspaces() ids = [w["id"] for w in ws_list] @@ -385,16 +392,23 @@ def test_legacy_workspace_with_only_yaml_appears_in_list(self, manager): assert (ws_dir / WORKSPACE_META_FILENAME).exists() def test_legacy_workspace_with_only_session_state_appears_in_list(self, manager): - """A directory with only session_state.json should be auto-repaired.""" + """A directory with session_state.json (containing tables) is + auto-repaired and visible in list_workspaces. The displayName + is inferred from session_state.""" ws_dir = manager.root / "state_only" ws_dir.mkdir(parents=True) (ws_dir / "session_state.json").write_text( json.dumps({ - "tables": [], + "tables": [{"id": "t1", "name": "T1"}], "activeWorkspace": {"displayName": "My Old Session"}, }), encoding="utf-8", ) + # Re-save so meta is written with tableCount > 0. + manager.save_session_state("state_only", { + "tables": [{"id": "t1", "name": "T1"}], + "activeWorkspace": {"displayName": "My Old Session"}, + }) ws_list = manager.list_workspaces() ids = [w["id"] for w in ws_list] @@ -405,7 +419,9 @@ def test_legacy_workspace_with_only_session_state_appears_in_list(self, manager) assert entry["display_name"] == "My Old Session" def test_legacy_workspace_with_empty_dir_appears_in_list(self, manager): - """Even a bare directory (no metadata files at all) should be listed.""" + """A bare directory with no metadata at all is auto-repaired by + _ensure_meta (meta.json gets created with fallback displayName) + and appears in list_workspaces.""" ws_dir = manager.root / "bare" ws_dir.mkdir(parents=True) @@ -413,7 +429,7 @@ def test_legacy_workspace_with_empty_dir_appears_in_list(self, manager): ids = [w["id"] for w in ws_list] assert "bare" in ids - # workspace_meta.json auto-created with fallback displayName = dir name + # Auto-repair created the meta with a fallback displayName. meta = json.loads((ws_dir / WORKSPACE_META_FILENAME).read_text(encoding="utf-8")) assert meta["displayName"] == "bare" @@ -452,3 +468,43 @@ def test_move_legacy_workspace_auto_repairs_meta(self, tmp_path): # Destination should have workspace_meta.json dst_ws = dst.get_workspace_path("old_ws") assert (dst_ws / WORKSPACE_META_FILENAME).exists() + + +class TestEmptyWorkspaceVisibility: + """list_workspaces() lists every workspace directory, including + empty "Untitled Session" entries from abandoned data-loading + chats. Users manage (rename/delete) these themselves via the + sidebar — they are not hidden.""" + + def test_empty_workspace_is_visible(self, manager): + manager.create_workspace("ghost") + # No save_session_state — meta has no tableCount/chartCount. + + ws_list = manager.list_workspaces() + + assert any(w["id"] == "ghost" for w in ws_list) + assert manager.workspace_exists("ghost") + + def test_workspace_with_tables_is_visible(self, manager): + manager.create_workspace("real") + manager.save_session_state("real", { + "tables": [{"id": "t1", "name": "T1"}], + "activeWorkspace": {"id": "real", "displayName": "Real"}, + }) + + ws_list = manager.list_workspaces() + + assert any(w["id"] == "real" for w in ws_list) + + def test_zero_count_workspace_is_visible(self, manager): + """A workspace whose tables were all deleted (zero tables) still + appears in the list — the user decides whether to remove it.""" + manager.create_workspace("emptied") + manager.save_session_state("emptied", { + "tables": [], + "activeWorkspace": {"id": "emptied", "displayName": "Emptied"}, + }) + + ws_list = manager.list_workspaces() + + assert any(w["id"] == "emptied" for w in ws_list) From 18cf3603dfd7da2197f4ba131e80ecb2fab285f3 Mon Sep 17 00:00:00 2001 From: Chenglong Wang Date: Thu, 28 May 2026 22:21:14 -0700 Subject: [PATCH 02/40] small fixes --- src/views/DataFormulator.tsx | 4 +-- src/views/DataSourceSidebar.tsx | 64 ++++++++------------------------- 2 files changed, 17 insertions(+), 51 deletions(-) diff --git a/src/views/DataFormulator.tsx b/src/views/DataFormulator.tsx index 00e8086e..b340a477 100644 --- a/src/views/DataFormulator.tsx +++ b/src/views/DataFormulator.tsx @@ -571,7 +571,7 @@ export const DataFormulatorFC = ({ }) => { const fixedSplitPane = ( openUploadDialog((tab ?? 'add-connection') as UploadTabType)} + onOpenUploadDialog={(tab) => openUploadDialog((tab ?? 'menu') as UploadTabType)} connectorRefreshKey={connectorRefreshKey} /> { {tables.length > 0 ? fixedSplitPane : ( openUploadDialog((tab ?? 'add-connection') as UploadTabType)} + onOpenUploadDialog={(tab) => openUploadDialog((tab ?? 'menu') as UploadTabType)} connectorRefreshKey={connectorRefreshKey} /> {dataUploadRequestBox} diff --git a/src/views/DataSourceSidebar.tsx b/src/views/DataSourceSidebar.tsx index a8c4715d..7381a423 100644 --- a/src/views/DataSourceSidebar.tsx +++ b/src/views/DataSourceSidebar.tsx @@ -42,7 +42,6 @@ import { VirtualizedCatalogTree } from '../components/VirtualizedCatalogTree'; import StorageIcon from '@mui/icons-material/Storage'; import AddIcon from '@mui/icons-material/Add'; -import FileUploadOutlinedIcon from '@mui/icons-material/FileUploadOutlined'; import FolderOpenIcon from '@mui/icons-material/FolderOpen'; import FolderOutlinedIcon from '@mui/icons-material/FolderOutlined'; import UploadFileIcon from '@mui/icons-material/UploadFile'; @@ -51,9 +50,6 @@ import ChevronLeftIcon from '@mui/icons-material/ChevronLeft'; import ExpandMoreIcon from '@mui/icons-material/ExpandMore'; import ChevronRightIcon from '@mui/icons-material/ChevronRight'; import RefreshIcon from '@mui/icons-material/Refresh'; -import ContentPasteOutlinedIcon from '@mui/icons-material/ContentPasteOutlined'; -import SmartToyOutlinedIcon from '@mui/icons-material/SmartToyOutlined'; -import LinkOutlinedIcon from '@mui/icons-material/LinkOutlined'; import LinkOffOutlinedIcon from '@mui/icons-material/LinkOffOutlined'; import DeleteOutlineIcon from '@mui/icons-material/DeleteOutline'; import EditOutlinedIcon from '@mui/icons-material/EditOutlined'; @@ -159,7 +155,7 @@ export const DataSourceSidebar: React.FC<{ // built-in sample_datasets connector is shown there, giving users // something useful to explore immediately. The upgrade message only // appears when they try to add a new connector or link a folder. - const [initialTab, setInitialTab] = useState<'upload' | 'sources' | 'sessions' | 'knowledge'>('sources'); + const [initialTab, setInitialTab] = useState<'sources' | 'sessions' | 'knowledge'>('sources'); // External callers (e.g. SaveExperienceButton on success) can ask the // sidebar to open and switch to a specific tab. @@ -277,6 +273,18 @@ export const DataSourceSidebar: React.FC<{ pt: 1, gap: 0.5, }}> + {/* Primary action — adding data is the main task. Styled like + the view-switcher icons but kept in primary color as a + subtle cue; opens the upload dialog (landing menu). */} + + onOpenUploadDialog?.()} sx={{ + color: 'primary.main', + borderRadius: 1, + '&:hover': { bgcolor: 'action.hover' }, + }}> + + + { setInitialTab('sessions'); if (!isOpen) toggle(); else if (initialTab !== 'sessions') setInitialTab('sessions'); else toggle(); }} sx={{ color: isOpen && initialTab === 'sessions' ? 'primary.main' : 'text.secondary', @@ -295,15 +303,6 @@ export const DataSourceSidebar: React.FC<{ - - { setInitialTab('upload'); if (!isOpen) toggle(); else if (initialTab !== 'upload') setInitialTab('upload'); else toggle(); }} sx={{ - color: isOpen && initialTab === 'upload' ? 'primary.main' : 'text.secondary', - bgcolor: isOpen && initialTab === 'upload' ? 'action.selected' : 'transparent', - borderRadius: 1, - }}> - - - { setInitialTab('knowledge'); if (!isOpen) toggle(); else if (initialTab !== 'knowledge') setInitialTab('knowledge'); else toggle(); }} sx={{ color: isOpen && initialTab === 'knowledge' ? 'primary.main' : 'text.secondary', @@ -347,7 +346,7 @@ const DataSourceSidebarPanel: React.FC<{ panelWidth: number; onOpenUploadDialog?: (tab?: string) => void; onCollapse: () => void; - initialTab?: 'upload' | 'sources' | 'sessions' | 'knowledge'; + initialTab?: 'sources' | 'sessions' | 'knowledge'; connectorRefreshKey?: number; disableConnectors?: boolean; }> = ({ panelWidth, onOpenUploadDialog, onCollapse, initialTab = 'sources', connectorRefreshKey = 0, disableConnectors = false }) => { @@ -419,7 +418,7 @@ const DataSourceSidebarPanel: React.FC<{ const [searchingCatalog, setSearchingCatalog] = useState>({}); // Sidebar tab: 'sources' or 'sessions' or 'knowledge' - const [activeTab, setActiveTab] = useState<'upload' | 'sources' | 'sessions' | 'knowledge'>(initialTab); + const [activeTab, setActiveTab] = useState<'sources' | 'sessions' | 'knowledge'>(initialTab); // Sync tab when rail icon switches it useEffect(() => { @@ -1292,39 +1291,6 @@ const DataSourceSidebarPanel: React.FC<{ overflow: 'hidden', }}> - {/* ── Upload Data tab ── */} - {activeTab === 'upload' && ( - - - - {t('sidebar.uploadData', { defaultValue: 'Upload Data' })} - - - - - - - - - {[ - { icon: , label: t('upload.uploadFile', { defaultValue: 'Upload file' }), tab: 'upload' }, - { icon: , label: t('upload.pasteData', { defaultValue: 'Paste data' }), tab: 'paste' }, - { icon: , label: t('upload.extractData', { defaultValue: 'Data Assistant' }), tab: 'extract' }, - { icon: , label: t('upload.loadFromUrl', { defaultValue: 'Load from URL' }), tab: 'url' }, - ].map((item, i) => ( - onOpenUploadDialog?.(item.tab)} - sx={{ display: 'flex', alignItems: 'center', gap: 0.75, px: 1.5, py: 0.75, cursor: 'pointer', color: 'text.primary', '&:hover': { bgcolor: 'action.hover' }, userSelect: 'none' }} - > - {item.icon} - {item.label} - - ))} - - - )} - {/* ── Data Connectors tab ── Sample datasets remain available even when external connectors are disabled; the Add Connector / Link Folder From 4cb0a2f4f5e32bd5132a84b77cdcb3cc12f50f56 Mon Sep 17 00:00:00 2001 From: Chenglong Wang Date: Thu, 28 May 2026 22:50:07 -0700 Subject: [PATCH 03/40] cleanup --- src/views/ChartRecBox.tsx | 14 +++++++------- src/views/SimpleChartRecBox.tsx | 34 +++++---------------------------- 2 files changed, 12 insertions(+), 36 deletions(-) diff --git a/src/views/ChartRecBox.tsx b/src/views/ChartRecBox.tsx index 872dc3c1..9eb26085 100644 --- a/src/views/ChartRecBox.tsx +++ b/src/views/ChartRecBox.tsx @@ -292,10 +292,10 @@ export const ChartRecBox: FC = function ({ tableId, placeHolde type={current ? undefined : 'button'} onClick={current ? undefined : () => dispatch(dfActions.setFocused({ type: 'table', tableId: table.id }))} sx={{ - display: 'inline-flex', alignItems: 'center', gap: current ? '6px' : '3px', + display: 'inline-flex', alignItems: 'center', gap: '3px', border: 'none', background: 'transparent', p: 0, fontFamily: theme.typography.fontFamily, - fontSize: current ? 16 : 11, lineHeight: 1.4, + fontSize: 11, lineHeight: 1.4, color: current ? 'primary.main' : 'text.secondary', fontWeight: current ? 600 : 400, cursor: current ? 'default' : 'pointer', @@ -304,7 +304,7 @@ export const ChartRecBox: FC = function ({ tableId, placeHolde '&:hover': current ? undefined : { color: 'primary.main' }, }} > - + {table.displayId} ); @@ -682,10 +682,10 @@ export const ChartRecBox: FC = function ({ tableId, placeHolde ); }; - // Center cluster auto-scales with chart count; neighbour - // clusters are halved and dimmed to read as context. - const centerN = Math.min(chartsForTable(currentTable.id).length, 8); - const centerScale = centerN <= 3 ? 1 : centerN <= 5 ? 0.82 : 0.66; + // All clusters render at the same scale; the current + // cluster is only distinguished by not being dimmed and by + // showing more thumbnails. + const centerScale = 0.5; const sideScale = 0.5; return ( diff --git a/src/views/SimpleChartRecBox.tsx b/src/views/SimpleChartRecBox.tsx index c96dec65..5e7bd63b 100644 --- a/src/views/SimpleChartRecBox.tsx +++ b/src/views/SimpleChartRecBox.tsx @@ -41,7 +41,7 @@ import StopIcon from '@mui/icons-material/Stop'; import AutoGraphIcon from '@mui/icons-material/AutoGraph'; import DescriptionOutlinedIcon from '@mui/icons-material/DescriptionOutlined'; import { UnifiedDataUploadDialog } from './UnifiedDataUploadDialog'; -import { transition } from '../app/tokens'; +import { borderColor, transition } from '../app/tokens'; import { Theme } from '@mui/material/styles'; import { useTranslation } from 'react-i18next'; import { shouldAutoFocusGeneratedChart } from '../app/agentInteractionPolicy'; @@ -1380,12 +1380,6 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ }, [pendingClarification, dispatch, t]); const isReportMode = selectedAgent === 'report'; - const gradientBorder = isReportMode - ? `linear-gradient(135deg, ${alpha(theme.palette.warning.main, 0.6)}, ${alpha(theme.palette.warning.dark, 0.5)})` - : `linear-gradient(135deg, ${alpha(theme.palette.primary.main, 0.6)}, ${alpha(theme.palette.secondary.main, 0.55)})`; - const workingBorder = isReportMode - ? `linear-gradient(135deg, ${alpha(theme.palette.warning.main, 0.3)}, ${alpha(theme.palette.warning.dark, 0.25)})` - : `linear-gradient(135deg, ${alpha(theme.palette.primary.main, 0.3)}, ${alpha(theme.palette.secondary.main, 0.25)})`; // Landing / "no thread yet" highlight: when the user has loaded data // but hasn't started an exploration on the focused table (no real @@ -1419,12 +1413,9 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ mx: 1, mb: 1, mt: 0.5, px: 1.25, pt: 1, pb: 0.5, borderRadius: '12px', - // The 2-tone border is drawn by the `::before` gradient - // overlay below (works through border-radius + masks). We - // intentionally leave the Card's own border off so the two - // don't fight; focus state uses a shadow halo instead of a - // border-color shift. - border: 'none', + // Standard single-tone input style (matches AgentChatInput): a + // solid divider border that turns the accent color on focus. + border: `1px solid ${borderColor.divider}`, outline: 'none', position: 'relative', overflow: isChatFormulating ? 'hidden' : 'visible', @@ -1454,24 +1445,9 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ } : {}), '&:focus-within': { animation: 'none', + borderColor: isReportMode ? theme.palette.warning.main : theme.palette.primary.main, boxShadow: `0 0 0 2px ${alpha(isReportMode ? theme.palette.warning.main : theme.palette.primary.main, 0.15)}, 0 2px 10px rgba(32, 33, 36, 0.14)`, }, - // Gradient border via pseudo-element (works with border-radius) - '&::before': { - content: '""', - position: 'absolute', - inset: 0, - borderRadius: 'inherit', - padding: '1.5px', - background: isChatFormulating - ? workingBorder - : gradientBorder, - WebkitMask: 'linear-gradient(#fff 0 0) content-box, linear-gradient(#fff 0 0)', - WebkitMaskComposite: 'xor', - maskComposite: 'exclude', - pointerEvents: 'none', - zIndex: 3, - }, }} > {clarificationQuestions?.kind === 'clarification' && clarificationQuestions.questions && pendingClarification && !isChatFormulating && ( From c616338d67c4c12cb7290139ce046213abc44098 Mon Sep 17 00:00:00 2001 From: Chenglong Wang Date: Fri, 29 May 2026 00:06:50 -0700 Subject: [PATCH 04/40] some updates --- src/views/DataView.tsx | 92 +++++++++++++++++++++++++++++++-- src/views/VisualizationView.tsx | 20 ++++--- 2 files changed, 98 insertions(+), 14 deletions(-) diff --git a/src/views/DataView.tsx b/src/views/DataView.tsx index aa1263d0..f6c4a79f 100644 --- a/src/views/DataView.tsx +++ b/src/views/DataView.tsx @@ -2,11 +2,15 @@ // Licensed under the MIT License. import React, { FC, useEffect, useMemo, useCallback } from 'react'; +import ReactDOM from 'react-dom'; import _ from 'lodash'; -import { Typography, Box, Link, Breadcrumbs, useTheme, Fade } from '@mui/material'; +import { Typography, Box, Link, Breadcrumbs, useTheme, Fade, IconButton, Tooltip } from '@mui/material'; import { alpha } from '@mui/material/styles'; +import { useTranslation } from 'react-i18next'; +import OpenInFullIcon from '@mui/icons-material/OpenInFull'; +import CloseFullscreenIcon from '@mui/icons-material/CloseFullscreen'; import '../scss/DataView.scss'; @@ -16,11 +20,19 @@ import { useDispatch, useSelector } from 'react-redux'; import { Type } from '../data/types'; import { SelectableDataGrid } from './SelectableDataGrid'; import { formatCellValue, getColumnAlign } from './ViewUtils'; +import { borderColor } from '../app/tokens'; export interface FreeDataViewProps { + // When true, render a maximize/restore toggle that pops the table into a + // full-canvas overlay. Used wherever the grid is shown inline (under a + // chart, or as the focused-table preview). + maximizable?: boolean; } -export const FreeDataViewFC: FC = function DataView() { +export const FreeDataViewFC: FC = function DataView({ maximizable }) { + + const { t } = useTranslation(); + const [maximized, setMaximized] = React.useState(false); const dispatch = useDispatch(); @@ -32,6 +44,7 @@ export const FreeDataViewFC: FC = function DataView() { const focusedTableId = useMemo(() => { if (!focusedId) return undefined; if (focusedId.type === 'table') return focusedId.tableId; + if (focusedId.type !== 'chart') return undefined; const chartId = focusedId.chartId; const chart = allCharts.find(c => c.id === chartId); return chart?.tableRef; @@ -108,7 +121,7 @@ export const FreeDataViewFC: FC = function DataView() { ]; }, [targetTable, rowData, conceptShelfItems]); - return ( + const grid = ( @@ -124,4 +137,77 @@ export const FreeDataViewFC: FC = function DataView() { ); + + if (!maximizable) { + return grid; + } + + const toggleButton = ( + + setMaximized(m => !m)} + sx={{ + color: 'text.secondary', + '&:hover': { color: 'primary.main', backgroundColor: 'transparent' }, + }} + > + {maximized ? : } + + + ); + + // The toggle button sits just outside the table to the right (a slim panel), + // so it never overlaps the column headers and the card keeps its original look. + // In maximized mode the surrounding overlay already provides the card frame. + const cardSx = maximized ? { overflow: 'hidden' } : { + overflow: 'hidden', + borderRadius: '8px', + border: `1px solid ${borderColor.divider}`, + transition: 'box-shadow 0.2s ease', + '&:hover': { boxShadow: '0 0 8px rgba(25, 118, 210, 0.25)' }, + }; + const framed = ( + + + {grid} + + + {toggleButton} + + + ); + + if (maximized) { + const canvas = typeof document !== 'undefined' ? document.getElementById('vis-view-canvas') : null; + const overlay = ( + <> + {/* Transparent click-catcher — click outside to restore. Scoped to the visualization view. */} + setMaximized(false)} + sx={{ position: 'absolute', inset: 0, zIndex: 1299 }} + /> + {/* Table overlay filling the visualization view. */} + + {framed} + + + ); + return ( + <> + {/* Keep the inline slot occupied so surrounding layout doesn't jump. */} + + {canvas ? ReactDOM.createPortal(overlay, canvas) : overlay} + + ); + } + + return framed; } \ No newline at end of file diff --git a/src/views/VisualizationView.tsx b/src/views/VisualizationView.tsx index 96d91d8c..7b6d18b4 100644 --- a/src/views/VisualizationView.tsx +++ b/src/views/VisualizationView.tsx @@ -932,11 +932,12 @@ export const ChartEditorFC: FC<{}> = function ChartEditorFC({}) { return sum + Math.max(80, Math.min(280, contentLen * 10)) + 60; }, ROW_ID_COL_WIDTH); const SCROLLBAR_WIDTH = 17; - const adaptiveWidth = Math.max(MIN_TABLE_WIDTH, Math.min(MAX_TABLE_WIDTH, totalColWidth + SCROLLBAR_WIDTH + 16)); + // +34px gutter so the maximize button can sit just outside the table on the right. + const adaptiveWidth = Math.max(MIN_TABLE_WIDTH, Math.min(MAX_TABLE_WIDTH, totalColWidth + SCROLLBAR_WIDTH + 16)) + 34; return ( - - + + ); })()} @@ -1096,7 +1097,7 @@ export const ChartEditorFC: FC<{}> = function ChartEditorFC({}) { , [localScaleFactor, t]); - return + return {synthesisRunning ? = function VisualizationView } return ( - + @@ -1281,18 +1282,15 @@ export const VisualizationViewFC: FC = function VisualizationView return sum + Math.max(80, Math.min(280, contentLen * 10)) + 60; }, ROW_ID_COL_WIDTH); const SCROLLBAR_WIDTH = 17; - const adaptiveWidth = Math.max(MIN_TABLE_WIDTH, Math.min(MAX_TABLE_WIDTH, totalColWidth + SCROLLBAR_WIDTH + 16)); + // +34px gutter so the maximize button can sit just outside the table on the right. + const adaptiveWidth = Math.max(MIN_TABLE_WIDTH, Math.min(MAX_TABLE_WIDTH, totalColWidth + SCROLLBAR_WIDTH + 16)) + 34; return ( - + ); })()} From a59ec9e04fd3c0c447f3fbffbe2a0f0573b7dd04 Mon Sep 17 00:00:00 2001 From: Chenglong Wang Date: Fri, 29 May 2026 09:25:24 -0700 Subject: [PATCH 05/40] some cleanup --- src/views/DataFormulator.tsx | 22 ++++---- src/views/DataThread.tsx | 14 ++--- src/views/SimpleChartRecBox.tsx | 98 ++++++++++++++++++++++++++------- src/views/threadLayout.ts | 39 +++++++++++++ 4 files changed, 133 insertions(+), 40 deletions(-) create mode 100644 src/views/threadLayout.ts diff --git a/src/views/DataFormulator.tsx b/src/views/DataFormulator.tsx index b340a477..90547525 100644 --- a/src/views/DataFormulator.tsx +++ b/src/views/DataFormulator.tsx @@ -40,6 +40,7 @@ import { DndProvider } from 'react-dnd' import { HTML5Backend } from 'react-dnd-html5-backend' import { toolName } from '../app/App'; import { DataThread } from './DataThread'; +import { threadPaneWidth } from './threadLayout'; import dfLogo from '../assets/df-logo.png'; import exampleImageTable from "../assets/example-image-table.png"; @@ -443,12 +444,9 @@ export const DataFormulatorFC = ({ }) => { //boxShadow: '0 0 5px rgba(0,0,0,0.1)', } - // Discrete column snapping for DataThread - const CARD_WIDTH = 220; - const CARD_GAP = 12; - const COLUMN_WIDTH = CARD_WIDTH + CARD_GAP; - const PANE_PADDING = 48; - const columnSize = (n: number) => n * COLUMN_WIDTH + PANE_PADDING; + // Discrete column snapping for DataThread. + // Column geometry is defined once in ./threadLayout and shared with + // DataThread so the pane snap points line up with the rendered columns. const allotmentRef = useRef(null); const containerRef = useRef(null); @@ -459,13 +457,13 @@ export const DataFormulatorFC = ({ }) => { let bestCols = 1; let bestDist = Infinity; for (let n = 1; n <= 3; n++) { - const dist = Math.abs(raw - columnSize(n)); + const dist = Math.abs(raw - threadPaneWidth(n)); if (dist < bestDist) { bestDist = dist; bestCols = n; } } - const snapped = columnSize(bestCols); + const snapped = threadPaneWidth(bestCols); if (Math.abs(raw - snapped) > 2) { const totalWidth = sizes.reduce((a, b) => a + b, 0); allotmentRef.current.resize([snapped, totalWidth - snapped]); @@ -545,10 +543,10 @@ export const DataFormulatorFC = ({ }) => { let newSize: number | null = null; if (prev <= 1 && threadCount > 1) { // Case 1: was 1 thread, now 2+ → expand to 2 columns - newSize = columnSize(2); + newSize = threadPaneWidth(2); } else if (prev > 1 && threadCount <= 1) { // Case 2: was 2+ threads, now 1 → shrink to 1 column - newSize = columnSize(1); + newSize = threadPaneWidth(1); } // Case 3: was 2+ threads and still 2+ → don't change (respect user's manual setting) @@ -581,7 +579,9 @@ export const DataFormulatorFC = ({ }) => { position: 'relative'}}> {tables.length > 0 ? ( - + = function ({ sx }) { // only one column fits, splitting a long thread into segments adds visual // overhead (continuation headers + ghost parents) without any layout // benefit, since the segments would just stack in the same single column. - const CARD_GAP = 12; // padding + spacing between cards in a column - const PANEL_PADDING = 16; - // 220 visual card width + 14px right gutter (CARD_CONTENT_PR) so cards - // keep their original size while gaining a right margin that balances - // the left timeline gutter. - const CARD_WIDTH = 234; - const COLUMN_WIDTH = CARD_WIDTH + CARD_GAP; - // n columns need: n*CARD_WIDTH + (n-1)*CARD_GAP + PANEL_PADDING - // Solving for n: n <= (containerWidth - PANEL_PADDING + CARD_GAP) / COLUMN_WIDTH - const fittableColumns = Math.max(1, Math.min(3, Math.floor((containerWidth - PANEL_PADDING + CARD_GAP) / COLUMN_WIDTH))); + // Column geometry (CARD_WIDTH / CARD_GAP / PANEL_PADDING) is defined once + // in ./threadLayout and shared with DataFormulator's pane snapping. + const fittableColumns = fittableThreadColumns(containerWidth); // Adaptively split long derivation chains so the resulting segments fill // the available columns evenly. See `computeSplitExtraLeaves` for the diff --git a/src/views/SimpleChartRecBox.tsx b/src/views/SimpleChartRecBox.tsx index 5e7bd63b..23b30c7c 100644 --- a/src/views/SimpleChartRecBox.tsx +++ b/src/views/SimpleChartRecBox.tsx @@ -40,7 +40,7 @@ import StopIcon from '@mui/icons-material/Stop'; import AutoGraphIcon from '@mui/icons-material/AutoGraph'; import DescriptionOutlinedIcon from '@mui/icons-material/DescriptionOutlined'; -import { UnifiedDataUploadDialog } from './UnifiedDataUploadDialog'; +import InsertDriveFileOutlinedIcon from '@mui/icons-material/InsertDriveFileOutlined'; import { borderColor, transition } from '../app/tokens'; import { Theme } from '@mui/material/styles'; import { useTranslation } from 'react-i18next'; @@ -151,7 +151,8 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ const [mentionHighlightIdx, setMentionHighlightIdx] = useState(0); const [selectedAgent, setSelectedAgent] = useState<'explore' | 'report'>('explore'); const [attachedImages, setAttachedImages] = useState([]); - const [uploadDialogOpen, setUploadDialogOpen] = useState(false); + const [attachedFiles, setAttachedFiles] = useState<{ name: string; content: string }[]>([]); + const fileInputRef = useRef(null); const agentAbortRef = useRef(null); const userChartFocusLockedRef = useRef(false); const lastAutoFocusedChartIdRef = useRef(null); @@ -296,6 +297,31 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ } }, []); + // Attach files as conversation context. Images become reference images + // (sent to the model as attachments); text-like files are read as text + // and folded into the agent prompt as context. + const handleAttachFiles = React.useCallback((fileList: FileList | null) => { + if (!fileList) return; + const MAX_TEXT_CHARS = 50000; + Array.from(fileList).forEach(file => { + if (file.type.startsWith('image/')) { + const reader = new FileReader(); + reader.onload = () => setAttachedImages(prev => [...prev, reader.result as string]); + reader.readAsDataURL(file); + } else { + const reader = new FileReader(); + reader.onload = () => { + let content = (reader.result as string) || ''; + if (content.length > MAX_TEXT_CHARS) { + content = content.slice(0, MAX_TEXT_CHARS) + '\n…[truncated]'; + } + setAttachedFiles(prev => [...prev, { name: file.name, content }]); + }; + reader.readAsText(file); + } + }); + }, []); + // Collect table IDs from root up to (and including) the focused table for agent action matching const threadTableIds = React.useMemo(() => { if (!focusedTableId) return new Set(); @@ -373,6 +399,14 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ }, displayPrompt?: string) => { if (!focusedTableId || (!clarificationContext && prompt.trim() === "")) return; + // Fold attached reference files into the prompt the agent sees, while + // keeping the timeline bubble (displayContent) clean for the user. + const fileContext = attachedFiles.length > 0 + ? '\n\n' + attachedFiles.map(f => `[Attached file: ${f.name}]\n${f.content}`).join('\n\n') + : ''; + const agentPrompt = prompt + fileContext; + const cleanDisplay = displayPrompt ?? (fileContext ? prompt : undefined); + const rootTables = tables.filter(t => t.derive === undefined || t.anchored); const currentTable = tables.find(t => t.id === focusedTableId); const priorityIds = (currentTable?.derive && !currentTable.anchored) @@ -404,8 +438,8 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ // 'clarifying' status and pendingClarification storage. if (isResume && pendingClarification?.draftId) { dispatch(dfActions.appendDraftInteraction({ draftId: pendingClarification.draftId, entry: { - from: 'user', to: 'data-agent', role: 'prompt', content: prompt, - ...(displayPrompt ? { displayContent: displayPrompt } : {}), + from: 'user', to: 'data-agent', role: 'prompt', content: agentPrompt, + ...(cleanDisplay ? { displayContent: cleanDisplay } : {}), timestamp: Date.now() }})); dispatch(dfActions.updateDraftClarification({ draftId: pendingClarification.draftId, pendingClarification: null })); @@ -552,10 +586,10 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ // backend appends it to the trajectory as a normal user message. // No special clarification payload needed. requestBody.trajectory = clarificationContext!.trajectory; - requestBody.user_question = prompt; + requestBody.user_question = agentPrompt; requestBody.completed_step_count = clarificationContext!.completedStepCount; } else { - requestBody.user_question = prompt; + requestBody.user_question = agentPrompt; if (focusedThread) requestBody.focused_thread = focusedThread; if (otherThreads) requestBody.other_threads = otherThreads; } @@ -603,13 +637,13 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ currentDraftParentTableId = existingDraft?.derive?.trigger?.tableId || null; currentDraftInteraction = [...(existingDraft?.derive?.trigger?.interaction || [])]; // The user reply was already appended above, add to local accumulator too - currentDraftInteraction.push({ from: 'user', to: 'data-agent', role: 'prompt', content: prompt, - ...(displayPrompt ? { displayContent: displayPrompt } : {}), + currentDraftInteraction.push({ from: 'user', to: 'data-agent', role: 'prompt', content: agentPrompt, + ...(cleanDisplay ? { displayContent: cleanDisplay } : {}), timestamp: Date.now() }); } else { const initialEntries: InteractionEntry[] = [ - { from: 'user', to: 'data-agent', role: 'prompt', content: prompt, - ...(displayPrompt ? { displayContent: displayPrompt } : {}), + { from: 'user', to: 'data-agent', role: 'prompt', content: agentPrompt, + ...(cleanDisplay ? { displayContent: cleanDisplay } : {}), timestamp: Date.now() } ]; createNextDraft(lastCreatedTableId || focusedTableId!, initialEntries); @@ -940,6 +974,7 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ clearTimeout(timeoutId); setChatPrompt(""); setAttachedImages([]); + setAttachedFiles([]); isCompleted = true; } @@ -988,6 +1023,7 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ clearTimeout(timeoutId); setChatPrompt(""); setAttachedImages([]); + setAttachedFiles([]); isCompleted = true; } @@ -1028,6 +1064,7 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ if (completionResult) { setChatPrompt(""); setAttachedImages([]); + setAttachedFiles([]); } }; @@ -1110,7 +1147,7 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ } } })(); - }, [focusedTableId, tables, draftNodes, activeModel, config, conceptShelfItems, dispatch, t]); + }, [focusedTableId, tables, draftNodes, activeModel, config, conceptShelfItems, dispatch, t, attachedImages, attachedFiles]); // ── Report generation via report agent ────────────────────────── @@ -1473,7 +1510,7 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ {/* @-mention table chips and image attachments. Skip the table-chip row entirely when there's only one root table — there's nothing else the user could @-mention, so the chip is noise. */} - {((primaryTableIds.length > 0 && rootTables.length > 1) || attachedImages.length > 0) && !isChatFormulating && ( + {((primaryTableIds.length > 0 && rootTables.length > 1) || attachedImages.length > 0 || attachedFiles.length > 0) && !isChatFormulating && ( {rootTables.length > 1 && primaryTableIds.map(id => { const tbl = tables.find(t => t.id === id); @@ -1517,6 +1554,27 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ }} /> ))} + {attachedFiles.map((file, idx) => ( + } + label={file.name} + onDelete={() => setAttachedFiles(prev => prev.filter((_, i) => i !== idx))} + sx={{ + height: 20, + fontSize: 10, + maxWidth: 160, + color: theme.palette.text.secondary, + backgroundColor: 'rgba(0,0,0,0.04)', + border: 'none', + borderRadius: '4px', + '& .MuiChip-label': { px: '4px', overflow: 'hidden', textOverflow: 'ellipsis' }, + '& .MuiChip-icon': { ml: '4px', mr: '-2px' }, + '& .MuiChip-deleteIcon': { fontSize: 12, color: theme.palette.text.disabled, mr: '2px' }, + }} + /> + ))} )} {/* @-mention dropdown */} @@ -1645,10 +1703,17 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ {/* Action buttons */} - + { handleAttachFiles(e.target.files); if (e.target) e.target.value = ''; }} + /> + { e.stopPropagation(); setUploadDialogOpen(true); }} + onClick={(e) => { e.stopPropagation(); fileInputRef.current?.click(); }} sx={{ p: 0.5, color: theme.palette.text.secondary, @@ -1762,11 +1827,6 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ {/* The input box */} {inputBox} - setUploadDialogOpen(false)} - initialTab="menu" - /> ); }; diff --git a/src/views/threadLayout.ts b/src/views/threadLayout.ts new file mode 100644 index 00000000..fa793f2c --- /dev/null +++ b/src/views/threadLayout.ts @@ -0,0 +1,39 @@ +// Single source of truth for DataThread column geometry. +// +// Both the DataThread panel (which renders the thread columns) and +// DataFormulator (which snaps the resizable Allotment pane to whole-column +// widths) must agree on these values, otherwise the pane snap points won't +// line up with the actual rendered columns. Keep all width/padding tuning +// here. + +/** Visual width of a single thread card / column (px). */ +export const CARD_WIDTH = 248; + +/** Horizontal gap between adjacent columns (px). */ +export const CARD_GAP = 8; + +/** Total horizontal padding inside the thread panel (left + right, px). */ +export const PANEL_PADDING = 32; + +/** Max number of columns the thread panel will ever lay out. */ +export const MAX_THREAD_COLUMNS = 3; + +/** + * Pixel width required to display exactly `n` columns: + * n cards + (n-1) gaps + panel padding. + */ +export const threadPaneWidth = (n: number): number => + n * CARD_WIDTH + Math.max(0, n - 1) * CARD_GAP + PANEL_PADDING; + +/** + * How many whole columns fit within `containerWidth`, clamped to + * [1, MAX_THREAD_COLUMNS]. Inverse of `threadPaneWidth`. + */ +export const fittableThreadColumns = (containerWidth: number): number => + Math.max( + 1, + Math.min( + MAX_THREAD_COLUMNS, + Math.floor((containerWidth - PANEL_PADDING + CARD_GAP) / (CARD_WIDTH + CARD_GAP)), + ), + ); From 3a23dc3039f6299845f444f42b34180ecd2cf525 Mon Sep 17 00:00:00 2001 From: Chenglong Wang Date: Fri, 29 May 2026 16:28:43 -0700 Subject: [PATCH 06/40] workflow design --- py-src/data_formulator/agent_config.py | 2 +- .../agents/agent_chart_insight.py | 4 +- .../agents/agent_data_loading_chat.py | 8 +- .../agents/agent_interactive_explore.py | 4 +- ...e_distill.py => agent_workflow_distill.py} | 208 ++++++++++----- py-src/data_formulator/agents/data_agent.py | 33 ++- py-src/data_formulator/app.py | 2 +- py-src/data_formulator/knowledge/store.py | 136 ++++++---- py-src/data_formulator/routes/knowledge.py | 121 +++++---- src/api/knowledgeApi.ts | 28 +- src/app/useKnowledgeStore.ts | 14 +- src/i18n/locales/en/common.json | 46 ++-- src/i18n/locales/zh/common.json | 48 ++-- src/views/DataFrameTable.tsx | 27 +- src/views/DataSourceSidebar.tsx | 2 +- src/views/DataThread.tsx | 11 - src/views/InteractionEntryCard.tsx | 5 + src/views/KnowledgePanel.tsx | 244 +++++++----------- src/views/SessionDistill.tsx | 56 ++-- src/views/SimpleChartRecBox.tsx | 46 +++- ...xperienceContext.ts => workflowContext.ts} | 4 +- .../test_agent_knowledge_integration.py | 8 +- ...ce_distill.py => test_workflow_distill.py} | 162 +++++++----- .../backend/knowledge/test_knowledge_store.py | 69 +++-- tests/backend/routes/test_knowledge_routes.py | 137 +++++----- 25 files changed, 794 insertions(+), 631 deletions(-) rename py-src/data_formulator/agents/{agent_experience_distill.py => agent_workflow_distill.py} (58%) rename src/views/{experienceContext.ts => workflowContext.ts} (98%) rename tests/backend/agents/{test_experience_distill.py => test_workflow_distill.py} (74%) diff --git a/py-src/data_formulator/agent_config.py b/py-src/data_formulator/agent_config.py index bec4c670..67dbbe31 100644 --- a/py-src/data_formulator/agent_config.py +++ b/py-src/data_formulator/agent_config.py @@ -56,7 +56,7 @@ # ── Light: single-turn extractors / classifiers / formatters ──────────── "data_load": "minimal", # one-shot type inference "data_clean": "minimal", # extract tables from text - "experience_distill": "minimal", # summarise an analysis context + "workflow_distill": "minimal", # summarise an analysis context "chart_insight": "minimal", # title + 1–3 takeaways from a chart "chart_restyle": "minimal", # apply style edits to a Vega-Lite spec "code_explanation": "minimal", # describe derived fields diff --git a/py-src/data_formulator/agents/agent_chart_insight.py b/py-src/data_formulator/agents/agent_chart_insight.py index a3ae8aba..c280efc2 100644 --- a/py-src/data_formulator/agents/agent_chart_insight.py +++ b/py-src/data_formulator/agents/agent_chart_insight.py @@ -64,7 +64,7 @@ def run(self, chart_image_base64, chart_type, field_names, input_tables=None, n= search_query = " ".join([chart_type] + field_names[:5]).strip() if search_query: relevant = self._knowledge_store.search( - search_query, categories=["experiences"], max_results=3, + search_query, categories=["workflows"], max_results=3, ) if relevant: kb_parts = ["Relevant analysis knowledge:"] @@ -72,7 +72,7 @@ def run(self, chart_image_base64, chart_type, field_names, input_tables=None, n= kb_parts.append(f"- {item['title']}: {item['snippet'][:200]}") context_parts.append("\n".join(kb_parts)) except Exception: - logger.warning("Failed to search knowledge experiences", exc_info=True) + logger.warning("Failed to search knowledge workflows", exc_info=True) context = "\n".join(context_parts) diff --git a/py-src/data_formulator/agents/agent_data_loading_chat.py b/py-src/data_formulator/agents/agent_data_loading_chat.py index 61d3a0e6..55f2640a 100644 --- a/py-src/data_formulator/agents/agent_data_loading_chat.py +++ b/py-src/data_formulator/agents/agent_data_loading_chat.py @@ -1292,7 +1292,7 @@ def _build_system_prompt(self, last_user_text: str = ""): """Build the system prompt with current workspace context. *last_user_text* is used to search the knowledge store for - experiences relevant to the user's current request. Falls back + workflows relevant to the user's current request. Falls back to a generic query when empty. """ table_names = "none" @@ -1324,7 +1324,7 @@ def _build_system_prompt(self, last_user_text: str = ""): if self._knowledge_store: prompt += self._knowledge_store.format_rules_block() - # Inject relevant experiences from knowledge store + # Inject relevant workflows from knowledge store if self._knowledge_store: try: search_query = ( @@ -1334,7 +1334,7 @@ def _build_system_prompt(self, last_user_text: str = ""): ) relevant = self._knowledge_store.search( search_query, - categories=["experiences"], + categories=["workflows"], max_results=3, ) if relevant: @@ -1343,7 +1343,7 @@ def _build_system_prompt(self, last_user_text: str = ""): knowledge_block += f"\n### {item['title']}\n{item['snippet']}\n" prompt += "\n\n" + knowledge_block except Exception: - logger.warning("Failed to search knowledge experiences", exc_info=True) + logger.warning("Failed to search knowledge workflows", exc_info=True) if self.language_instruction: prompt += "\n\n" + self.language_instruction diff --git a/py-src/data_formulator/agents/agent_interactive_explore.py b/py-src/data_formulator/agents/agent_interactive_explore.py index 67847ec2..0f5f90fb 100644 --- a/py-src/data_formulator/agents/agent_interactive_explore.py +++ b/py-src/data_formulator/agents/agent_interactive_explore.py @@ -162,7 +162,7 @@ def run(self, input_tables, start_question=None, if start_question: context += f"\n\n[START QUESTION]\n\n{start_question}" - # ── Inject relevant experiences from knowledge store ────────── + # ── Inject relevant workflows from knowledge store ────────── if self._knowledge_store: try: query = start_question or "" @@ -170,7 +170,7 @@ def run(self, input_tables, start_question=None, search_query = " ".join([query] + table_names[:5]).strip() if search_query: relevant = self._knowledge_store.search( - search_query, categories=["experiences"], max_results=3, + search_query, categories=["workflows"], max_results=3, ) if relevant: knowledge_block = "[RELEVANT KNOWLEDGE]\n" diff --git a/py-src/data_formulator/agents/agent_experience_distill.py b/py-src/data_formulator/agents/agent_workflow_distill.py similarity index 58% rename from py-src/data_formulator/agents/agent_experience_distill.py rename to py-src/data_formulator/agents/agent_workflow_distill.py index cc738495..3f3d9c6d 100644 --- a/py-src/data_formulator/agents/agent_experience_distill.py +++ b/py-src/data_formulator/agents/agent_workflow_distill.py @@ -1,17 +1,17 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -"""Experience distillation agent — extracts reusable knowledge from analysis context. +"""Workflow distillation agent — extracts a replayable workflow from analysis context. Given a user-visible analysis context (timeline of events) plus an optional user instruction, this agent calls an LLM to produce a structured Markdown -experience document with YAML front matter suitable for storage in the +workflow document with YAML front matter suitable for storage in the knowledge base. Usage:: - agent = ExperienceDistillAgent(client) - md_content = agent.run(experience_context, user_instruction="...") + agent = WorkflowDistillAgent(client) + md_content = agent.run(workflow_context, user_instruction="...") """ from __future__ import annotations @@ -25,18 +25,19 @@ logger = logging.getLogger(__name__) -_AGENT_ID = "experience_distill" +_AGENT_ID = "workflow_distill" SYSTEM_PROMPT = """\ -You are a knowledge distiller. Given the chronological events of a data -analysis session plus an optional user instruction, write a short reusable -Markdown note that will help with similar future tasks. +You are a workflow distiller. Given the chronological events of a data +analysis session plus an optional user instruction, extract a short, +**replayable workflow** that captures *what the user wanted and got* — so +the same analysis can be reproduced later on a similarly-shaped dataset. The session contains one or more threads (separate analysis branches in the same session) each rendered under a `### Thread N` header. When -multiple threads are provided, synthesise lessons that hold across them -— do NOT enumerate per-thread. +multiple threads are provided, merge them into one coherent ordered +workflow — do NOT enumerate per-thread. The events use three types: - `message` — directed speech, formatted as `[→/] `. @@ -46,56 +47,136 @@ (followed by columns, row count, sample, and code). - `create_chart` — a chart emitted on a table (mark + encoding summary). -If a user instruction is provided, focus the note on that instruction. -Otherwise, distill the most transferable methodology from the events. +Your job is to recover the **ordered list of requests** the user actually +wanted, and the outputs (tables/charts) they ended up keeping. Beyond the +concrete steps, also distill the analysis at TWO levels of abstraction so +it can be reused later: +- **Adapting to similar data** (concrete) — how to rerun essentially the + same analysis on a near-identical dataset, e.g. the business report for + a different month, region, or product line. Same shape and intent, only + the specific inputs/filters change. +- **Generalizing to other data** (abstract, dataset-agnostic) — the + underlying analytical pattern, independent of this domain: the kinds of + questions, computations, and charts involved, phrased so they transfer + to a different domain or a differently-shaped dataset. + +CRITICAL extraction rules — keep only what the user wanted and got: +- Each step = one user request, written in plain language. Say BOTH the + question being explored AND what was produced to answer it — including + the chart that was created and the key fields it uses (e.g. "Ask how + sales trend over time, and plot monthly total sales as a line chart"; + "Compare regions by breaking revenue down per region as a sorted bar + chart"). Order them as the analysis progressed. +- DROP corrective back-and-forth. If the user changed their mind + ("no, it should be…", "actually use median instead"), keep ONLY the + final resolved intent — not the wrong first attempt or the correction. +- DROP abandoned work. If a chart or table was created and then deleted + or never kept, leave it out entirely. +- DROP mechanics. Do NOT include error-repair loops, dtype fixes, tool + call noise, or low-level code. Describe intent, not implementation. +- Do NOT lean on code or exact column names unless a name is essential to + the request's meaning. Keep steps dataset-agnostic where possible so + they replay on a new slice of similar data. +- Capture genuine gotchas separately as short notes (advisory warnings to + carry forward), NOT as steps to re-perform. + +If a user instruction is provided, let it steer what to keep or emphasise. Output format (Markdown with YAML front matter, nothing else): ``` --- -subtitle: -tags: [] +subtitle: +filename: created: updated: source: distill source_context: --- -## When to Use - - -## Method - - -## Pitfalls & Tips - +## Goal + + +## Steps +1. +2. +3. <…> + +## Adapting to similar data + + +## Generalizing to other data + + +## Notes + ``` Rules: -- Subtitle must be a short, scannable noun phrase (3-8 words) that captures - the technique or pattern. The hosting application prefixes it with the - session name to form the full title (e.g. "Experience from : "), - so do NOT include the session name in the subtitle. Do NOT pack scenario, - takeaway, and steps into the subtitle — leave details for `## When to Use` - and `## Method`. - Good: "Year-over-year volatility comparison". "Repairing pandas dtype mismatches". - Bad: "Time series analysis workflow: aggregate, visualize trends, quantify YoY spikes, and compare volatility across periods". -- Focus on *transferable* methods and caveats, not case-specific details. -- Keep the body under 500 words. -- No raw data, PII, secrets, or specific values unless they show a universal pattern. -- Write the subtitle, headings, body, and tags in {output_language}. +- Subtitle must DESCRIBE what the workflow is about in PLAIN LANGUAGE that + a non-expert can fully understand at a glance, so they can decide + whether to replay it on new data. Favor clarity over brevity: it can be + a full sentence (up to ~25 words) if that makes the analysis genuinely + understandable. Write it like you would explain the analysis to a + colleague in one breath, covering the subject and the main thing you do + with it. The hosting application uses this subtitle directly as the + workflow's display title, so make it self-contained and do NOT prefix it + with the session name. + - Start with a concrete action verb (Plot, Compare, Break down, Rank, + Track, Summarize, Find…). + - Name the real-world subject in everyday words (sales, revenue, + customers, events), NOT the internal mechanics or derived-column + names you happened to create. + - AVOID abstract or technical jargon and invented noun-phrases + ("deltas", "composition", "window", "distribution shift"). If a + technique matters, phrase it plainly ("change from one period to the + next" instead of "deltas"). + Good: "Plot monthly sales over time and compare each year against the + previous one to spot volatile periods". + "Break revenue down by region and show how each region + contributes to the total as a stacked area chart". + "Track how many events happen in each time window and what kinds + of events make up each window". + Bad: "Time series analysis". "Data workflow". "Chart exploration". + "Event window deltas with composition". "Distribution shift inspection". +- Filename must be a SHORT (2-5 word) lowercase name for the file — just + the core subject and action, e.g. "monthly sales trend", "region revenue + breakdown". No dates, no file extension, no session name. It is only + used to name the file on disk; the descriptive subtitle is what users see. +- Steps must be ordered and reproducible. Each step should make clear the + question being explored and the chart/output produced to answer it. +- "Adapting to similar data" stays close to this analysis (same domain, + same shape) — only the concrete inputs change. "Generalizing to other + data" must be domain-neutral: strip out this dataset's subject matter and + describe only the transferable analytical pattern (question types, + computations, chart kinds). Do NOT just repeat the steps in either + section; add genuine reuse guidance. Keep each section brief. +- Be as long as the analysis needs — do not omit meaningful steps, + questions, or charts just to stay short. Stay focused, but completeness + matters more than brevity. +- No raw data, PII, secrets, or specific values unless essential to a request. +- Write the subtitle, headings, and body in {output_language}. YAML front-matter keys stay in English. {language_instruction} """ -class ExperienceDistillAgent: - """Distills analysis context into a reusable experience document.""" - # Language display names for experience-specific prompts +class WorkflowDistillAgent: + """Distills analysis context into a reusable workflow document.""" + + # Language display names for workflow-specific prompts _LANG_NAMES: dict[str, str] = { "zh": "Simplified Chinese (简体中文)", "ja": "Japanese (日本語)", @@ -121,7 +202,7 @@ def __init__( self.timeout_seconds = int(timeout_seconds) if timeout_seconds else self.DEFAULT_TIMEOUT def run(self, context: dict[str, Any], user_instruction: str = "") -> str: - """Distill an experience document from user-visible session context.""" + """Distill a workflow document from user-visible session context.""" summary = self._extract_context_summary(context) today = datetime.now(timezone.utc).strftime("%Y-%m-%d") context_id = str(context.get("context_id", "") or "") @@ -130,7 +211,7 @@ def run(self, context: dict[str, Any], user_instruction: str = "") -> str: instruction_block = ( f"\n[USER INSTRUCTION]\n{user_instruction.strip()}\n" - f"Focus the distilled experience on the above instruction.\n" + f"Focus the distilled workflow on the above instruction.\n" ) if user_instruction and user_instruction.strip() else "" workspace_block = ( @@ -158,9 +239,11 @@ def run(self, context: dict[str, Any], user_instruction: str = "") -> str: {"role": "user", "content": user_msg}, ] - from data_formulator.knowledge.store import KNOWLEDGE_LIMITS + from data_formulator.knowledge.store import KNOWLEDGE_LIMITS, WORKFLOW_HARD_MAX content = self._call_with_length_retry( - messages, KNOWLEDGE_LIMITS.get("experiences", 2000), + messages, + KNOWLEDGE_LIMITS.get("workflows", 6000), + WORKFLOW_HARD_MAX, ) if not content.strip().startswith("---"): @@ -182,7 +265,7 @@ def _prompt_format_kwargs(self) -> dict[str, str]: lang_block = ( f"[LANGUAGE INSTRUCTION]\n" f"The user's language is **{display_name}**.\n" - f"Write the title, all section headings, all body text, and tags " + f"Write the title, all section headings, and all body text " f"in {display_name}. YAML front-matter keys stay in English." ) return { @@ -199,39 +282,43 @@ def _prompt_format_kwargs(self) -> dict[str, str]: def _call_with_length_retry( self, messages: list[dict], - body_limit: int, + soft_limit: int, + hard_limit: int, ) -> str: - """Call LLM and retry once if the body exceeds *body_limit* characters. + """Call the LLM, nudging it to stay near *soft_limit* characters. - If the retry *still* overshoots, hard-truncate the body so the - document is saved instead of the entire distillation being lost. + ``soft_limit`` is advisory guidance: if the first response overshoots + it we retry once asking the model to condense. We only ever + hard-truncate at ``hard_limit`` — a much larger safety ceiling — so + rich, multi-section workflows are kept intact while runaway output + is still bounded. """ from data_formulator.knowledge.store import parse_front_matter content = self._call_llm(messages) _, body = parse_front_matter(content) - if len(body.strip()) <= body_limit: + if len(body.strip()) <= soft_limit: return content - retry_target = max(body_limit - self.RETRY_MARGIN, 1) + retry_target = max(soft_limit - self.RETRY_MARGIN, 1) logger.info( - "Distilled content too long (%d > %d), retrying with condensation prompt (target ≤ %d)", - len(body.strip()), body_limit, retry_target, + "Distilled content over soft target (%d > %d), retrying with condensation prompt (target ≤ %d)", + len(body.strip()), soft_limit, retry_target, ) messages = messages + [ {"role": "assistant", "content": content}, {"role": "user", "content": ( - f"Your output body is {len(body.strip())} characters, which exceeds " - f"the limit of {body_limit}. Please condense the document to fit " - f"within {retry_target} characters while keeping the most important " - f"insights. Output ONLY the revised Markdown document." + f"Your output body is {len(body.strip())} characters, which is " + f"longer than ideal. Please tighten the document to around " + f"{retry_target} characters while keeping the most important " + f"insights and all sections. Output ONLY the revised Markdown document." )}, ] retried = self._call_llm(messages) - # Hard-trim if the retry still overshoots — better a slightly - # truncated experience than a save failure. - return self._truncate_body_to_limit(retried, body_limit) + # Hard-trim only if the retry blows past the absolute ceiling — + # better a slightly truncated workflow than a save failure. + return self._truncate_body_to_limit(retried, hard_limit) @classmethod def _truncate_body_to_limit(cls, content: str, body_limit: int) -> str: @@ -385,7 +472,7 @@ def _render_events(cls, events: list[Any]) -> str: return "\n".join(parts) if parts else "(empty context)" def _call_llm(self, messages: list[dict]) -> str: - """Single LLM call to generate the experience document.""" + """Single LLM call to generate the workflow document.""" resp = self.client.get_completion( messages, reasoning_effort=reasoning_effort_for(_AGENT_ID, self.client.model), timeout=self.timeout_seconds, ) @@ -401,7 +488,6 @@ def _add_fallback_front_matter( header = ( f"---\ntitle: {title}\n" - f"tags: []\n" f"created: {today}\n" f"updated: {today}\n" f"source: distill\n" diff --git a/py-src/data_formulator/agents/data_agent.py b/py-src/data_formulator/agents/data_agent.py index 8e9cd39a..9a9d10b2 100644 --- a/py-src/data_formulator/agents/data_agent.py +++ b/py-src/data_formulator/agents/data_agent.py @@ -153,7 +153,7 @@ def _rescue_validate_action(data: dict) -> list[str]: "function": { "name": "search_knowledge", "description": ( - "Search the user's knowledge base (rules, experiences) " + "Search the user's knowledge base (rules, workflows) " "for relevant entries. Returns title, category, snippet, and " "path for each match. Use read_knowledge to get full content." ), @@ -168,7 +168,7 @@ def _rescue_validate_action(data: dict) -> list[str]: "type": "array", "items": { "type": "string", - "enum": ["rules", "experiences"], + "enum": ["rules", "workflows"], }, "description": "Optional: limit search to specific categories.", }, @@ -190,7 +190,7 @@ def _rescue_validate_action(data: dict) -> list[str]: "properties": { "category": { "type": "string", - "enum": ["rules", "experiences"], + "enum": ["rules", "workflows"], "description": "Knowledge category.", }, "path": { @@ -224,7 +224,7 @@ def _rescue_validate_action(data: dict) -> list[str]: - **inspect_source_data(table_names)** — get schema, stats, and sample rows for source tables (cheaper than explore for basic inspection). - **search_knowledge(query, categories?)** — search the user's knowledge base - (rules, experiences) for relevant entries. + (rules, workflows) for relevant entries. - **read_knowledge(category, path)** — read the full content of a knowledge entry. You analyse data that is **already in the workspace**. If the user's @@ -1379,14 +1379,14 @@ def _build_initial_messages( if peripheral_block: user_content += f"{peripheral_block}\n\n" - # Search and inject relevant knowledge (experiences + non-alwaysApply rules) + # Search and inject relevant knowledge (workflows + non-alwaysApply rules) table_names = [t.get("name", "") for t in input_tables if t.get("name")] relevant_knowledge = self._search_relevant_knowledge(user_question, table_names) - # Always include the experience distilled from the active workspace + # Always include the workflow distilled from the active workspace # (design-docs/24 §3.6) so the session has stable working memory # across turns regardless of search relevance. - session_exp = self._load_active_session_experience() + session_exp = self._load_active_session_workflow() if session_exp: existing_paths = { (item["category"], item["path"]) for item in relevant_knowledge @@ -1891,7 +1891,7 @@ def _search_relevant_knowledge( table_names: list[str], max_items: int = 5, ) -> list[dict[str, Any]]: - """Search experiences and non-alwaysApply rules relevant to the current session. + """Search workflows and non-alwaysApply rules relevant to the current session. Uses the user question as the search query and passes table names separately for tag-overlap boosting. alwaysApply rules are @@ -1904,7 +1904,7 @@ def _search_relevant_knowledge( try: results = self._knowledge_store.search( user_question, - categories=["rules", "experiences"], + categories=["rules", "workflows"], max_results=max_items, table_names=table_names[:5], ) @@ -1913,11 +1913,11 @@ def _search_relevant_knowledge( logger.warning("Failed to search knowledge", exc_info=True) return [] - def _load_active_session_experience(self) -> dict[str, Any] | None: - """Return the experience distilled from the active workspace, if any. + def _load_active_session_workflow(self) -> dict[str, Any] | None: + """Return the workflow distilled from the active workspace, if any. The session-scoped distillation flow (design-docs/24) writes one - experience per workspace, stamped with ``source_workspace_id``. + workflow per workspace, stamped with ``source_workspace_id``. We always inject that file into the agent's context so the agent has stable working memory for the active session in addition to whatever the relevance search picked. @@ -1932,14 +1932,14 @@ def _load_active_session_experience(self) -> dict[str, Any] | None: if not ws_id: return None try: - entry = self._knowledge_store.find_experience_by_workspace_id(ws_id) + entry = self._knowledge_store.find_workflow_by_workspace_id(ws_id) except Exception: - logger.warning("find_experience_by_workspace_id failed", exc_info=True) + logger.warning("find_workflow_by_workspace_id failed", exc_info=True) return None if not entry: return None try: - content = self._knowledge_store.read("experiences", entry["path"]) + content = self._knowledge_store.read("workflows", entry["path"]) except Exception: return None from data_formulator.knowledge.store import parse_front_matter @@ -1948,9 +1948,8 @@ def _load_active_session_experience(self) -> dict[str, Any] | None: if not snippet: return None return { - "category": "experiences", + "category": "workflows", "title": entry.get("title", entry.get("path", "")), - "tags": entry.get("tags", []), "path": entry["path"], "snippet": snippet, "source": entry.get("source", "distill"), diff --git a/py-src/data_formulator/app.py b/py-src/data_formulator/app.py index 47d2bda8..ef9dd4cb 100644 --- a/py-src/data_formulator/app.py +++ b/py-src/data_formulator/app.py @@ -219,7 +219,7 @@ def _register_blueprints(): from data_formulator.routes.credentials import credential_bp app.register_blueprint(credential_bp) - # Register knowledge management API (rules, skills, experiences) + # Register knowledge management API (rules, skills, workflows) from data_formulator.routes.knowledge import knowledge_bp app.register_blueprint(knowledge_bp) diff --git a/py-src/data_formulator/knowledge/store.py b/py-src/data_formulator/knowledge/store.py index 0b290093..08463437 100644 --- a/py-src/data_formulator/knowledge/store.py +++ b/py-src/data_formulator/knowledge/store.py @@ -1,10 +1,10 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -"""Knowledge store — manages user knowledge files (rules, experiences). +"""Knowledge store — manages user knowledge files (rules, workflows). Each user has a ``knowledge/`` directory under their home with two -sub-directories: ``rules`` and ``experiences``. Every knowledge entry is a +sub-directories: ``rules`` and ``workflows``. Every knowledge entry is a Markdown file with YAML front matter. All file I/O is routed through :class:`ConfinedDir` for path safety. @@ -12,7 +12,7 @@ Directory depth constraints: - ``rules``: flat — only files directly under ``rules/`` (1 path part) -- ``experiences``: one level of sub-directories (up to 2 path parts) +- ``workflows``: one level of sub-directories (up to 2 path parts) """ from __future__ import annotations @@ -27,19 +27,27 @@ logger = logging.getLogger(__name__) -VALID_CATEGORIES = frozenset({"rules", "experiences"}) +VALID_CATEGORIES = frozenset({"rules", "workflows"}) _MAX_DEPTH = { "rules": 1, - "experiences": 2, # one sub-dir: "category/file.md" + "workflows": 2, # one sub-dir: "category/file.md" } KNOWLEDGE_LIMITS: dict[str, int] = { "rule_description_max": 100, "rules": 350, - "experiences": 2000, + # Soft length guidance for distilled workflows: the target the distill + # agent aims for, NOT a hard cap. Workflows may exceed it when an + # analysis genuinely needs the room (e.g. multiple abstraction levels). + # Writes are only rejected past WORKFLOW_HARD_MAX below. + "workflows": 6000, } +# Absolute safety ceiling for a workflow body. Guards against runaway LLM +# output while still letting rich, multi-section workflows through. +WORKFLOW_HARD_MAX: int = 24000 + # --------------------------------------------------------------------------- # Tokenization helpers for improved search scoring # --------------------------------------------------------------------------- @@ -151,14 +159,13 @@ class KnowledgeItemMeta: """ __slots__ = ( - "title", "tags", "source", "created", "description", "always_apply", + "title", "source", "created", "description", "always_apply", "source_workspace_id", "source_workspace_name", ) def __init__( self, title: str, - tags: list[str], source: str, created: str, description: str, @@ -167,7 +174,6 @@ def __init__( source_workspace_name: str = "", ): self.title = title - self.tags = tags self.source = source self.created = created self.description = description @@ -181,14 +187,6 @@ def from_raw(cls, meta: dict[str, Any], fallback_stem: str = "") -> "KnowledgeIt title = meta.get("title", fallback_stem) title = str(title) if title is not None else fallback_stem - raw_tags = meta.get("tags", []) - if isinstance(raw_tags, list): - tags = [str(t) for t in raw_tags] - elif raw_tags is None: - tags = [] - else: - tags = [str(raw_tags)] - source = str(meta.get("source", "manual") or "manual") created = str(meta.get("created", "") or "") description = str(meta.get("description", "") or "") @@ -198,7 +196,6 @@ def from_raw(cls, meta: dict[str, Any], fallback_stem: str = "") -> "KnowledgeIt return cls( title=title, - tags=tags, source=source, created=created, description=description, @@ -246,26 +243,64 @@ class KnowledgeStore: store = KnowledgeStore(user_home) items = store.list_all("rules") - content = store.read("experiences", "data-cleaning/handle-missing.md") + content = store.read("workflows", "data-cleaning/handle-missing.md") store.write("rules", "date-format.md", md_content) store.delete("rules", "date-format.md") - results = store.search("ROI", categories=["rules", "experiences"]) + results = store.search("ROI", categories=["rules", "workflows"]) """ def __init__(self, user_home: Path | str) -> None: user_home = Path(user_home) self._root = ConfinedDir(user_home / "knowledge", mkdir=True) + self._migrate_experiences_to_workflows() self._jails: dict[str, ConfinedDir] = { "rules": ConfinedDir(self._root.root / "rules", mkdir=True), - "experiences": ConfinedDir(self._root.root / "experiences", mkdir=True), + "workflows": ConfinedDir(self._root.root / "workflows", mkdir=True), } self._migrate_flat() # -- migration --------------------------------------------------------- + def _migrate_experiences_to_workflows(self) -> None: + """Move legacy ``experiences/`` files into ``workflows/`` (one-time). + + The feature was renamed from "experiences" to "workflows"; existing + users have files under ``knowledge/experiences/``. Move them so the + rename is transparent. + """ + old_root = self._root.root / "experiences" + if not old_root.is_dir(): + return + new_root = self._root.root / "workflows" + new_root.mkdir(parents=True, exist_ok=True) + for md_file in list(old_root.rglob("*.md")): + rel = md_file.relative_to(old_root) + dest = new_root / rel + dest.parent.mkdir(parents=True, exist_ok=True) + if dest.exists(): + stem = rel.stem + suffix_n = 1 + while dest.exists(): + dest = dest.parent / f"{stem}-{suffix_n}.md" + suffix_n += 1 + try: + md_file.rename(dest) + logger.info("Migrated experiences/%s → workflows/%s", rel, dest.name) + except Exception: + logger.warning("Failed to migrate experience file %s", md_file, exc_info=True) + # Remove the now-empty legacy tree (best effort) + try: + for sub in sorted(old_root.rglob("*"), reverse=True): + if sub.is_dir() and not any(sub.iterdir()): + sub.rmdir() + if not any(old_root.iterdir()): + old_root.rmdir() + except Exception: + logger.warning("Failed to clean up legacy experiences dir", exc_info=True) + def _migrate_flat(self) -> None: - """Move any experiences/subdir/file.md → experiences/file.md (one-time migration).""" - exp_root = self._jails["experiences"].root + """Move any workflows/subdir/file.md → workflows/file.md (one-time migration).""" + exp_root = self._jails["workflows"].root for md_file in list(exp_root.rglob("*.md")): rel = md_file.relative_to(exp_root) if len(rel.parts) <= 1: @@ -285,9 +320,9 @@ def _migrate_flat(self) -> None: parent = md_file.parent if parent != exp_root and not any(parent.iterdir()): parent.rmdir() - logger.info("Migrated knowledge experience %s → %s", rel, dest.name) + logger.info("Migrated knowledge workflow %s → %s", rel, dest.name) except Exception: - logger.warning("Failed to migrate experience file %s", md_file, exc_info=True) + logger.warning("Failed to migrate workflow file %s", md_file, exc_info=True) # -- path validation --------------------------------------------------- @@ -326,7 +361,7 @@ def _jail(self, category: str) -> ConfinedDir: def list_all(self, category: str) -> list[dict[str, Any]]: """List all knowledge entries in *category*. - Returns a list of dicts with ``title``, ``tags``, ``path``, + Returns a list of dicts with ``title``, ``path``, ``source``, and ``created`` parsed from front matter. For rules, also includes ``description`` and ``alwaysApply``. """ @@ -345,7 +380,6 @@ def list_all(self, category: str) -> list[dict[str, Any]]: rel = str(md_file.relative_to(jail.root)).replace("\\", "/") item: dict[str, Any] = { "title": km.title, - "tags": km.tags, "path": rel, "source": km.source, "created": km.created, @@ -353,9 +387,9 @@ def list_all(self, category: str) -> list[dict[str, Any]]: if category == "rules": item["description"] = km.description item["alwaysApply"] = km.always_apply - if category == "experiences": + if category == "workflows": # Surface session-distillation provenance so the frontend can - # find an existing session experience by workspace id + # find an existing session workflow by workspace id # without re-reading every file. See design-docs/24. if km.source_workspace_id: item["sourceWorkspaceId"] = km.source_workspace_id @@ -394,7 +428,15 @@ def write(self, category: str, path: str, content: str) -> Path: body_limit = KNOWLEDGE_LIMITS.get(category) if body_limit is not None: body_len = len(body.strip()) - if body_len > body_limit: + if category == "workflows": + # Soft guidance: the body_limit is a target the distill agent + # aims for, not a hard cap. Only reject far past the ceiling. + if body_len > WORKFLOW_HARD_MAX: + raise ValueError( + f"workflows body exceeds {WORKFLOW_HARD_MAX} characters " + f"(got {body_len})" + ) + elif body_len > body_limit: raise ValueError( f"{category} body exceeds {body_limit} characters " f"(got {body_len})" @@ -407,12 +449,12 @@ def delete(self, category: str, path: str) -> None: self.validate_path(category, path) self._jail(category).unlink(path) - # -- session experience helpers ---------------------------------------- + # -- session workflow helpers ---------------------------------------- - def find_experience_by_workspace_id( + def find_workflow_by_workspace_id( self, workspace_id: str, ) -> dict[str, Any] | None: - """Return the experience entry whose front matter records this workspace id. + """Return the workflow entry whose front matter records this workspace id. Used by the session-scoped distillation flow (design-docs/24) to upsert: when re-distilling the same session, overwrite the same @@ -421,11 +463,11 @@ def find_experience_by_workspace_id( if not workspace_id or not workspace_id.strip(): return None try: - for item in self.list_all("experiences"): + for item in self.list_all("workflows"): if item.get("sourceWorkspaceId") == workspace_id: return item except Exception: - logger.warning("find_experience_by_workspace_id failed", exc_info=True) + logger.warning("find_workflow_by_workspace_id failed", exc_info=True) return None # -- alwaysApply rules helper ------------------------------------------ @@ -511,12 +553,13 @@ def search( """Search across knowledge categories. Tokenizes *query* into keywords and scores each entry using - multi-field weighted matching (title > tags > filename > body). - Whole-string exact matches and table-name / tag overlaps receive + multi-field weighted matching (title > filename > body). + Whole-string exact matches and table-name overlaps receive additional bonuses. Non-manual sources are slightly discounted. *table_names* (optional) are table names from the current session; - when a table name appears in an entry's tags the entry is boosted. + when a table name appears in an entry's title or body the entry is + boosted. """ if not query or not query.strip(): return [] @@ -542,7 +585,7 @@ def search( continue score = self._match_score( - q, km.title, km.tags, md_file.stem, body[:200], + q, km.title, md_file.stem, body[:200], source=km.source, table_names=table_names, ) if score <= 0: @@ -552,7 +595,6 @@ def search( scored.append((score, { "category": cat, "title": km.title, - "tags": km.tags, "path": rel, "snippet": body[:500].strip(), "source": km.source, @@ -565,7 +607,6 @@ def search( def _match_score( query: str, title: str, - tags: list[str], stem: str, body_prefix: str, *, @@ -589,13 +630,10 @@ def _match_score( title_l = title.lower() stem_l = stem.lower() body_l = body_prefix.lower() - tags_l = [t.lower() for t in tags] for token in tokens: if token in title_l: score += 100 / n - if any(token in tl for tl in tags_l): - score += 50 / n if token in stem_l: score += 30 / n if token in body_l: @@ -604,14 +642,14 @@ def _match_score( # Whole-string bonus (handles short queries like "ROI") if q and q in title.lower(): score += 50 - if q and any(q in t.lower() for t in tags): - score += 50 - # Table-name → tag overlap bonus + # Table-name overlap bonus (title / body) if table_names: - tags_l_set = {t.lower() for t in tags} + title_l = title.lower() + body_l = body_prefix.lower() for tn in table_names: - if any(tn.lower() in tl for tl in tags_l_set): + tnl = tn.lower() + if tnl in title_l or tnl in body_l: score += 30 # Non-manual source slight discount diff --git a/py-src/data_formulator/routes/knowledge.py b/py-src/data_formulator/routes/knowledge.py index 901024c1..1a458ba9 100644 --- a/py-src/data_formulator/routes/knowledge.py +++ b/py-src/data_formulator/routes/knowledge.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -"""Knowledge management API — CRUD + search + experience distillation. +"""Knowledge management API — CRUD + search + workflow distillation. All endpoints use ``POST`` with JSON body. Access is scoped to the current user via ``get_identity_id()`` and confined via ``ConfinedDir``. @@ -155,44 +155,44 @@ def knowledge_search(): return json_ok({"results": results}) -# ── distill experience ──────────────────────────────────────────────────── +# ── distill workflow ──────────────────────────────────────────────────── -@knowledge_bp.route("/distill-experience", methods=["POST"]) -def distill_experience(): - """Distill user-visible analysis context into a reusable experience. +@knowledge_bp.route("/distill-workflow", methods=["POST"]) +def distill_workflow(): + """Distill user-visible analysis context into a reusable workflow. Session-scoped payload (design-docs/24): - ``experience_context`` carries a list of ``threads`` (one per leaf + ``workflow_context`` carries a list of ``threads`` (one per leaf derived table the user has on screen), each with its own chronological ``events`` array. ``workspace_id`` + ``workspace_name`` bind the resulting file to the active session so re-distilling upserts the same file. - Required body fields: ``experience_context`` and ``model``. + Required body fields: ``workflow_context`` and ``model``. Optional: ``user_instruction`` (natural-language focus hint for the LLM), - ``category_hint`` (sub-directory under experiences/). + ``category_hint`` (sub-directory under workflows/). """ data = request.get_json(silent=True) or {} - experience_context = data.get("experience_context") - if not isinstance(experience_context, dict): - raise AppError(ErrorCode.INVALID_REQUEST, "'experience_context' is required") + workflow_context = data.get("workflow_context") + if not isinstance(workflow_context, dict): + raise AppError(ErrorCode.INVALID_REQUEST, "'workflow_context' is required") - threads = experience_context.get("threads") + threads = workflow_context.get("threads") if not isinstance(threads, list) or not threads: raise AppError( ErrorCode.INVALID_REQUEST, - "'experience_context.threads' is required and must be a non-empty list", + "'workflow_context.threads' is required and must be a non-empty list", ) - workspace_id_raw = experience_context.get("workspace_id", "") + workspace_id_raw = workflow_context.get("workspace_id", "") workspace_id = workspace_id_raw.strip() if isinstance(workspace_id_raw, str) else "" - workspace_name_raw = experience_context.get("workspace_name", "") + workspace_name_raw = workflow_context.get("workspace_name", "") workspace_name = workspace_name_raw.strip() if isinstance(workspace_name_raw, str) else "" if not workspace_id or not workspace_name: raise AppError( ErrorCode.INVALID_REQUEST, - "'experience_context.workspace_id' and 'workspace_name' are required", + "'workflow_context.workspace_id' and 'workspace_name' are required", ) model_config = data.get("model") @@ -215,53 +215,55 @@ def distill_experience(): # Build client and run distillation from data_formulator.routes.agents import get_client, _get_ui_lang - from data_formulator.agents.agent_experience_distill import ExperienceDistillAgent + from data_formulator.agents.agent_workflow_distill import WorkflowDistillAgent client = get_client(model_config) - agent = ExperienceDistillAgent( + agent = WorkflowDistillAgent( client=client, language_code=_get_ui_lang(), timeout_seconds=timeout_seconds, ) try: - md_content = agent.run(experience_context, user_instruction=user_instruction) + md_content = agent.run(workflow_context, user_instruction=user_instruction) except Exception as exc: - logger.warning("Experience distillation LLM call failed: %s", type(exc).__name__) + logger.warning("Workflow distillation LLM call failed: %s", type(exc).__name__) from data_formulator.error_handler import classify_and_wrap_llm_error raise classify_and_wrap_llm_error(exc) from exc - # Save to knowledge/experiences/ + # Save to knowledge/workflows/ store = KnowledgeStore(user_home) - # Bind the file to the workspace, override title to - # "Experience from : ", and upsert below. - md_content = _apply_session_front_matter(md_content, workspace_id, workspace_name) + # Bind the file to the workspace, set the title to the agent-generated + # descriptive subtitle, and upsert below. + md_content, title_core, filename_hint = _apply_session_front_matter( + md_content, workspace_id, workspace_name, + ) - filename = _experience_filename(workspace_name) + filename = _workflow_filename(filename_hint or title_core or workspace_name) rel_path = f"{category_hint}/{filename}" if category_hint else filename - # Upsert: if a previous experience exists for this workspace at a + # Upsert: if a previous workflow exists for this workspace at a # different path (e.g. user renamed the workspace), delete it after a # successful write so we keep one file per session. - existing = store.find_experience_by_workspace_id(workspace_id) + existing = store.find_workflow_by_workspace_id(workspace_id) try: - store.write("experiences", rel_path, md_content) + store.write("workflows", rel_path, md_content) except ValueError as exc: raise AppError(ErrorCode.INVALID_REQUEST, str(exc)) from exc if existing and existing.get("path") and existing["path"] != rel_path: try: - store.delete("experiences", existing["path"]) + store.delete("workflows", existing["path"]) except Exception: logger.warning( - "Failed to delete stale session experience at %s", + "Failed to delete stale session workflow at %s", existing.get("path"), exc_info=True, ) - return json_ok({"path": rel_path, "category": "experiences"}) + return json_ok({"path": rel_path, "category": "workflows"}) # ── helpers for session-scoped distillation ─────────────────────────────── @@ -269,16 +271,21 @@ def distill_experience(): def _apply_session_front_matter( content: str, workspace_id: str, workspace_name: str, -) -> str: - """Override / inject session-binding fields in the experience front matter. - - - Composes the visible ``title`` as ``Experience from : `` - using the LLM-emitted ``subtitle`` (preferred) or pre-existing - ``title``. The original ``subtitle`` field is removed from the - front matter once consumed. +) -> tuple[str, str, str]: + """Override / inject session-binding fields in the workflow front matter. + + - Sets the visible ``title`` to the agent-emitted descriptive + ``subtitle`` (preferred) or the pre-existing ``title``, with any + legacy ``Workflow from : `` prefix stripped. The ``subtitle`` + field is removed from the front matter once consumed. + - Consumes the agent-emitted short ``filename`` hint (removed from the + front matter) and returns it so the caller can name the file without + using the long descriptive title. - Stamps ``source_workspace_id`` and ``source_workspace_name`` so the file can be looked up on subsequent distillations. - Forces ``source: distill`` (idempotent if already set). + + Returns ``(content_with_front_matter, title_core, filename_hint)``. """ from data_formulator.knowledge.store import parse_front_matter @@ -287,27 +294,31 @@ def _apply_session_front_matter( meta = {} subtitle = str(meta.pop("subtitle", "") or "").strip() + filename_hint = str(meta.pop("filename", "") or "").strip() existing_title = str(meta.get("title", "") or "").strip() - # Strip any "Experience from : " prefix from a prior pass so - # update-mode runs don't double-prefix when the LLM echoes the title. - title_core = subtitle or _strip_experience_prefix(existing_title) + # Strip any legacy "Workflow from : " (or "Experience from") + # prefix so update-mode runs don't carry it forward. + title_core = subtitle or _strip_workflow_prefix(existing_title) if not title_core: title_core = workspace_name - new_title = f"Experience from {workspace_name}: {title_core}" - meta["title"] = new_title + meta["title"] = title_core meta["source"] = "distill" meta["source_workspace_id"] = workspace_id meta["source_workspace_name"] = workspace_name - return _serialize_front_matter(meta, body) + return _serialize_front_matter(meta, body), title_core, filename_hint + +_EXP_PREFIX_RE = re.compile(r"^\s*(?:Workflow|Experience) from .+?:\s*", re.IGNORECASE) -_EXP_PREFIX_RE = re.compile(r"^\s*Experience from .+?:\s*", re.IGNORECASE) +# Path separators, Windows-reserved chars and control chars that must never +# appear in a filename derived from untrusted LLM output. +_UNSAFE_FILENAME_CHARS = re.compile(r'[\\/:*?"<>|\x00-\x1f]+') -def _strip_experience_prefix(title: str) -> str: +def _strip_workflow_prefix(title: str) -> str: return _EXP_PREFIX_RE.sub("", title).strip() @@ -323,16 +334,22 @@ def _serialize_front_matter(meta: dict, body: str) -> str: return f"---\n{yaml_text}\n---\n\n{body_text}" -def _experience_filename(workspace_name: str) -> str: - """Derive a deterministic filename from the workspace name. +def _workflow_filename(title: str) -> str: + """Slugify an LLM-supplied name into a clean, safe ``.md`` filename. - Re-distilling the same session always lands on the same file. - Falls back to a literal slug when sanitisation rejects the name. + Re-distilling a session upserts by ``source_workspace_id`` (see caller), + so the file is replaced even when the name changes. ``safe_data_filename`` + enforces the security boundary (basename only, no ``.``/``..``); the slug + step just keeps separators and reserved chars out so the name is clean and + portable. Unicode (e.g. CJK) is preserved. """ from data_formulator.datalake.parquet_utils import safe_data_filename - slug = workspace_name.strip().replace(" ", "-").lower()[:80] or "session-experience" + cleaned = _UNSAFE_FILENAME_CHARS.sub("-", title) + cleaned = re.sub(r"\s+", "-", cleaned.strip()) + cleaned = re.sub(r"-{2,}", "-", cleaned) + slug = cleaned.strip(".-").lower()[:80] or "session-workflow" try: return safe_data_filename(f"{slug}.md") except ValueError: - return "session-experience.md" + return "session-workflow.md" diff --git a/src/api/knowledgeApi.ts b/src/api/knowledgeApi.ts index 7c149f3a..a722c00f 100644 --- a/src/api/knowledgeApi.ts +++ b/src/api/knowledgeApi.ts @@ -2,7 +2,7 @@ // Licensed under the MIT License. /** - * Knowledge API client — CRUD, search, and experience distillation. + * Knowledge API client — CRUD, search, and workflow distillation. * * All endpoints use POST with JSON body. Requests go through * {@link fetchWithIdentity} for identity headers and 401 retry. @@ -14,11 +14,10 @@ import { apiRequest } from '../app/apiClient'; // ── Types ──────────────────────────────────────────────────────────────── -export type KnowledgeCategory = 'rules' | 'experiences'; +export type KnowledgeCategory = 'rules' | 'workflows'; export interface KnowledgeItem { title: string; - tags: string[]; path: string; source: string; created: string; @@ -27,25 +26,24 @@ export interface KnowledgeItem { /** Rules only: if true the rule is always injected into the agent prompt. */ alwaysApply?: boolean; /** - * Experiences only: workspace id this experience was distilled from. + * Workflows only: workspace id this workflow was distilled from. * Set by the session-scoped distillation flow (design-docs/24); used - * by the KnowledgePanel to find the existing session experience. + * by the KnowledgePanel to find the existing session workflow. */ sourceWorkspaceId?: string; - /** Experiences only: workspace display name at distillation time. */ + /** Workflows only: workspace display name at distillation time. */ sourceWorkspaceName?: string; } export interface KnowledgeLimits { rule_description_max: number; rules: number; - experiences: number; + workflows: number; } export interface KnowledgeSearchResult { category: KnowledgeCategory; title: string; - tags: string[]; path: string; snippet: string; source: string; @@ -122,7 +120,7 @@ export async function searchKnowledge( return data.results ?? []; } -export interface DistillExperienceResult { +export interface DistillWorkflowResult { path: string; category: string; } @@ -134,7 +132,7 @@ export interface DistillExperienceResult { * a deterministic filename + title. `threads` carries one chronological * `events` list per leaf table on screen. */ -export interface SessionExperienceContext { +export interface SessionWorkflowContext { context_id?: string; workspace_id: string; workspace_name: string; @@ -146,18 +144,18 @@ export interface SessionExperienceContext { payload_notes?: string[]; } -export async function distillSessionExperience( - sessionContext: SessionExperienceContext, +export async function distillSessionWorkflow( + sessionContext: SessionWorkflowContext, model: Record, instruction?: string, timeoutSeconds?: number, signal?: AbortSignal, -): Promise { - const { data } = await apiRequest<{ path: string; category: string }>('/api/knowledge/distill-experience', { +): Promise { + const { data } = await apiRequest<{ path: string; category: string }>('/api/knowledge/distill-workflow', { method: 'POST', headers: JSON_HEADERS, body: JSON.stringify({ - experience_context: sessionContext, + workflow_context: sessionContext, model, user_instruction: instruction, timeout_seconds: timeoutSeconds, diff --git a/src/app/useKnowledgeStore.ts b/src/app/useKnowledgeStore.ts index 0a6ea65d..6adeb60c 100644 --- a/src/app/useKnowledgeStore.ts +++ b/src/app/useKnowledgeStore.ts @@ -5,7 +5,7 @@ * Knowledge state management — React hooks for knowledge CRUD & search. * * Uses plain React state (not Redux) because knowledge data is server-side - * and only needed by the KnowledgePanel and save-as-experience flows. + * and only needed by the KnowledgePanel and save-as-workflow flows. * Errors are dispatched to the global MessageSnackbar via dfActions.addMessages. */ @@ -40,16 +40,16 @@ export function useKnowledgeStore() { const { t } = useTranslation(); const [rules, setRules] = useState({ ...EMPTY_CATEGORY }); - const [experiences, setExperiences] = useState({ ...EMPTY_CATEGORY }); + const [workflows, setWorkflows] = useState({ ...EMPTY_CATEGORY }); const [searchResults, setSearchResults] = useState([]); const [searching, setSearching] = useState(false); - const DEFAULT_LIMITS: KnowledgeLimits = { rule_description_max: 100, rules: 350, experiences: 2000 }; + const DEFAULT_LIMITS: KnowledgeLimits = { rule_description_max: 100, rules: 350, workflows: 2000 }; const [limits, setLimits] = useState(DEFAULT_LIMITS); - const stateMap = { rules, experiences }; - const setterMap = useRef({ rules: setRules, experiences: setExperiences }); + const stateMap = { rules, workflows }; + const setterMap = useRef({ rules: setRules, workflows: setWorkflows }); const fetchList = useCallback(async (category: KnowledgeCategory) => { const setter = setterMap.current[category]; @@ -71,7 +71,7 @@ export function useKnowledgeStore() { const fetchAll = useCallback(async () => { await Promise.all([ fetchList('rules'), - fetchList('experiences'), + fetchList('workflows'), fetchKnowledgeLimits().then(setLimits).catch(() => { /* best-effort */ }), ]); }, [fetchList]); @@ -184,7 +184,7 @@ export function useKnowledgeStore() { return { rules, - experiences, + workflows, stateMap, limits, searchResults, diff --git a/src/i18n/locales/en/common.json b/src/i18n/locales/en/common.json index 7a52125b..8ef952df 100644 --- a/src/i18n/locales/en/common.json +++ b/src/i18n/locales/en/common.json @@ -875,9 +875,9 @@ "knowledge": { "title": "Agent Knowledge", "rules": "Rules", - "experiences": "Experiences", + "workflows": "Workflows", "rulesDescription": "Constraints and standards that agents must follow", - "experiencesDescription": "Reusable methods, tips, and knowledge distilled from analyses", + "workflowsDescription": "Reusable analysis workflows distilled from past sessions that agents can save and replay", "newItem": "New", "search": "Search", "searchPlaceholder": "Search knowledge...", @@ -902,29 +902,29 @@ "failedToSave": "Failed to save knowledge", "failedToDelete": "Failed to delete knowledge", "failedToSearch": "Search failed", - "saveAsExperience": "Save as Experience", - "saveAsExperienceTitle": "Save as Experience", - "distillHint": "Distill experience from this analysis for agents to reuse in future analysis.", + "saveAsExperience": "Save as Workflow", + "saveAsExperienceTitle": "Save as Workflow", + "distillHint": "Distill a workflow from this analysis for agents to save and replay in future sessions.", "distillFromHeading": "Distill from", "distillFromCaption": "Threads below will be sent to the LLM. Click a thread to inspect its events.", - "distillingOverlay": "Distilling experience… this may take a moment.", + "distillingOverlay": "Distilling workflow… this may take a moment.", "userInstruction": "User instruction (optional)", "userInstructionPlaceholder": "what to focus on, what to skip…", "distillationInstructions": "Distillation instructions (optional)", "distillationInstructionsPlaceholder": "e.g. focus on the data cleaning steps; skip exploratory chart variations; emphasise pitfalls we hit when joining tables…", - "distillExperience": "Distill Experience", - "distillStarted": "Distilling experience...", - "distilling": "Distilling experience...", - "distilled": "Experience saved", + "distillWorkflow": "Distill Workflow", + "distillStarted": "Distilling workflow...", + "distilling": "Distilling workflow...", + "distilled": "Workflow saved", "distillFailedRetry": "Save failed, retry", - "failedToDistill": "Failed to distill experience", - "distillSessionTitle": "Distill Session Experience", - "updateSessionTitle": "Update Session Experience", - "distillSessionHint": "Distill lessons from this analysis into a reusable knowledge document.", - "distillSessionUpdateHint": "Re-distill lessons from this analysis into the existing knowledge document.", + "failedToDistill": "Failed to distill workflow", + "distillSessionTitle": "Distill Session Workflow", + "updateSessionTitle": "Update Session Workflow", + "distillSessionHint": "Distill this analysis into a reusable workflow document that agents can replay.", + "distillSessionUpdateHint": "Re-distill this analysis into the existing workflow document.", "distillSessionNothing": "No completed analysis threads in this session yet.", "distillFromSession": "Distill from this session", - "experiencePlaceholderHint": "Save lessons learned", + "workflowPlaceholderHint": "Save this analysis as a workflow", "updateFromSession": "Update from this session", "updateFromSessionHint": "Refresh with new lessons", "addNewRule": "Add new rule", @@ -937,15 +937,21 @@ "itemCount": "({{count}})", "collapse": "Collapse", "expand": "Expand", - "emptyState": "Add rules or experiences to help AI agents work better.", - "rulesHint": "Rules — constraints the agent always follows. Click + to add your own.", - "experiencesHint": "Experiences — lessons distilled from your past analyses. Click the placeholder below to distill one from this session.", + "emptyState": "Add rules or workflows to help AI agents work better.", + "rulesHint": "Constraints the agent always follows.", + "workflowsHint": "Analyses distilled from past sessions that the agent can save and replay.", "markdownEditor": "Markdown Editor", "description": "Description", "descriptionPlaceholder": "Short summary of this rule (max {{max}} chars)", "alwaysApply": "Always loaded into AI", "alwaysApplyHint": "When enabled, this rule is always injected into every AI agent prompt, regardless of context", "charCount": "{{current}} / {{max}}", - "charCountExceeded": "Exceeds {{max}} character limit ({{current}} / {{max}})" + "charCountExceeded": "Exceeds {{max}} character limit ({{current}} / {{max}})", + "replay": "Replay", + "replayTooltip": "Replay this analysis on the current data", + "replayBusy": "The agent is busy — wait for it to finish before replaying.", + "replayNoData": "Load and focus a dataset before replaying a workflow.", + "replayStarted": "Replaying workflow on the current data…", + "replayPrompt": "Reproduce the following analysis workflow on the currently loaded data. Follow the steps in order, adapting any column references to the columns available in the current dataset. It's fine if the result isn't identical — reproduce the same overall analysis.\n\nBefore making large assumptions, check whether the current data can actually support the workflow. If there is a major discrepancy — e.g. a required field or measure is missing, the granularity or shape is very different, or a step has no sensible equivalent on this data — pause and ask me to confirm how to proceed (or briefly explain the mismatch and your proposed adaptation) instead of guessing. Minor differences (renamed columns, extra columns) can be adapted silently.\n\n{{content}}" } } diff --git a/src/i18n/locales/zh/common.json b/src/i18n/locales/zh/common.json index 1740e92a..244f93da 100644 --- a/src/i18n/locales/zh/common.json +++ b/src/i18n/locales/zh/common.json @@ -875,9 +875,9 @@ "knowledge": { "title": "Agent 知识", "rules": "规则", - "experiences": "经验", + "workflows": "工作流", "rulesDescription": "Agent 必须遵守的约束和编码规范", - "experiencesDescription": "从分析中提炼的可复用方法和技巧", + "workflowsDescription": "从过往会话中提炼、可供 Agent 保存与重放的可复用分析工作流", "newItem": "新建", "search": "搜索", "searchPlaceholder": "搜索知识...", @@ -902,29 +902,29 @@ "failedToSave": "保存知识失败", "failedToDelete": "删除知识失败", "failedToSearch": "搜索失败", - "saveAsExperience": "保存为经验", - "saveAsExperienceTitle": "保存为经验", - "distillHint": "从本次分析中提炼经验，供 Agent 在后续分析中复用。", + "saveAsExperience": "保存为工作流", + "saveAsExperienceTitle": "保存为工作流", + "distillHint": "从本次分析中提炼工作流，供 Agent 在后续会话中保存与重放。", "distillFromHeading": "提炼来源", "distillFromCaption": "以下线索将发送给 LLM。点击线索可查看其事件。", - "distillingOverlay": "正在提炼经验…请稍候。", + "distillingOverlay": "正在提炼工作流…请稍候。", "userInstruction": "用户指令（可选）", "userInstructionPlaceholder": "重点关注什么、跳过什么…", "distillationInstructions": "提炼指令（可选）", "distillationInstructionsPlaceholder": "例如：重点关注数据清洗步骤；跳过探索性图表变体；着重记录表连接时遇到的陷阱…", - "distillExperience": "提炼经验", - "distillStarted": "正在提炼经验...", - "distilling": "正在提炼经验...", - "distilled": "经验已保存", - "distillFailedRetry": "保存经验失败，重试", - "failedToDistill": "提炼经验失败", - "distillSessionTitle": "提炼会话经验", - "updateSessionTitle": "更新会话经验", - "distillSessionHint": "从本次分析中提炼经验，生成一篇可复用的知识文档。", - "distillSessionUpdateHint": "重新提炼本次分析的经验，覆盖现有的知识文档。", + "distillWorkflow": "提炼工作流", + "distillStarted": "正在提炼工作流...", + "distilling": "正在提炼工作流...", + "distilled": "工作流已保存", + "distillFailedRetry": "保存工作流失败，重试", + "failedToDistill": "提炼工作流失败", + "distillSessionTitle": "提炼会话工作流", + "updateSessionTitle": "更新会话工作流", + "distillSessionHint": "将本次分析提炼为一篇可供 Agent 重放的可复用工作流文档。", + "distillSessionUpdateHint": "将本次分析重新提炼到现有的工作流文档中。", "distillSessionNothing": "本会话还没有可提炼的分析线索。", "distillFromSession": "从本会话提炼", - "experiencePlaceholderHint": "保存分析中的经验", + "workflowPlaceholderHint": "将本次分析保存为工作流", "updateFromSession": "从本会话更新", "updateFromSessionHint": "用新经验刷新该条目", "addNewRule": "添加新规则", @@ -937,15 +937,21 @@ "itemCount": "（{{count}}）", "collapse": "收起", "expand": "展开", - "emptyState": "添加规则、技能或经验，帮助 AI Agent 更好地工作。", - "rulesHint": "规则 — Agent 始终遵守的约束。点击 + 添加你自己的规则。", - "experiencesHint": "经验 — 从你过往分析中提炼出的经验。点击下方占位项可从本会话提炼一条。", + "emptyState": "添加规则或工作流，帮助 AI Agent 更好地工作。", + "rulesHint": "Agent 始终遵守的约束。", + "workflowsHint": "从过往会话中提炼、Agent 可保存与重放的分析。", "markdownEditor": "Markdown 编辑器", "description": "描述", "descriptionPlaceholder": "规则的简短描述（最多 {{max}} 字符）", "alwaysApply": "始终加载到 AI", "alwaysApplyHint": "启用后，无论什么场景，此规则都会自动注入到每次 AI Agent 的提示词中", "charCount": "{{current}} / {{max}}", - "charCountExceeded": "超出 {{max}} 字符限制（{{current}} / {{max}}）" + "charCountExceeded": "超过 {{max}} 字符限制（{{current}} / {{max}}）", + "replay": "重放", + "replayTooltip": "在当前数据上重放此分析", + "replayBusy": "Agent 正忙——请等待其完成后再重放。", + "replayNoData": "请先加载并聚焦一个数据集，再重放工作流。", + "replayStarted": "正在当前数据上重放工作流…", + "replayPrompt": "在当前已加载的数据上复现以下分析流程。按顺序执行各步骤，并将其中的列引用调整为当前数据集中可用的列。结果不必完全一致——复现同样的整体分析即可。\n\n在做出较大假设之前，请先确认当前数据是否真的能支撑该流程。如果存在重大差异——例如缺少必需的字段或度量、数据粒度或结构差异很大、或某个步骤在当前数据上没有合理的对应方式——请暂停并向我确认如何继续（或简要说明不匹配之处及你建议的调整方案），而不要凭空猜测。对于细微差异（列被重命名、存在额外的列）可以直接静默调整。\n\n{{content}}" } } diff --git a/src/views/DataFrameTable.tsx b/src/views/DataFrameTable.tsx index dd32ecc9..afb03bbd 100644 --- a/src/views/DataFrameTable.tsx +++ b/src/views/DataFrameTable.tsx @@ -127,17 +127,24 @@ export const DataFrameTable: React.FC = ({ )} {displayCols.map((col, i) => { const desc = col !== '\u2026' ? columnDescriptions?.[col] : undefined; + if (desc) { + return ( + + + {col} + + + ); + } return ( - - - {col} - - + + {col} + ); })} diff --git a/src/views/DataSourceSidebar.tsx b/src/views/DataSourceSidebar.tsx index 7381a423..c0502fd0 100644 --- a/src/views/DataSourceSidebar.tsx +++ b/src/views/DataSourceSidebar.tsx @@ -157,7 +157,7 @@ export const DataSourceSidebar: React.FC<{ // appears when they try to add a new connector or link a folder. const [initialTab, setInitialTab] = useState<'sources' | 'sessions' | 'knowledge'>('sources'); - // External callers (e.g. SaveExperienceButton on success) can ask the + // External callers (e.g. workflow distill on success) can ask the // sidebar to open and switch to a specific tab. useEffect(() => { const handler = (e: Event) => { diff --git a/src/views/DataThread.tsx b/src/views/DataThread.tsx index 940cb4ef..248dbe75 100644 --- a/src/views/DataThread.tsx +++ b/src/views/DataThread.tsx @@ -1333,17 +1333,6 @@ let SingleThreadGroupView: FC<{ const mergeIds = derivedTable?.derive?.source as string[] | undefined; if (entry.role === 'instruction' && mergeNames && mergeNames.length > 0 && mergeIds && mergeIds.length > 0) { const nextKey = sourceSetKey(mergeIds); - // eslint-disable-next-line no-console - console.log('[merge-node check]', { - tableId, - parentTableId: parentTable?.id, - initialSourceIds, - prevSourceKey, - mergeIds, - mergeNames, - nextKey, - fires: nextKey !== prevSourceKey, - }); if (nextKey !== prevSourceKey) { const mergeColor = highlighted ? theme.palette.primary.main : theme.palette.text.secondary; timelineItems.push({ diff --git a/src/views/InteractionEntryCard.tsx b/src/views/InteractionEntryCard.tsx index 8cfd0000..79686ca2 100644 --- a/src/views/InteractionEntryCard.tsx +++ b/src/views/InteractionEntryCard.tsx @@ -242,6 +242,11 @@ export const InteractionEntryCard: React.FC = memo(({ // so they should read stronger than the agent's bubbles. backgroundColor: palette.bgcolor, border: `1px solid ${borderColor.component}`, + // Cap very long instructions (e.g. a replayed workflow) so the + // card stays compact; the full text scrolls within the cap. + maxHeight: 160, + overflowY: 'auto', + overscrollBehavior: 'contain', ...(highlighted ? { borderLeft: `2px solid ${palette.main}` } : {}), ...clickSx, }}> diff --git a/src/views/KnowledgePanel.tsx b/src/views/KnowledgePanel.tsx index 8d8a111f..05343a0a 100644 --- a/src/views/KnowledgePanel.tsx +++ b/src/views/KnowledgePanel.tsx @@ -4,16 +4,16 @@ /** * KnowledgePanel — panel for browsing and editing knowledge items. * - * Shows two collapsible sections: Rules (flat) and Experiences (flat). + * Shows two collapsible sections: Rules (flat) and Workflows (flat). * Items are tagged for organization; no subdirectory grouping. * Supports search, edit, and delete. Rules can be created directly by - * the user via the "+" affordance; experiences are produced by the + * the user via the "+" affordance; workflows are produced by the * agent's distillation flow (see SessionDistill). */ -import React, { useState, useCallback, useEffect, useRef } from 'react'; +import React, { useState, useCallback, useEffect } from 'react'; import { useTranslation } from 'react-i18next'; -import { useSelector } from 'react-redux'; +import { useSelector, useDispatch } from 'react-redux'; import { Box, Typography, @@ -26,7 +26,6 @@ import { DialogContent, DialogActions, CircularProgress, - Chip, Divider, } from '@mui/material'; import { alpha } from '@mui/material/styles'; @@ -34,6 +33,7 @@ import AddIcon from '@mui/icons-material/Add'; import DeleteOutlineIcon from '@mui/icons-material/DeleteOutline'; import DescriptionOutlinedIcon from '@mui/icons-material/DescriptionOutlined'; import SmartToyOutlinedIcon from '@mui/icons-material/SmartToyOutlined'; +import PlayArrowIcon from '@mui/icons-material/PlayArrow'; import RefreshIcon from '@mui/icons-material/Refresh'; import Editor from 'react-simple-code-editor'; @@ -41,9 +41,9 @@ import { useKnowledgeStore } from '../app/useKnowledgeStore'; import { deleteKnowledge, type KnowledgeCategory } from '../api/knowledgeApi'; import type { KnowledgeItem } from '../api/knowledgeApi'; import { borderColor, radius } from '../app/tokens'; -import { type DataFormulatorState } from '../app/dfSlice'; -import { isLeafDerivedTable, buildLeafEvents } from './experienceContext'; -import { SessionDistillDialog, findSessionExperience } from './SessionDistill'; +import { dfActions, type DataFormulatorState } from '../app/dfSlice'; +import { isLeafDerivedTable, buildLeafEvents } from './workflowContext'; +import { SessionDistillDialog, findSessionWorkflow } from './SessionDistill'; // Default file name and seed body for a brand-new rule. Rules are plain // Markdown — the user just edits the body; no front matter is required. @@ -58,19 +58,18 @@ Describe the constraints or conventions the agent should follow. interface ActionRowProps { icon: React.ReactNode; label: string; - hint: string; onClick: () => void; } -const ActionRow: React.FC = ({ icon, label, hint, onClick }) => ( +const ActionRow: React.FC = ({ icon, label, onClick }) => ( `1px solid ${alpha(theme.palette.primary.main, 0.5)}`, @@ -88,22 +87,12 @@ const ActionRow: React.FC = ({ icon, label, hint, onClick }) => userSelect: 'none', }} > - {icon} - - - {label} - - - {hint} - - + {icon} + + {label} + ); @@ -112,8 +101,9 @@ const ActionRow: React.FC = ({ icon, label, hint, onClick }) => export const KnowledgePanel: React.FC = () => { const { t } = useTranslation(); const store = useKnowledgeStore(); + const dispatch = useDispatch(); - // For the "distill from this session" placeholder under EXPERIENCES. + // For the "distill from this session" placeholder under WORKFLOWS. const tables = useSelector((s: DataFormulatorState) => s.tables); const charts = useSelector((s: DataFormulatorState) => s.charts); const conceptShelfItems = useSelector((s: DataFormulatorState) => s.conceptShelfItems); @@ -183,35 +173,6 @@ export const KnowledgePanel: React.FC = () => { setEditorLoading(false); }, [store]); - // Pending request to auto-open an entry once it appears in the store - // (e.g. after the SessionDistillDialog finishes distilling). - const pendingOpenRef = useRef<{ category: KnowledgeCategory; path: string } | null>(null); - - useEffect(() => { - const handler = (e: Event) => { - const detail = (e as CustomEvent).detail || {}; - const category = (detail.category as KnowledgeCategory | undefined) ?? 'experiences'; - const path = detail.path as string | undefined; - if (path) { - pendingOpenRef.current = { category, path }; - } - }; - window.addEventListener('open-knowledge-panel', handler); - return () => window.removeEventListener('open-knowledge-panel', handler); - }, []); - - // When the requested entry shows up in the store, open its editor. - useEffect(() => { - const pending = pendingOpenRef.current; - if (!pending) return; - const cat = store.stateMap[pending.category]; - if (!cat?.loaded) return; - const item = cat.items.find(i => i.path === pending.path); - if (!item) return; - pendingOpenRef.current = null; - openEditDialog(pending.category, item); - }, [store.stateMap, openEditDialog]); - const handleSave = useCallback(async () => { if (!editorPath.trim() || !editorContent.trim()) return; setEditorSaving(true); @@ -237,9 +198,9 @@ export const KnowledgePanel: React.FC = () => { }, [deleteTarget, store]); // ── Distill from current session ──────────────────────────────────── - // The EXPERIENCES placeholder under EXPERIENCES is bound to the + // The WORKFLOWS placeholder is bound to the // active workspace. When the workspace already has a distilled - // experience (matched by `sourceWorkspaceId` in front matter) we + // workflow (matched by `sourceWorkspaceId` in front matter) we // expose an inline ⟳ Update affordance on the existing entry; // otherwise the placeholder opens the dialog in *create* mode. // See design-docs/24-session-scoped-distillation.md. @@ -265,9 +226,9 @@ export const KnowledgePanel: React.FC = () => { const selectedModel = allModels.find(m => m.id === selectedModelId); const canDistillFromSession = hasDistillableSession && !!selectedModel && !!activeWorkspace; - const sessionExperience = React.useMemo( - () => findSessionExperience( - store.stateMap['experiences'].items, + const sessionWorkflow = React.useMemo( + () => findSessionWorkflow( + store.stateMap['workflows'].items, activeWorkspace?.id, ), [store.stateMap, activeWorkspace?.id], @@ -278,6 +239,21 @@ export const KnowledgePanel: React.FC = () => { setSessionDialogOpen(true); }, []); + // ── Replay a workflow ──────────────────────────────────────────── + // Reads the workflow body and asks the data agent (in SimpleChartRecBox) + // to reproduce the captured workflow on the currently loaded data. v1 is + // deliberately simple: we hand the whole workflow to the agent in one + // request via a window event and let it figure out the rest. + // See discussion/replayable-experience-workflow.md. + const handleReplay = useCallback(async (item: KnowledgeItem) => { + const content = await store.read('workflows', item.path); + if (content == null) return; + const prompt = t('knowledge.replayPrompt', { content }); + window.dispatchEvent(new CustomEvent('df-replay-workflow', { + detail: { prompt, title: item.title }, + })); + }, [store, t]); + // ── Render section ────────────────────────────────────────────────── @@ -285,81 +261,85 @@ export const KnowledgePanel: React.FC = () => { category: KnowledgeCategory, item: KnowledgeItem, ) => { - const displayName = item.path || item.title; + const displayTitle = (item.title || '').replace(/^\s*(?:Workflow|Experience) from .+?:\s*/i, '').trim(); + const primary = displayTitle || item.title || item.path; return ( openEditDialog(category, item)} sx={{ display: 'flex', alignItems: 'flex-start', gap: 0.75, - px: 1.5, py: 0.75, + px: 1.5, py: 0.625, cursor: 'pointer', color: 'text.primary', '&:hover': { bgcolor: 'action.hover' }, - '&:hover .item-actions': { visibility: 'visible' }, + '&:hover .item-actions': { display: 'inline-flex' }, userSelect: 'none', }} > - + - - {displayName} + + {primary} - {item.tags.length > 0 && ( - - {item.tags.map(tag => ( - - ))} - - )} {item.source === 'agent_summarized' && ( )} - + + {category === 'workflows' && ( + + { e.stopPropagation(); handleReplay(item); }} + sx={{ + p: 0.25, + color: 'primary.main', + '&:hover': { bgcolor: theme => alpha(theme.palette.primary.main, 0.08) }, + }} + > + + + + )} { e.stopPropagation(); setDeleteTarget({ category, path: item.path, title: item.title }); }} - sx={{ p: 0.25, color: 'text.secondary', '&:hover': { color: 'error.main' } }} + sx={{ p: 0.25, display: 'none', color: 'text.secondary', '&:hover': { color: 'error.main' } }} > - + ); - }, [openEditDialog, t]); + }, [openEditDialog, t, handleReplay]); const renderCategorySection = useCallback(( category: KnowledgeCategory, label: string, + hint: string, ) => { const state = store.stateMap[category]; // Persistent action row at the top of the section. Rules: opens - // the create dialog. Experiences: opens the session distill + // the create dialog. Workflows: opens the session distill // dialog in create or update mode depending on whether the active - // workspace already has a distilled experience. + // workspace already has a distilled workflow. // See design-docs/24-session-scoped-distillation.md. const renderActionRow = () => { if (category === 'rules') { return ( } + icon={} label={t('knowledge.addNewRule', { defaultValue: 'Add new rule' })} - hint={t('knowledge.addNewRuleHint', { defaultValue: 'Set a convention for the agent' })} onClick={() => openCreateDialog('rules')} /> ); } - // experiences + // workflows if (!canDistillFromSession) { // No active workspace, no model, or no distillable thread // yet — show a passive hint instead of a dead action. @@ -370,15 +350,12 @@ export const KnowledgePanel: React.FC = () => { ); } - const updateMode = !!sessionExperience; + const updateMode = !!sessionWorkflow; if (sessionDistilling) { return ( } - label={t('knowledge.distilling', { defaultValue: 'Distilling experience…' })} - hint={updateMode - ? t('knowledge.updateFromSessionHint', { defaultValue: 'Refresh with new lessons' }) - : t('knowledge.experiencePlaceholderHint', { defaultValue: 'Save lessons learned' })} + icon={} + label={t('knowledge.distilling', { defaultValue: 'Distilling workflow…' })} onClick={() => openSessionDistillDialog(updateMode)} /> ); @@ -386,32 +363,44 @@ export const KnowledgePanel: React.FC = () => { return ( - : } + ? + : } label={updateMode ? t('knowledge.updateFromSession', { defaultValue: 'Update from this session' }) : t('knowledge.distillFromSession', { defaultValue: 'Distill from this session' })} - hint={updateMode - ? t('knowledge.updateFromSessionHint', { defaultValue: 'Refresh with new lessons' }) - : t('knowledge.experiencePlaceholderHint', { defaultValue: 'Save lessons learned' })} onClick={() => openSessionDistillDialog(updateMode)} /> ); }; return ( - + - + {label} + {/* Always-visible guidance for the section, set off by a + subtle left accent line below the title. */} + alpha(theme.palette.primary.main, 0.25), + }} + > + + {hint} + + + {state.loading && ( @@ -421,36 +410,18 @@ export const KnowledgePanel: React.FC = () => { {state.items.map(item => renderItem(category, item))} ); - }, [store.stateMap, renderItem, openCreateDialog, t, canDistillFromSession, sessionExperience, sessionDistilling, openSessionDistillDialog]); + }, [store.stateMap, renderItem, openCreateDialog, t, canDistillFromSession, sessionWorkflow, sessionDistilling, openSessionDistillDialog]); // ── Main render ───────────────────────────────────────────────────── return ( - {/* Persistent hint — explains Rules vs Experiences without - requiring the user to scroll past empty-state messages. */} - - - {t('knowledge.rulesHint')} - - - {t('knowledge.experiencesHint')} - - - - {/* Content area */} + {/* Content area. Rules vs Workflows guidance is surfaced via an + info icon next to each section title (see renderCategorySection). */} - {renderCategorySection('rules', t('knowledge.rules'))} - {renderCategorySection('experiences', t('knowledge.experiences'))} + {renderCategorySection('rules', t('knowledge.rules'), t('knowledge.rulesHint'))} + {renderCategorySection('workflows', t('knowledge.workflows'), t('knowledge.workflowsHint'))} @@ -515,22 +486,6 @@ export const KnowledgePanel: React.FC = () => { }} /> - {(() => { - const bodyLimit = store.limits[editorCategory as keyof typeof store.limits] as number | undefined; - if (!bodyLimit) return null; - const bodyLen = editorContent.trim().length; - const exceeded = bodyLen > bodyLimit; - return ( - bodyLimit * 0.9 ? 'warning.main' : 'text.disabled', - }}> - {exceeded - ? t('knowledge.charCountExceeded', { max: bodyLimit, current: bodyLen }) - : t('knowledge.charCount', { max: bodyLimit, current: bodyLen })} - - ); - })()} )} @@ -555,7 +510,6 @@ export const KnowledgePanel: React.FC = () => { editorSaving || !editorContent.trim() || !editorPath.trim() - || editorContent.trim().length > (store.limits[editorCategory as keyof typeof store.limits] as number ?? Infinity) } variant="contained" sx={{ textTransform: 'none', fontSize: 12 }} diff --git a/src/views/SessionDistill.tsx b/src/views/SessionDistill.tsx index fbff4f3b..fd9efeae 100644 --- a/src/views/SessionDistill.tsx +++ b/src/views/SessionDistill.tsx @@ -2,19 +2,19 @@ // Licensed under the MIT License. /** - * SessionDistill — session-scoped experience distillation. + * SessionDistill — session-scoped workflow distillation. * * Replaces the old per-result distillation flow with a single * session-bound entry. See design-docs/24-session-scoped-distillation.md. * * Exports: - * - buildSessionExperienceContext(workspace, threads): state-independent + * - buildSessionWorkflowContext(workspace, threads): state-independent * payload builder (with size budgeting, see §3.5 of the design doc). * - collectSessionThreads(tables, charts, fields): leaf discovery + per-leaf * event walk against live DataFormulator state. * - SessionDistillDialog: the dialog used by KnowledgePanel for both * create and update modes. - * - findSessionExperience: lookup an existing session experience by + * - findSessionWorkflow: lookup an existing session workflow by * workspace id. */ @@ -51,16 +51,16 @@ import { import { store, type AppDispatch } from '../app/store'; import { handleApiError } from '../app/errorHandler'; import { - distillSessionExperience, + distillSessionWorkflow, type KnowledgeItem, - type SessionExperienceContext, + type SessionWorkflowContext, } from '../api/knowledgeApi'; import { buildLeafEvents, buildDistillModelConfig, isLeafDerivedTable, TOOL_USES_CODE_FONT, -} from './experienceContext'; +} from './workflowContext'; // --------------------------------------------------------------------------- // Payload size budget (design-docs/24 §3.5) @@ -81,7 +81,7 @@ const SESSION_EVENT_BUDGET = 60_000; // bytes of JSON-serialized events // --------------------------------------------------------------------------- /** - * One pre-built thread, ready for `buildSessionExperienceContext`. + * One pre-built thread, ready for `buildSessionWorkflowContext`. * * Callers produce these by walking their own tables (see * `collectSessionThreads` for the in-app implementation) or with hand-built @@ -97,7 +97,7 @@ export interface SessionThread { export interface BuildSessionResult { /** Payload as it will be sent (after trimming). */ - payload: SessionExperienceContext; + payload: SessionWorkflowContext; /** Display threads with labels for the preview UI (post-trim). */ threads: SessionThread[]; /** Aggregate stats for the preview (post-trim). */ @@ -107,14 +107,14 @@ export interface BuildSessionResult { } // --------------------------------------------------------------------------- -// findSessionExperience +// findSessionWorkflow // --------------------------------------------------------------------------- /** - * Find the experience entry distilled from the given workspace, if any. + * Find the workflow entry distilled from the given workspace, if any. * Returns the first match; the backend ensures at most one per workspace. */ -export function findSessionExperience( +export function findSessionWorkflow( items: KnowledgeItem[] | undefined, workspaceId: string | undefined, ): KnowledgeItem | undefined { @@ -132,7 +132,7 @@ export function findSessionExperience( * * Threads with no user message are filtered out. Returns `[]` when the * session has no distillable thread. Not used in tests — tests construct - * `SessionThread[]` directly and call `buildSessionExperienceContext`. + * `SessionThread[]` directly and call `buildSessionWorkflowContext`. */ export function collectSessionThreads( tables: DictTable[], @@ -162,16 +162,16 @@ export function collectSessionThreads( } // --------------------------------------------------------------------------- -// buildSessionExperienceContext — pure (workspace, threads) → payload +// buildSessionWorkflowContext — pure (workspace, threads) → payload // --------------------------------------------------------------------------- /** - * Assemble the multi-thread payload sent to `/api/knowledge/distill-experience`. + * Assemble the multi-thread payload sent to `/api/knowledge/distill-workflow`. * * State-independent: takes pre-built threads and a workspace identity. * Returns `null` when `threads` is empty. */ -export function buildSessionExperienceContext( +export function buildSessionWorkflowContext( workspace: { id: string; displayName: string }, threads: SessionThread[], ): BuildSessionResult | null { @@ -179,7 +179,7 @@ export function buildSessionExperienceContext( const { trimmedThreads, notes } = trimToBudget(threads, SESSION_EVENT_BUDGET); - const payload: SessionExperienceContext = { + const payload: SessionWorkflowContext = { context_id: workspace.id, workspace_id: workspace.id, workspace_name: workspace.displayName, @@ -308,7 +308,7 @@ export const SessionDistillDialog: React.FC = ({ const built = useMemo(() => { if (!open || !activeWorkspace) return null; const threads = collectSessionThreads(tables, charts, conceptShelfItems); - return buildSessionExperienceContext(activeWorkspace, threads); + return buildSessionWorkflowContext(activeWorkspace, threads); }, [open, activeWorkspace, tables, charts, conceptShelfItems]); const [userInstruction, setUserInstruction] = useState(''); @@ -330,6 +330,10 @@ export const SessionDistillDialog: React.FC = ({ setStatus('running'); onRunningChange?.(true); const instruction = userInstruction.trim() || undefined; + // Close the dialog right away — distillation continues in the + // background and surfaces its result via the events/toast below. + setUserInstruction(''); + onClose(); try { const modelConfig = buildDistillModelConfig(selectedModel as ModelConfig); @@ -338,7 +342,7 @@ export const SessionDistillDialog: React.FC = ({ const timeoutId = setTimeout(() => controller.abort(), timeoutSeconds * 1000); let result; try { - result = await distillSessionExperience( + result = await distillSessionWorkflow( built.payload, modelConfig, instruction, timeoutSeconds, controller.signal, ); } finally { @@ -351,14 +355,12 @@ export const SessionDistillDialog: React.FC = ({ value: t('knowledge.distilled'), })); window.dispatchEvent(new CustomEvent('knowledge-changed', { - detail: { category: 'experiences' }, + detail: { category: 'workflows' }, })); window.dispatchEvent(new CustomEvent('open-knowledge-panel', { - detail: { category: 'experiences', path: result.path }, + detail: { category: 'workflows', path: result.path }, })); setStatus('idle'); - setUserInstruction(''); - onClose(); } catch (e: unknown) { setStatus('failed'); handleApiError(e, 'knowledge'); @@ -375,8 +377,8 @@ export const SessionDistillDialog: React.FC = ({ {updateMode - ? t('knowledge.updateSessionTitle', { defaultValue: 'Update Session Experience' }) - : t('knowledge.distillSessionTitle', { defaultValue: 'Distill Session Experience' })} + ? t('knowledge.updateSessionTitle', { defaultValue: 'Update Session Workflow' }) + : t('knowledge.distillSessionTitle', { defaultValue: 'Distill Session Workflow' })} = ({ {updateMode ? t('knowledge.distillSessionUpdateHint', { - defaultValue: 'Re-distill lessons from this analysis into the existing knowledge document.', + defaultValue: 'Re-distill this analysis into the existing workflow document.', }) : t('knowledge.distillSessionHint', { - defaultValue: 'Distill lessons from this analysis into a reusable knowledge document.', + defaultValue: 'Distill this analysis into a reusable workflow document that agents can replay.', })} @@ -479,7 +481,7 @@ export const SessionDistillDialog: React.FC = ({ ? t('knowledge.distilling') : updateMode ? t('knowledge.updateSession', { defaultValue: 'Update' }) - : t('knowledge.distillExperience')} + : t('knowledge.distillWorkflow')} diff --git a/src/views/SimpleChartRecBox.tsx b/src/views/SimpleChartRecBox.tsx index 23b30c7c..e94d2192 100644 --- a/src/views/SimpleChartRecBox.tsx +++ b/src/views/SimpleChartRecBox.tsx @@ -38,8 +38,6 @@ import AddIcon from '@mui/icons-material/Add'; import TipsAndUpdatesIcon from '@mui/icons-material/TipsAndUpdates'; import StopIcon from '@mui/icons-material/Stop'; -import AutoGraphIcon from '@mui/icons-material/AutoGraph'; -import DescriptionOutlinedIcon from '@mui/icons-material/DescriptionOutlined'; import InsertDriveFileOutlinedIcon from '@mui/icons-material/InsertDriveFileOutlined'; import { borderColor, transition } from '../app/tokens'; import { Theme } from '@mui/material/styles'; @@ -70,7 +68,7 @@ const AgentWorkingOverlay: FC<{ message?: string; elapsed?: number; theme: Theme }}> - + {t('chartRec.agentWorking')} @@ -96,13 +94,13 @@ const AgentWorkingOverlay: FC<{ message?: string; elapsed?: number; theme: Theme )} {latestMessage}{elapsedSuffix} @@ -1354,6 +1352,39 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ exploreFromChat(prompt, undefined, displayPrompt); }, [reportFromChat, exploreFromChat, selectedAgent, clarificationQuestions, clarifyAnswers]); + // Replay a workflow: the KnowledgePanel fires `df-replay-workflow` + // with a prompt describing the captured workflow; we hand it straight to + // the data agent on the currently focused dataset. v1 is deliberately + // simple — one request, let the agent reproduce the analysis on its own. + // See discussion/replayable-experience-workflow.md. + useEffect(() => { + const handler = (e: Event) => { + const prompt = (e as CustomEvent).detail?.prompt as string | undefined; + if (!prompt) return; + if (isChatFormulating) { + dispatch(dfActions.addMessages({ + timestamp: Date.now(), type: 'error', + component: 'data-agent', value: t('knowledge.replayBusy'), + })); + return; + } + if (!focusedTableId) { + dispatch(dfActions.addMessages({ + timestamp: Date.now(), type: 'error', + component: 'data-agent', value: t('knowledge.replayNoData'), + })); + return; + } + dispatch(dfActions.addMessages({ + timestamp: Date.now(), type: 'info', + component: 'data-agent', value: t('knowledge.replayStarted'), + })); + exploreFromChat(prompt); + }; + window.addEventListener('df-replay-workflow', handler); + return () => window.removeEventListener('df-replay-workflow', handler); + }, [exploreFromChat, isChatFormulating, focusedTableId, dispatch, t]); + const resumeFromClarification = useCallback((responses: ClarificationResponse[]) => { if (!pendingClarification) return; // Pass the formatted display string as `prompt` — it powers both the @@ -1744,9 +1775,6 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ '&:hover': { backgroundColor: alpha(isReportMode ? theme.palette.warning.main : theme.palette.primary.main, 0.08) }, }} > - {selectedAgent === 'explore' - ? - : } {selectedAgent === 'explore' ? t('chartRec.modeExplore') : t('chartRec.modeReport')} @@ -1761,7 +1789,7 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ submitChat(t('chartRec.exploreIdeasPrompt'), undefined, t('chartRec.askedForRecommendations'))} > diff --git a/src/views/experienceContext.ts b/src/views/workflowContext.ts similarity index 98% rename from src/views/experienceContext.ts rename to src/views/workflowContext.ts index 98ec1c80..dac6e006 100644 --- a/src/views/experienceContext.ts +++ b/src/views/workflowContext.ts @@ -2,8 +2,8 @@ // Licensed under the MIT License. /** - * experienceContext — pure helpers that turn DataFormulator state into - * the timeline payload sent to `/api/knowledge/distill-experience`. + * workflowContext — pure helpers that turn DataFormulator state into + * the timeline payload sent to `/api/knowledge/distill-workflow`. * * No React, no Redux. Used by: * - SessionDistill.collectSessionThreads (live distillation) diff --git a/tests/backend/agents/test_agent_knowledge_integration.py b/tests/backend/agents/test_agent_knowledge_integration.py index 3efc65df..4d738635 100644 --- a/tests/backend/agents/test_agent_knowledge_integration.py +++ b/tests/backend/agents/test_agent_knowledge_integration.py @@ -62,7 +62,7 @@ def user_home(tmp_path): rules_dir.mkdir(parents=True) (rules_dir / "roi.md").write_text(RULE_MD, encoding="utf-8") - exp_dir = tmp_path / "knowledge" / "experiences" / "cleaning" + exp_dir = tmp_path / "knowledge" / "workflows" / "cleaning" exp_dir.mkdir(parents=True) (exp_dir / "missing.md").write_text(SKILL_MD, encoding="utf-8") @@ -170,11 +170,11 @@ def test_no_match_no_injection(self, mock_client, mock_workspace, user_home): def test_max_five_items(self, mock_client, mock_workspace, tmp_path): rules_dir = tmp_path / "knowledge" / "rules" rules_dir.mkdir(parents=True) - exp_dir = tmp_path / "knowledge" / "experiences" / "common" + exp_dir = tmp_path / "knowledge" / "workflows" / "common" exp_dir.mkdir(parents=True) for i in range(10): (exp_dir / f"exp-{i}.md").write_text( - f"---\ntitle: Common Experience {i}\ntags: [common]\n" + f"---\ntitle: Common Workflow {i}\ntags: [common]\n" f"created: 2026-04-26\nupdated: 2026-04-26\n---\n" f"Content about common topic {i}.\n", encoding="utf-8", @@ -247,7 +247,7 @@ def test_agent_works_without_knowledge(self, mock_client, mock_workspace): def test_empty_knowledge_dir(self, mock_client, mock_workspace, tmp_path): """Agent with empty knowledge dir works normally.""" (tmp_path / "knowledge" / "rules").mkdir(parents=True) - (tmp_path / "knowledge" / "experiences").mkdir(parents=True) + (tmp_path / "knowledge" / "workflows").mkdir(parents=True) agent = _make_agent(mock_client, mock_workspace, tmp_path) prompt = agent._build_system_prompt() assert "User Rules" not in prompt diff --git a/tests/backend/agents/test_experience_distill.py b/tests/backend/agents/test_workflow_distill.py similarity index 74% rename from tests/backend/agents/test_experience_distill.py rename to tests/backend/agents/test_workflow_distill.py index a3b823c8..e44d4a86 100644 --- a/tests/backend/agents/test_experience_distill.py +++ b/tests/backend/agents/test_workflow_distill.py @@ -1,13 +1,13 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -"""Tests for ExperienceDistillAgent and the /api/knowledge/distill-experience endpoint. +"""Tests for WorkflowDistillAgent and the /api/knowledge/distill-workflow endpoint. Covers: -- _extract_context_summary correctly extracts experience context +- _extract_context_summary correctly extracts workflow context - Output Markdown includes valid YAML front matter - front matter contains source: distill and source metadata -- Generated experience file written to correct directory +- Generated workflow file written to correct directory - category_hint controls sub-directory """ @@ -18,8 +18,14 @@ import flask import pytest -from data_formulator.agents.agent_experience_distill import ExperienceDistillAgent -from data_formulator.knowledge.store import parse_front_matter +from data_formulator.agents.agent_workflow_distill import WorkflowDistillAgent +from data_formulator.knowledge.store import ( + KNOWLEDGE_LIMITS, + WORKFLOW_HARD_MAX, + parse_front_matter, +) + +WORKFLOW_SOFT_LIMIT = KNOWLEDGE_LIMITS["workflows"] pytestmark = [pytest.mark.backend] @@ -73,7 +79,7 @@ }, ] -SAMPLE_EXPERIENCE_CONTEXT = { +SAMPLE_WORKFLOW_CONTEXT = { "context_id": "ws-1", "workspace_id": "ws-1", "workspace_name": "Sales Region Analysis", @@ -86,7 +92,7 @@ class TestExtractContextSummary: def test_renders_each_event_type(self): - summary = ExperienceDistillAgent._extract_context_summary(SAMPLE_EXPERIENCE_CONTEXT) + summary = WorkflowDistillAgent._extract_context_summary(SAMPLE_WORKFLOW_CONTEXT) # message events assert "[user→data-agent/prompt]" in summary assert "Show sales by region" in summary @@ -113,7 +119,7 @@ def test_renders_each_event_type(self): assert "encoding: x=region(nominal)" in summary def test_empty_events_returns_marker(self): - summary = ExperienceDistillAgent._extract_context_summary({}) + summary = WorkflowDistillAgent._extract_context_summary({}) assert summary == "(empty context)" def test_user_content_is_not_displaycontent(self): @@ -131,7 +137,7 @@ def test_user_content_is_not_displaycontent(self): }], }], } - summary = ExperienceDistillAgent._extract_context_summary(ctx) + summary = WorkflowDistillAgent._extract_context_summary(ctx) assert "raw text" in summary def test_skips_non_dict_events(self): @@ -140,7 +146,7 @@ def test_skips_non_dict_events(self): {"type": "message", "from": "user", "to": "data-agent", "role": "prompt", "content": "ok"}, ]}]} - summary = ExperienceDistillAgent._extract_context_summary(ctx) + summary = WorkflowDistillAgent._extract_context_summary(ctx) assert "[user→data-agent/prompt]" in summary # No crashes; the bogus entries are silently dropped. @@ -154,7 +160,7 @@ def test_create_table_basic(self): "sample_rows": [{"a": 1}], "code": "x = 1", }]}]} - summary = ExperienceDistillAgent._extract_context_summary(ctx) + summary = WorkflowDistillAgent._extract_context_summary(ctx) assert "[create_table] t1" in summary def test_create_chart_without_encoding(self): @@ -163,7 +169,7 @@ def test_create_chart_without_encoding(self): "related_table_id": "t1", "mark_or_type": "line", }]}]} - summary = ExperienceDistillAgent._extract_context_summary(ctx) + summary = WorkflowDistillAgent._extract_context_summary(ctx) assert "[create_chart] line on t1" in summary assert "encoding:" not in summary @@ -187,7 +193,7 @@ def test_renders_multi_thread_with_headers(self): }, ], } - summary = ExperienceDistillAgent._extract_context_summary(ctx) + summary = WorkflowDistillAgent._extract_context_summary(ctx) assert "### Thread 1 (id=leaf-a)" in summary assert "### Thread 2 (id=leaf-b)" in summary assert "load gas prices" in summary @@ -236,10 +242,10 @@ def _mock_client(self): def test_produces_valid_markdown(self): client = self._mock_client() - agent = ExperienceDistillAgent(client=client) + agent = WorkflowDistillAgent(client=client) with patch.object(agent, "_call_llm", return_value=MOCK_CONTEXT_RESPONSE): - result = agent.run(SAMPLE_EXPERIENCE_CONTEXT) + result = agent.run(SAMPLE_WORKFLOW_CONTEXT) assert result.startswith("---") meta, body = parse_front_matter(result) @@ -249,11 +255,11 @@ def test_produces_valid_markdown(self): def test_fallback_front_matter_added(self): client = self._mock_client() - agent = ExperienceDistillAgent(client=client) + agent = WorkflowDistillAgent(client=client) no_fm_response = "# Sales Analysis\n\nJust some content." with patch.object(agent, "_call_llm", return_value=no_fm_response): - result = agent.run(SAMPLE_EXPERIENCE_CONTEXT) + result = agent.run(SAMPLE_WORKFLOW_CONTEXT) assert result.startswith("---") meta, _ = parse_front_matter(result) @@ -261,11 +267,11 @@ def test_fallback_front_matter_added(self): assert meta["source_context"] == "ws-1" def test_retries_once_when_body_too_long(self): - """If first LLM call produces body > limit, agent retries with condensation prompt.""" + """If first LLM call produces body over the soft target, agent retries with condensation prompt.""" client = self._mock_client() - agent = ExperienceDistillAgent(client=client) + agent = WorkflowDistillAgent(client=client) - long_body = "x" * 3000 + long_body = "x" * (WORKFLOW_SOFT_LIMIT + 1000) long_response = ( "---\ntitle: Long\ntags: []\ncreated: 2026-01-01\n" "updated: 2026-01-01\nsource: distill\nsource_context: t1\n---\n\n" @@ -283,18 +289,18 @@ def fake_call_llm(messages): return short_response with patch.object(agent, "_call_llm", side_effect=fake_call_llm): - result = agent.run(SAMPLE_EXPERIENCE_CONTEXT) + result = agent.run(SAMPLE_WORKFLOW_CONTEXT) assert call_count == 2 _, body = parse_front_matter(result) - assert len(body.strip()) <= 2000 + assert len(body.strip()) <= WORKFLOW_SOFT_LIMIT def test_retry_asks_for_slack_under_limit(self): - """The retry prompt asks the model for less than the hard limit.""" + """The retry prompt asks the model for less than the soft target.""" client = self._mock_client() - agent = ExperienceDistillAgent(client=client) + agent = WorkflowDistillAgent(client=client) - long_body = "x" * 3000 + long_body = "x" * (WORKFLOW_SOFT_LIMIT + 1000) long_response = ( "---\ntitle: L\ntags: []\ncreated: 2026-01-01\n" "updated: 2026-01-01\nsource: distill\nsource_context: t1\n---\n\n" @@ -309,21 +315,21 @@ def fake_call_llm(messages): return long_response if len(captured) == 1 else MOCK_CONTEXT_RESPONSE with patch.object(agent, "_call_llm", side_effect=fake_call_llm): - agent.run(SAMPLE_EXPERIENCE_CONTEXT) + agent.run(SAMPLE_WORKFLOW_CONTEXT) assert len(captured) == 2 retry_prompt = captured[1][-1]["content"] - # Must mention the slacked target (limit minus margin), not the raw limit. - expected_target = 2000 - agent.RETRY_MARGIN - assert f"within {expected_target} characters" in retry_prompt + # Must mention the slacked target (soft limit minus margin). + expected_target = WORKFLOW_SOFT_LIMIT - agent.RETRY_MARGIN + assert f"around {expected_target} characters" in retry_prompt def test_hard_trims_when_retry_still_over_limit(self): - """If the retry still overshoots, body is hard-trimmed to fit the limit.""" + """If the retry still blows past the hard ceiling, body is hard-trimmed to fit it.""" client = self._mock_client() - agent = ExperienceDistillAgent(client=client) + agent = WorkflowDistillAgent(client=client) - first_body = "x" * 3000 - retry_body = "y" * 2014 # mimics the real-world failure: 14 over + first_body = "x" * (WORKFLOW_SOFT_LIMIT + 1000) + retry_body = "y" * (WORKFLOW_HARD_MAX + 14) # mimics retry still over the ceiling front_matter = ( "---\ntitle: T\ntags: []\ncreated: 2026-01-01\n" "updated: 2026-01-01\nsource: distill\nsource_context: t1\n---\n\n" @@ -339,13 +345,13 @@ def fake_call_llm(messages): return resp with patch.object(agent, "_call_llm", side_effect=fake_call_llm): - result = agent.run(SAMPLE_EXPERIENCE_CONTEXT) + result = agent.run(SAMPLE_WORKFLOW_CONTEXT) # Both LLM calls happened. assert call_count == 2 - # Final body fits the hard limit (no save failure). + # Final body fits the hard ceiling (no save failure). _, body = parse_front_matter(result) - assert len(body.strip()) <= 2000 + assert len(body.strip()) <= WORKFLOW_HARD_MAX # Truncation marker is present so the user can see it was trimmed. assert "truncated" in body # Front matter preserved. @@ -355,7 +361,7 @@ def fake_call_llm(messages): def test_no_retry_when_body_within_limit(self): """If first LLM call is within limit, no retry happens.""" client = self._mock_client() - agent = ExperienceDistillAgent(client=client) + agent = WorkflowDistillAgent(client=client) call_count = 0 @@ -365,14 +371,14 @@ def fake_call_llm(messages): return MOCK_CONTEXT_RESPONSE with patch.object(agent, "_call_llm", side_effect=fake_call_llm): - agent.run(SAMPLE_EXPERIENCE_CONTEXT) + agent.run(SAMPLE_WORKFLOW_CONTEXT) assert call_count == 1 def test_language_instruction_injected_into_system_prompt(self): client = self._mock_client() zh_instruction = "[LANGUAGE INSTRUCTION]\nWrite in Simplified Chinese." - agent = ExperienceDistillAgent(client=client, language_instruction=zh_instruction) + agent = WorkflowDistillAgent(client=client, language_instruction=zh_instruction) captured_messages = [] @@ -381,7 +387,7 @@ def fake_call_llm(messages): return MOCK_CONTEXT_RESPONSE with patch.object(agent, "_call_llm", side_effect=fake_call_llm): - agent.run(SAMPLE_EXPERIENCE_CONTEXT) + agent.run(SAMPLE_WORKFLOW_CONTEXT) system_content = captured_messages[0]["content"] assert "[LANGUAGE INSTRUCTION]" in system_content @@ -389,7 +395,7 @@ def fake_call_llm(messages): def test_language_code_zh_injects_chinese_instruction(self): client = self._mock_client() - agent = ExperienceDistillAgent(client=client, language_code="zh") + agent = WorkflowDistillAgent(client=client, language_code="zh") captured_messages = [] @@ -398,7 +404,7 @@ def fake_call_llm(messages): return MOCK_CONTEXT_RESPONSE with patch.object(agent, "_call_llm", side_effect=fake_call_llm): - agent.run(SAMPLE_EXPERIENCE_CONTEXT) + agent.run(SAMPLE_WORKFLOW_CONTEXT) system_content = captured_messages[0]["content"] assert "Simplified Chinese" in system_content @@ -406,7 +412,7 @@ def fake_call_llm(messages): def test_language_code_en_no_extra_instruction(self): client = self._mock_client() - agent = ExperienceDistillAgent(client=client, language_code="en") + agent = WorkflowDistillAgent(client=client, language_code="en") captured_messages = [] @@ -415,27 +421,45 @@ def fake_call_llm(messages): return MOCK_CONTEXT_RESPONSE with patch.object(agent, "_call_llm", side_effect=fake_call_llm): - agent.run(SAMPLE_EXPERIENCE_CONTEXT) + agent.run(SAMPLE_WORKFLOW_CONTEXT) system_content = captured_messages[0]["content"] assert "in English" in system_content assert "[LANGUAGE INSTRUCTION]" not in system_content -# ── _experience_filename ────────────────────────────────────────────────── +# ── _workflow_filename ────────────────────────────────────────────────── -class TestExperienceFilename: - def test_derives_from_workspace_name(self): - from data_formulator.routes.knowledge import _experience_filename - name = _experience_filename("Sales Analysis Pattern") +class TestWorkflowFilename: + def test_derives_from_title(self): + from data_formulator.routes.knowledge import _workflow_filename + name = _workflow_filename("Sales Analysis Pattern") assert name.endswith(".md") assert "sales-analysis-pattern" in name.lower() - def test_fallback_when_workspace_name_blank(self): - from data_formulator.routes.knowledge import _experience_filename - name = _experience_filename(" ") - assert name == "session-experience.md" + def test_fallback_when_title_blank(self): + from data_formulator.routes.knowledge import _workflow_filename + name = _workflow_filename(" ") + assert name == "session-workflow.md" + + def test_rejects_path_traversal(self): + from data_formulator.routes.knowledge import _workflow_filename + # An LLM-supplied name must never escape the workflows directory. + for evil in ("../../etc/passwd", "..\\..\\win", "/etc/shadow", "a/b/c"): + name = _workflow_filename(evil) + assert "/" not in name + assert "\\" not in name + assert ".." not in name + assert name.endswith(".md") + + def test_strips_reserved_and_control_chars(self): + from data_formulator.routes.knowledge import _workflow_filename + name = _workflow_filename('sales:report*?"<>|\x00 v1') + assert name.endswith(".md") + for ch in ':*?"<>|\x00': + assert ch not in name + assert name == "sales-report-v1.md" # ── API endpoint ────────────────────────────────────────────────────────── @@ -453,7 +477,7 @@ def app(self, tmp_path): _app.register_blueprint(knowledge_bp) register_error_handlers(_app) - (tmp_path / "knowledge" / "experiences").mkdir(parents=True) + (tmp_path / "knowledge" / "workflows").mkdir(parents=True) with patch("data_formulator.routes.knowledge.get_identity_id", return_value="test-user"), \ patch("data_formulator.routes.knowledge.get_user_home", return_value=tmp_path): @@ -464,14 +488,14 @@ def client(self, app): return app.test_client() def test_missing_context_returns_error(self, client): - resp = client.post("/api/knowledge/distill-experience", + resp = client.post("/api/knowledge/distill-workflow", json={"model": {"endpoint": "openai", "model": "gpt-4o"}}) data = resp.get_json() assert data["status"] == "error" def test_missing_model_returns_error(self, client): - resp = client.post("/api/knowledge/distill-experience", - json={"experience_context": SAMPLE_EXPERIENCE_CONTEXT}) + resp = client.post("/api/knowledge/distill-workflow", + json={"workflow_context": SAMPLE_WORKFLOW_CONTEXT}) data = resp.get_json() assert data["status"] == "error" @@ -483,9 +507,9 @@ def test_missing_events_returns_error(self, client): "workspace_name": "Demo", "threads": [], } - resp = client.post("/api/knowledge/distill-experience", + resp = client.post("/api/knowledge/distill-workflow", json={ - "experience_context": bad_context, + "workflow_context": bad_context, "model": {"endpoint": "openai", "model": "gpt-4o", "api_key": "test"}, }) data = resp.get_json() @@ -497,9 +521,9 @@ def test_missing_events_field_returns_error(self, client): "workspace_id": "ws-1", "workspace_name": "Demo", } # no 'threads' key - resp = client.post("/api/knowledge/distill-experience", + resp = client.post("/api/knowledge/distill-workflow", json={ - "experience_context": bad_context, + "workflow_context": bad_context, "model": {"endpoint": "openai", "model": "gpt-4o", "api_key": "test"}, }) data = resp.get_json() @@ -508,22 +532,22 @@ def test_missing_events_field_returns_error(self, client): def test_successful_distill(self, client, tmp_path): with patch("data_formulator.routes.agents.get_client") as mock_gc, \ patch("data_formulator.routes.agents.get_language_instruction", return_value=""), \ - patch("data_formulator.agents.agent_experience_distill.ExperienceDistillAgent.run", + patch("data_formulator.agents.agent_workflow_distill.WorkflowDistillAgent.run", return_value=MOCK_CONTEXT_RESPONSE): mock_gc.return_value = MagicMock() - resp = client.post("/api/knowledge/distill-experience", + resp = client.post("/api/knowledge/distill-workflow", json={ - "experience_context": SAMPLE_EXPERIENCE_CONTEXT, + "workflow_context": SAMPLE_WORKFLOW_CONTEXT, "model": {"endpoint": "openai", "model": "gpt-4o", "api_key": "test"}, }) data = resp.get_json() assert data["status"] == "success" - assert data["data"]["category"] == "experiences" + assert data["data"]["category"] == "workflows" assert data["data"]["path"].endswith(".md") # Verify file was written - exp_dir = tmp_path / "knowledge" / "experiences" + exp_dir = tmp_path / "knowledge" / "workflows" md_files = list(exp_dir.rglob("*.md")) assert len(md_files) >= 1 assert not (tmp_path / "agent-logs").exists() @@ -531,13 +555,13 @@ def test_successful_distill(self, client, tmp_path): def test_category_hint_creates_subdir(self, client, tmp_path): with patch("data_formulator.routes.agents.get_client") as mock_gc, \ patch("data_formulator.routes.agents.get_language_instruction", return_value=""), \ - patch("data_formulator.agents.agent_experience_distill.ExperienceDistillAgent.run", + patch("data_formulator.agents.agent_workflow_distill.WorkflowDistillAgent.run", return_value=MOCK_CONTEXT_RESPONSE): mock_gc.return_value = MagicMock() - resp = client.post("/api/knowledge/distill-experience", + resp = client.post("/api/knowledge/distill-workflow", json={ - "experience_context": SAMPLE_EXPERIENCE_CONTEXT, + "workflow_context": SAMPLE_WORKFLOW_CONTEXT, "model": {"endpoint": "openai", "model": "gpt-4o", "api_key": "test"}, "category_hint": "sales", }) diff --git a/tests/backend/knowledge/test_knowledge_store.py b/tests/backend/knowledge/test_knowledge_store.py index 2444195b..f69ce37c 100644 --- a/tests/backend/knowledge/test_knowledge_store.py +++ b/tests/backend/knowledge/test_knowledge_store.py @@ -5,11 +5,11 @@ Covers: - list_all, read, write, delete for each category -- path depth constraints (rules=flat, experiences=1 sub-dir) +- path depth constraints (rules=flat, workflows=1 sub-dir) - .md extension enforcement - ConfinedDir traversal rejection - front matter parsing and graceful degradation -- search: title, tags, filename, body matching + ranking + limit +- search: title, filename, body matching + ranking + limit - search skips alwaysApply rules (they are injected via system prompt) - tokenization: English stopwords, CJK/ASCII mixed splitting - scoring: partial token match, source discount, table_names boost @@ -73,21 +73,20 @@ def test_lists_rules(self, store, tmp_path): items = store.list_all("rules") assert len(items) == 1 assert items[0]["title"] == "ROI Calculation" - assert items[0]["tags"] == ["finance", "computation"] assert items[0]["path"] == "roi.md" assert items[0]["source"] == "manual" - def test_lists_experiences_in_subdirs(self, store, tmp_path): - exp_dir = tmp_path / "knowledge" / "experiences" / "cleaning" + def test_lists_workflows_in_subdirs(self, store, tmp_path): + exp_dir = tmp_path / "knowledge" / "workflows" / "cleaning" exp_dir.mkdir(parents=True) (exp_dir / "missing.md").write_text(SAMPLE_MD_SKILL, encoding="utf-8") - items = store.list_all("experiences") + items = store.list_all("workflows") assert len(items) == 1 assert items[0]["path"] == "cleaning/missing.md" def test_empty_category_returns_empty(self, store): - items = store.list_all("experiences") + items = store.list_all("workflows") assert items == [] def test_front_matter_title_fallback_to_stem(self, store, tmp_path): @@ -139,9 +138,9 @@ def test_preserves_existing_front_matter(self, store): content = store.read("rules", "fm.md") assert "title: ROI Calculation" in content - def test_writes_experiences_in_subdir(self, store, tmp_path): - store.write("experiences", "cleaning/handle-missing.md", SAMPLE_MD_SKILL) - assert (tmp_path / "knowledge" / "experiences" / "cleaning" / "handle-missing.md").exists() + def test_writes_workflows_in_subdir(self, store, tmp_path): + store.write("workflows", "cleaning/handle-missing.md", SAMPLE_MD_SKILL) + assert (tmp_path / "knowledge" / "workflows" / "cleaning" / "handle-missing.md").exists() # ── CRUD: delete ────────────────────────────────────────────────────────── @@ -169,12 +168,12 @@ def test_rules_subdir_rejected(self): with pytest.raises(ValueError, match="sub-directories"): KnowledgeStore.validate_path("rules", "sub/file.md") - def test_experiences_one_subdir_ok(self): - KnowledgeStore.validate_path("experiences", "cat/file.md") + def test_workflows_one_subdir_ok(self): + KnowledgeStore.validate_path("workflows", "cat/file.md") - def test_experiences_two_subdirs_rejected(self): + def test_workflows_two_subdirs_rejected(self): with pytest.raises(ValueError, match="one level"): - KnowledgeStore.validate_path("experiences", "cat/sub/file.md") + KnowledgeStore.validate_path("workflows", "cat/sub/file.md") def test_skills_rejected_as_invalid(self): with pytest.raises(ValueError, match="Invalid category"): @@ -228,7 +227,7 @@ def _setup_knowledge(self, store, tmp_path): rules_dir = tmp_path / "knowledge" / "rules" (rules_dir / "roi.md").write_text(SAMPLE_MD, encoding="utf-8") - exp_dir = tmp_path / "knowledge" / "experiences" / "cleaning" + exp_dir = tmp_path / "knowledge" / "workflows" / "cleaning" exp_dir.mkdir(parents=True) (exp_dir / "missing.md").write_text(SAMPLE_MD_SKILL, encoding="utf-8") @@ -237,11 +236,6 @@ def test_search_by_title(self, store): assert len(results) >= 1 assert results[0]["title"] == "Handle Missing Values" - def test_search_by_tags(self, store): - results = store.search("pandas") - assert len(results) >= 1 - assert results[0]["title"] == "Handle Missing Values" - def test_search_by_filename(self, store): results = store.search("missing") assert len(results) >= 1 @@ -269,7 +263,7 @@ def test_max_results_limit(self, store, tmp_path): assert len(results) <= 5 def test_search_filters_by_category(self, store): - results = store.search("ROI", categories=["experiences"]) + results = store.search("ROI", categories=["workflows"]) assert len(results) == 0 def test_search_skips_always_apply_rules(self, store, tmp_path): @@ -304,13 +298,12 @@ def test_partial_token_match_finds_results(self, store): assert results[0]["title"] == "Handle Missing Values" def test_table_names_boost(self, store, tmp_path): - """Entries tagged with a session table name get boosted.""" - exp_dir = tmp_path / "knowledge" / "experiences" / "analysis" + """Entries mentioning a session table name (title/body) get boosted.""" + exp_dir = tmp_path / "knowledge" / "workflows" / "analysis" exp_dir.mkdir(parents=True) (exp_dir / "sales-tip.md").write_text( - "---\ntitle: Sales Analysis Tips\n" - "tags: [sales_data, revenue]\nsource: manual\n---\n" - "When analysing sales, check for seasonality.\n", + "---\ntitle: Sales Analysis Tips\nsource: manual\n---\n" + "When analysing sales_data, check for seasonality.\n", encoding="utf-8", ) results = store.search("analysis tips", table_names=["sales_data"]) @@ -319,7 +312,7 @@ def test_table_names_boost(self, store, tmp_path): def test_non_manual_source_discounted(self, store, tmp_path): """Non-manual entries score lower than equivalent manual entries.""" - exp_dir = tmp_path / "knowledge" / "experiences" + exp_dir = tmp_path / "knowledge" / "workflows" (exp_dir / "auto-tip.md").write_text( "---\ntitle: Tip One\ntags: [tip]\nsource: distill\n---\nSome tip.\n", encoding="utf-8", @@ -328,7 +321,7 @@ def test_non_manual_source_discounted(self, store, tmp_path): "---\ntitle: Tip One\ntags: [tip]\nsource: manual\n---\nSome tip.\n", encoding="utf-8", ) - results = store.search("Tip One", categories=["experiences"]) + results = store.search("Tip One", categories=["workflows"]) assert len(results) == 2 assert results[0]["source"] == "manual" assert results[1]["source"] == "distill" @@ -472,7 +465,7 @@ def test_all_stopwords_returns_empty(self): class TestMatchScore: def test_single_token_title_hit(self): score = KnowledgeStore._match_score( - "ROI", "ROI Calculation", [], "roi", "", + "ROI", "ROI Calculation", "roi", "", ) assert score > 0 @@ -481,49 +474,49 @@ def test_partial_tokens_accumulate(self): score = KnowledgeStore._match_score( "quarterly sales trend", "Sales Trend Analysis", - [], "analysis", "", + "analysis", "", ) assert score > 0 def test_whole_string_bonus(self): full = KnowledgeStore._match_score( - "ROI", "ROI Calculation", [], "roi", "", + "ROI", "ROI Calculation", "roi", "", ) no_title = KnowledgeStore._match_score( - "ROI", "Something Else", [], "roi", "", + "ROI", "Something Else", "roi", "", ) assert full > no_title def test_source_discount(self): manual = KnowledgeStore._match_score( - "ROI", "ROI Guide", ["finance"], "roi", "", + "ROI", "ROI Guide", "roi", "", source="manual", ) auto = KnowledgeStore._match_score( - "ROI", "ROI Guide", ["finance"], "roi", "", + "ROI", "ROI Guide", "roi", "", source="distill", ) assert auto == pytest.approx(manual * 0.9) def test_table_names_boost(self): without = KnowledgeStore._match_score( - "analysis", "Analysis Tips", ["sales_data"], "tips", "", + "analysis", "Analysis Tips", "tips", "about sales_data", ) with_tn = KnowledgeStore._match_score( - "analysis", "Analysis Tips", ["sales_data"], "tips", "", + "analysis", "Analysis Tips", "tips", "about sales_data", table_names=["sales_data"], ) assert with_tn > without def test_no_match_returns_zero(self): score = KnowledgeStore._match_score( - "xyznonexistent", "ROI Calculation", ["finance"], "roi", "body text", + "xyznonexistent", "ROI Calculation", "roi", "body text", ) assert score == 0 def test_cjk_mixed_query_matches(self): """Chinese+English query should match via extracted ASCII tokens.""" score = KnowledgeStore._match_score( - "帮我分析ROI", "ROI Calculation", ["finance"], "roi", "", + "帮我分析ROI", "ROI Calculation", "roi", "", ) assert score > 0 diff --git a/tests/backend/routes/test_knowledge_routes.py b/tests/backend/routes/test_knowledge_routes.py index ddc2b7ab..f5ac69ff 100644 --- a/tests/backend/routes/test_knowledge_routes.py +++ b/tests/backend/routes/test_knowledge_routes.py @@ -167,7 +167,7 @@ def test_delete_nonexistent(self, client): class TestKnowledgeSearch: def test_search_returns_results(self, client, tmp_path): - exp_dir = tmp_path / "knowledge" / "experiences" / "finance" + exp_dir = tmp_path / "knowledge" / "workflows" / "finance" exp_dir.mkdir(parents=True, exist_ok=True) (exp_dir / "roi.md").write_text(SAMPLE_MD, encoding="utf-8") @@ -191,7 +191,7 @@ def test_search_invalid_category(self, client): assert data["status"] == "error" def test_search_filters_by_category(self, client, tmp_path): - exp_dir = tmp_path / "knowledge" / "experiences" / "finance" + exp_dir = tmp_path / "knowledge" / "workflows" / "finance" exp_dir.mkdir(parents=True, exist_ok=True) (exp_dir / "roi.md").write_text(SAMPLE_MD, encoding="utf-8") @@ -202,7 +202,7 @@ def test_search_filters_by_category(self, client, tmp_path): assert len(data["data"]["results"]) == 0 -SESSION_EXPERIENCE_CONTEXT = { +SESSION_WORKFLOW_CONTEXT = { "context_id": "ws-1", "workspace_id": "ws-1", "workspace_name": "Gasoline prices 2024", @@ -233,6 +233,7 @@ def test_search_filters_by_category(self, client, tmp_path): DISTILLED_MD = """\ --- subtitle: monthly sales aggregation +filename: monthly sales tags: [sales, time-series] created: 2026-05-06 updated: 2026-05-06 @@ -251,37 +252,37 @@ def test_search_filters_by_category(self, client, tmp_path): """ -class TestDistillExperience: - def test_distill_experience_from_context(self, client, tmp_path): +class TestDistillWorkflow: + def test_distill_workflow_from_context(self, client, tmp_path): with patch("data_formulator.routes.agents.get_client", return_value=object()), \ patch("data_formulator.routes.agents.get_language_instruction", return_value=""), \ patch( - "data_formulator.agents.agent_experience_distill." - "ExperienceDistillAgent.run", + "data_formulator.agents.agent_workflow_distill." + "WorkflowDistillAgent.run", return_value=DISTILLED_MD, ) as run: - resp = client.post("/api/knowledge/distill-experience", json={ - "experience_context": SESSION_EXPERIENCE_CONTEXT, + resp = client.post("/api/knowledge/distill-workflow", json={ + "workflow_context": SESSION_WORKFLOW_CONTEXT, "model": {"endpoint": "openai", "key": "x", "model": "gpt"}, }) data = resp.get_json() assert data["status"] == "success" - assert data["data"]["category"] == "experiences" - assert (tmp_path / "knowledge" / "experiences" / data["data"]["path"]).exists() + assert data["data"]["category"] == "workflows" + assert (tmp_path / "knowledge" / "workflows" / data["data"]["path"]).exists() assert not (tmp_path / "agent-logs").exists() run.assert_called_once() - def test_distill_experience_llm_timeout_returns_structured_error(self, client): + def test_distill_workflow_llm_timeout_returns_structured_error(self, client): with patch("data_formulator.routes.agents.get_client", return_value=object()), \ patch("data_formulator.routes.agents.get_language_instruction", return_value=""), \ patch( - "data_formulator.agents.agent_experience_distill." - "ExperienceDistillAgent.run", + "data_formulator.agents.agent_workflow_distill." + "WorkflowDistillAgent.run", side_effect=TimeoutError("request timed out"), ): - resp = client.post("/api/knowledge/distill-experience", json={ - "experience_context": SESSION_EXPERIENCE_CONTEXT, + resp = client.post("/api/knowledge/distill-workflow", json={ + "workflow_context": SESSION_WORKFLOW_CONTEXT, "model": {"endpoint": "openai", "key": "x", "model": "gpt"}, }) @@ -291,55 +292,60 @@ def test_distill_experience_llm_timeout_returns_structured_error(self, client): assert data["error"]["code"] == "LLM_TIMEOUT" assert data["error"]["retry"] is True - def test_distill_experience_missing_context(self, client): - resp = client.post("/api/knowledge/distill-experience", json={ + def test_distill_workflow_missing_context(self, client): + resp = client.post("/api/knowledge/distill-workflow", json={ "model": {"endpoint": "openai", "key": "x", "model": "gpt"}, }) data = resp.get_json() assert data["status"] == "error" - def test_distill_experience_missing_threads(self, client): - bad_context = {k: v for k, v in SESSION_EXPERIENCE_CONTEXT.items() if k != "threads"} - resp = client.post("/api/knowledge/distill-experience", json={ - "experience_context": bad_context, + def test_distill_workflow_missing_threads(self, client): + bad_context = {k: v for k, v in SESSION_WORKFLOW_CONTEXT.items() if k != "threads"} + resp = client.post("/api/knowledge/distill-workflow", json={ + "workflow_context": bad_context, "model": {"endpoint": "openai", "key": "x", "model": "gpt"}, }) data = resp.get_json() assert data["status"] == "error" - def test_distill_experience_missing_workspace(self, client): - bad_context = {k: v for k, v in SESSION_EXPERIENCE_CONTEXT.items() + def test_distill_workflow_missing_workspace(self, client): + bad_context = {k: v for k, v in SESSION_WORKFLOW_CONTEXT.items() if k not in ("workspace_id", "workspace_name")} - resp = client.post("/api/knowledge/distill-experience", json={ - "experience_context": bad_context, + resp = client.post("/api/knowledge/distill-workflow", json={ + "workflow_context": bad_context, "model": {"endpoint": "openai", "key": "x", "model": "gpt"}, }) data = resp.get_json() assert data["status"] == "error" - def test_distill_session_overrides_title_with_workspace_name(self, client, tmp_path): - """Session-scoped distillation composes 'Experience from : '.""" + def test_distill_session_uses_descriptive_title(self, client, tmp_path): + """Session-scoped distillation uses the agent subtitle as the title.""" with patch("data_formulator.routes.agents.get_client", return_value=object()), \ patch("data_formulator.routes.agents.get_language_instruction", return_value=""), \ patch( - "data_formulator.agents.agent_experience_distill." - "ExperienceDistillAgent.run", + "data_formulator.agents.agent_workflow_distill." + "WorkflowDistillAgent.run", return_value=DISTILLED_MD, ): - resp = client.post("/api/knowledge/distill-experience", json={ - "experience_context": SESSION_EXPERIENCE_CONTEXT, + resp = client.post("/api/knowledge/distill-workflow", json={ + "workflow_context": SESSION_WORKFLOW_CONTEXT, "model": {"endpoint": "openai", "key": "x", "model": "gpt"}, }) data = resp.get_json() assert data["status"] == "success" path = data["data"]["path"] - # Filename is derived from the workspace name, not the LLM subtitle. - assert path == "gasoline-prices-2024.md" - saved = (tmp_path / "knowledge" / "experiences" / path).read_text(encoding="utf-8") - assert "title: 'Experience from Gasoline prices 2024: monthly sales aggregation'" in saved \ - or "title: \"Experience from Gasoline prices 2024: monthly sales aggregation\"" in saved \ - or "title: Experience from Gasoline prices 2024: monthly sales aggregation" in saved + # Filename is derived from the short agent-emitted `filename` hint, + # not the long descriptive title. + assert path == "monthly-sales.md" + saved = (tmp_path / "knowledge" / "workflows" / path).read_text(encoding="utf-8") + assert "title: monthly sales aggregation" in saved \ + or "title: 'monthly sales aggregation'" in saved \ + or "title: \"monthly sales aggregation\"" in saved + # No legacy "Workflow from :" prefix on the title. + assert "Workflow from" not in saved + # The filename hint is consumed, not persisted in the front matter. + assert "filename:" not in saved # Workspace stamps are present so the file can be looked up later. assert "source_workspace_id: ws-1" in saved assert "source_workspace_name: Gasoline prices 2024" in saved @@ -347,42 +353,46 @@ def test_distill_session_overrides_title_with_workspace_name(self, client, tmp_p assert "## Method" in saved def test_distill_session_upserts_existing_workspace_file(self, client, tmp_path): - """Re-distilling the same workspace overwrites the same file.""" + """Re-distilling the same workspace replaces the prior file.""" + second_md = DISTILLED_MD.replace( + "filename: monthly sales", + "filename: annual revenue", + ) with patch("data_formulator.routes.agents.get_client", return_value=object()), \ patch("data_formulator.routes.agents.get_language_instruction", return_value=""), \ patch( - "data_formulator.agents.agent_experience_distill." - "ExperienceDistillAgent.run", - return_value=DISTILLED_MD, + "data_formulator.agents.agent_workflow_distill." + "WorkflowDistillAgent.run", + side_effect=[DISTILLED_MD, second_md], ): - client.post("/api/knowledge/distill-experience", json={ - "experience_context": SESSION_EXPERIENCE_CONTEXT, + client.post("/api/knowledge/distill-workflow", json={ + "workflow_context": SESSION_WORKFLOW_CONTEXT, "model": {"endpoint": "openai", "key": "x", "model": "gpt"}, }) - # Re-distill: workspace renamed, so the slug changes — old file - # should be removed in favour of the new one. - renamed = {**SESSION_EXPERIENCE_CONTEXT, "workspace_name": "Diesel 2024"} - resp = client.post("/api/knowledge/distill-experience", json={ - "experience_context": renamed, + # Re-distill: the filename hint changes, so the slug changes — old + # file should be removed in favour of the new one (matched by + # source_workspace_id). + resp = client.post("/api/knowledge/distill-workflow", json={ + "workflow_context": SESSION_WORKFLOW_CONTEXT, "model": {"endpoint": "openai", "key": "x", "model": "gpt"}, }) data = resp.get_json() assert data["status"] == "success" new_path = data["data"]["path"] - exp_dir = tmp_path / "knowledge" / "experiences" + exp_dir = tmp_path / "knowledge" / "workflows" # Stale slug deleted, new slug present. - assert not (exp_dir / "gasoline-prices-2024.md").exists() + assert not (exp_dir / "monthly-sales.md").exists() assert (exp_dir / new_path).exists() - assert new_path == "diesel-2024.md" + assert new_path == "annual-revenue.md" - def test_distill_session_skips_subtitle_double_prefix(self, client, tmp_path): - """Update-mode runs that re-emit a prefixed title don't double-prefix.""" - # Simulate a prior run where the LLM echoed an Experience-prefixed title + def test_distill_session_strips_legacy_title_prefix(self, client, tmp_path): + """Update-mode runs strip any legacy 'Workflow from :' prefix.""" + # Simulate a prior run where the LLM echoed a Workflow-prefixed title # without a subtitle. prior_md = ( "---\n" - "title: 'Experience from Gasoline prices 2024: prior insight'\n" + "title: 'Workflow from Gasoline prices 2024: prior insight'\n" "tags: [a]\n" "created: 2026-05-06\n" "updated: 2026-05-06\n" @@ -392,17 +402,18 @@ def test_distill_session_skips_subtitle_double_prefix(self, client, tmp_path): with patch("data_formulator.routes.agents.get_client", return_value=object()), \ patch("data_formulator.routes.agents.get_language_instruction", return_value=""), \ patch( - "data_formulator.agents.agent_experience_distill." - "ExperienceDistillAgent.run", + "data_formulator.agents.agent_workflow_distill." + "WorkflowDistillAgent.run", return_value=prior_md, ): - resp = client.post("/api/knowledge/distill-experience", json={ - "experience_context": SESSION_EXPERIENCE_CONTEXT, + resp = client.post("/api/knowledge/distill-workflow", json={ + "workflow_context": SESSION_WORKFLOW_CONTEXT, "model": {"endpoint": "openai", "key": "x", "model": "gpt"}, }) data = resp.get_json() assert data["status"] == "success" - saved = (tmp_path / "knowledge" / "experiences" / data["data"]["path"]).read_text(encoding="utf-8") - # The "Experience from ..." prefix is stripped before re-prefixing. - assert saved.count("Experience from") == 1 + saved = (tmp_path / "knowledge" / "workflows" / data["data"]["path"]).read_text(encoding="utf-8") + # The legacy "Workflow from ..." prefix is fully stripped. + assert "Workflow from" not in saved + assert "prior insight" in saved From 1571113ac7b74ce1a0c32b434a9576d6fdce2b40 Mon Sep 17 00:00:00 2001 From: y-agent-ai Date: Sat, 30 May 2026 18:22:30 +0800 Subject: [PATCH 07/40] refactor(loading): Refactor AnvilLoader and add custom parameter support 1. Add custom property support for height , label , and sx to AnvilLoader 2. Replace globally hardcoded loading text with customizable label parameter 3. Optimize loading overlay styles with new frosted glass background effect 4. Unify loading state display in App.tsx and VisualizationView --- src/app/App.tsx | 2 +- src/components/AnvilLoader.tsx | 46 +++++++++++++++++++++------------ src/views/VisualizationView.tsx | 10 ++++--- 3 files changed, 36 insertions(+), 22 deletions(-) diff --git a/src/app/App.tsx b/src/app/App.tsx index 17898f0c..22d76510 100644 --- a/src/app/App.tsx +++ b/src/app/App.tsx @@ -1253,7 +1253,7 @@ export const AppFC: FC = function AppFC(appProps) { {configLoaded && authChecked ? ( ) : ( - + )} {migrationBrowserId && ( ; +} + +export function AnvilLoader({ height = '100vh', label, sx }: AnvilLoaderProps) { return ( - - loading data formulator... - + {label !== undefined && ( + + {label} + + )} ); } diff --git a/src/views/VisualizationView.tsx b/src/views/VisualizationView.tsx index 7b6d18b4..585eba79 100644 --- a/src/views/VisualizationView.tsx +++ b/src/views/VisualizationView.tsx @@ -15,7 +15,6 @@ import { ListItemIcon, ListItemText, MenuItem, - LinearProgress, Card, ListSubheader, Menu, @@ -37,6 +36,7 @@ import _ from 'lodash'; import { borderColor, transition } from '../app/tokens'; import { WritingIndicator } from '../components/FunComponents'; +import { AnvilLoader } from '../components/AnvilLoader'; import ButtonGroup from '@mui/material/ButtonGroup'; @@ -1099,10 +1099,12 @@ export const ChartEditorFC: FC<{}> = function ChartEditorFC({}) { return {synthesisRunning ? - + : ''} {chartUnavailable ? "" : chartResizer} {content} From b9abafb163d59c8c4165075d8f573345b4d70235 Mon Sep 17 00:00:00 2001 From: cat0825 <1759138827@qq.com> Date: Sun, 31 May 2026 14:02:02 +0800 Subject: [PATCH 08/40] test: keep zh locale keys aligned --- src/i18n/locales/zh/dataLoading.json | 1 + tests/frontend/unit/app/i18nLocales.test.ts | 30 +++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 tests/frontend/unit/app/i18nLocales.test.ts diff --git a/src/i18n/locales/zh/dataLoading.json b/src/i18n/locales/zh/dataLoading.json index 4eab6fcf..6ffe595d 100644 --- a/src/i18n/locales/zh/dataLoading.json +++ b/src/i18n/locales/zh/dataLoading.json @@ -39,6 +39,7 @@ "rowLimit": "行数限制", "loadSelected": "加载选中的表", "loadedCount": "✓ 已加载 {{count}} 张表", + "loadedCount_plural": "✓ 已加载 {{count}} 张表", "preview": "预览", "hidePreview": "收起", "previewing": "正在预览...", diff --git a/tests/frontend/unit/app/i18nLocales.test.ts b/tests/frontend/unit/app/i18nLocales.test.ts new file mode 100644 index 00000000..dd6c9933 --- /dev/null +++ b/tests/frontend/unit/app/i18nLocales.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from "vitest"; + +import en from "../../../../src/i18n/locales/en"; +import zh from "../../../../src/i18n/locales/zh"; + +type TranslationValue = string | Record; +type TranslationMap = Record; + +function collectKeys(value: TranslationMap, prefix = ""): Set { + const keys = new Set(); + + for (const [key, child] of Object.entries(value)) { + const nextPrefix = prefix ? `${prefix}.${key}` : key; + if (typeof child === "string") { + keys.add(nextPrefix); + } else { + for (const childKey of collectKeys(child, nextPrefix)) { + keys.add(childKey); + } + } + } + + return keys; +} + +describe("i18n locale bundles", () => { + it("keeps Simplified Chinese translation keys aligned with English", () => { + expect(collectKeys(zh)).toEqual(collectKeys(en)); + }); +}); From 748a30ce45f8e01b95388aced0b2e27106d70b69 Mon Sep 17 00:00:00 2001 From: Chenglong Wang Date: Sun, 31 May 2026 12:26:36 -0700 Subject: [PATCH 09/40] bug fix and clean up --- .../agents/agent_workflow_distill.py | 190 ++++++++++-------- .../data_loader/sample_datasets_loader.py | 13 +- src/app/dfSlice.tsx | 10 + src/components/LoadPlanCard.tsx | 51 ++++- src/i18n/locales/en/common.json | 4 +- src/i18n/locales/en/dataLoading.json | 2 + src/i18n/locales/zh/dataLoading.json | 2 + src/views/DataLoadingChat.tsx | 24 ++- src/views/DataSourceSidebar.tsx | 28 ++- src/views/EncodingShelfCard.tsx | 60 ++++-- src/views/KnowledgePanel.tsx | 6 +- src/views/VisualizationView.tsx | 15 +- 12 files changed, 263 insertions(+), 142 deletions(-) diff --git a/py-src/data_formulator/agents/agent_workflow_distill.py b/py-src/data_formulator/agents/agent_workflow_distill.py index 3f3d9c6d..0d86aa78 100644 --- a/py-src/data_formulator/agents/agent_workflow_distill.py +++ b/py-src/data_formulator/agents/agent_workflow_distill.py @@ -30,9 +30,28 @@ SYSTEM_PROMPT = """\ You are a workflow distiller. Given the chronological events of a data -analysis session plus an optional user instruction, extract a short, -**replayable workflow** that captures *what the user wanted and got* — so -the same analysis can be reproduced later on a similarly-shaped dataset. +analysis session plus an optional user distillation instruction, extract a **replayable +workflow** that captures *what the user wanted and got* — and write it at +TWO levels so it can be reused in two different situations: + +1. An **Abstract workflow** — dataset-independent. The underlying analytical + pattern, stripped of this dataset's subject matter: the sequence of + questions, computations, and chart kinds, phrased in domain-neutral terms. + Following it on a *different and possibly very differently-shaped* dataset + should walk the same process and arrive at structurally similar + visualizations. +2. A **Concrete workflow** — for *similar* data (same shape, only minor + differences — a different period, region, or filter). It names the real + fields, aggregations, filters, and chart encodings used here, so the + analysis can be replayed closely with minimal thought. + +Both describe the SAME analysis at different distances. They should be +consistent, but they do NOT need an exact 1:1 step mapping — let each be as +long as it needs (typically 3-7 steps each). + +Where the analysis hinges on a few choices a user might change on replay (a +period, a filter, a top-N), surface them as named **parameters** with +`{{token}}` placeholders in the steps — see the `## Parameters` section below. The session contains one or more threads (separate analysis branches in the same session) each rendered under a `### Thread N` header. When @@ -47,38 +66,20 @@ (followed by columns, row count, sample, and code). - `create_chart` — a chart emitted on a table (mark + encoding summary). -Your job is to recover the **ordered list of requests** the user actually -wanted, and the outputs (tables/charts) they ended up keeping. Beyond the -concrete steps, also distill the analysis at TWO levels of abstraction so -it can be reused later: -- **Adapting to similar data** (concrete) — how to rerun essentially the - same analysis on a near-identical dataset, e.g. the business report for - a different month, region, or product line. Same shape and intent, only - the specific inputs/filters change. -- **Generalizing to other data** (abstract, dataset-agnostic) — the - underlying analytical pattern, independent of this domain: the kinds of - questions, computations, and charts involved, phrased so they transfer - to a different domain or a differently-shaped dataset. - CRITICAL extraction rules — keep only what the user wanted and got: -- Each step = one user request, written in plain language. Say BOTH the - question being explored AND what was produced to answer it — including - the chart that was created and the key fields it uses (e.g. "Ask how - sales trend over time, and plot monthly total sales as a line chart"; - "Compare regions by breaking revenue down per region as a sorted bar - chart"). Order them as the analysis progressed. +- Recover the ORDERED list of requests the user actually wanted, and the + outputs (tables/charts) they kept. Each step states BOTH the question + explored AND what was produced to answer it — including the chart and the + key fields it uses. - DROP corrective back-and-forth. If the user changed their mind ("no, it should be…", "actually use median instead"), keep ONLY the final resolved intent — not the wrong first attempt or the correction. - DROP abandoned work. If a chart or table was created and then deleted or never kept, leave it out entirely. - DROP mechanics. Do NOT include error-repair loops, dtype fixes, tool - call noise, or low-level code. Describe intent, not implementation. -- Do NOT lean on code or exact column names unless a name is essential to - the request's meaning. Keep steps dataset-agnostic where possible so - they replay on a new slice of similar data. -- Capture genuine gotchas separately as short notes (advisory warnings to - carry forward), NOT as steps to re-perform. + call noise, or low-level code dumps. Describe intent, not implementation. +- Capture genuine gotchas as short Notes (advisory warnings to carry + forward), NOT as steps to re-perform. If a user instruction is provided, let it steer what to keep or emphasise. @@ -86,8 +87,8 @@ ``` --- -subtitle: -filename: +subtitle: +filename: created: updated: source: distill @@ -96,74 +97,85 @@ ## Goal - -## Steps -1. -2. +what it produces. This is where the dataset-grounded explanation belongs — +you MAY name the real subject here (e.g. "originally distilled from a +monthly gasoline-price session").> + +## Parameters + +- `{{period}}` — the time range analysed; used here: 2024; on replay: ask. +- `{{top_n}}` — how many top categories to keep; used here: 10; on replay: keep. +- `{{region}}` — geographic filter applied; used here: National; on replay: ask. + +## Abstract workflow + +1. +2. 3. <…> -## Adapting to similar data - - -## Generalizing to other data - +## Concrete workflow + +1. +2. <…> ## Notes +analysis on new data — e.g. "sort by time before computing period-over-period +change". Omit this section entirely if there is nothing worth warning about.> ``` Rules: -- Subtitle must DESCRIBE what the workflow is about in PLAIN LANGUAGE that - a non-expert can fully understand at a glance, so they can decide - whether to replay it on new data. Favor clarity over brevity: it can be - a full sentence (up to ~25 words) if that makes the analysis genuinely - understandable. Write it like you would explain the analysis to a - colleague in one breath, covering the subject and the main thing you do - with it. The hosting application uses this subtitle directly as the - workflow's display title, so make it self-contained and do NOT prefix it - with the session name. - - Start with a concrete action verb (Plot, Compare, Break down, Rank, - Track, Summarize, Find…). - - Name the real-world subject in everyday words (sales, revenue, - customers, events), NOT the internal mechanics or derived-column - names you happened to create. - - AVOID abstract or technical jargon and invented noun-phrases - ("deltas", "composition", "window", "distribution shift"). If a - technique matters, phrase it plainly ("change from one period to the - next" instead of "deltas"). - Good: "Plot monthly sales over time and compare each year against the - previous one to spot volatile periods". - "Break revenue down by region and show how each region - contributes to the total as a stacked area chart". - "Track how many events happen in each time window and what kinds - of events make up each window". - Bad: "Time series analysis". "Data workflow". "Chart exploration". - "Event window deltas with composition". "Distribution shift inspection". +- The subtitle is the workflow's display TITLE. Make it ABSTRACT and + library-friendly: name the *kind of analysis* — a technique plus a GENERIC + subject (KPI, metric, category, event, cohort) — so someone browsing the + workflow library can tell whether this is the KIND of analysis they want to + reuse. Do NOT pin it to this dataset's specific subject, period, or column + names, and do NOT prefix it with the session name. + - Pair a real technique with a generic subject; avoid bare category words. + Good: "Year-over-year KPI volatility analysis". + "Category contribution-to-total breakdown". + "Time-windowed event composition analysis". + Bad: "Plot monthly gasoline prices in 2024 and compare each year". (too specific) + "Time series analysis". "Data workflow". "Chart exploration". (too vague) + The dataset-grounded, full-sentence explanation goes in `## Goal`, NOT the title. - Filename must be a SHORT (2-5 word) lowercase name for the file — just - the core subject and action, e.g. "monthly sales trend", "region revenue - breakdown". No dates, no file extension, no session name. It is only - used to name the file on disk; the descriptive subtitle is what users see. -- Steps must be ordered and reproducible. Each step should make clear the - question being explored and the chart/output produced to answer it. -- "Adapting to similar data" stays close to this analysis (same domain, - same shape) — only the concrete inputs change. "Generalizing to other - data" must be domain-neutral: strip out this dataset's subject matter and - describe only the transferable analytical pattern (question types, - computations, chart kinds). Do NOT just repeat the steps in either - section; add genuine reuse guidance. Keep each section brief. -- Be as long as the analysis needs — do not omit meaningful steps, - questions, or charts just to stay short. Stay focused, but completeness - matters more than brevity. + the technique/subject, e.g. "kpi volatility analysis", "region revenue + breakdown". No dates, no file extension, no session name. It only names the + file on disk; the subtitle is what users see. +- Abstract workflow must be domain-neutral — strip this dataset's subject + matter and column names; describe only the transferable pattern (question + types, computations, chart kinds). Concrete workflow must be runnable on a + near-identical dataset: real field names, the aggregation, the filter to + vary, the chart mark + key encodings. Do NOT have the two sections merely + repeat each other — each adds its own grain of reuse guidance. +- Parameters are optional and a judgment call: surface only the FEW knobs + that materially change the outcome and that a user would revisit on replay + (often 0-4). When in doubt, leave the value inline — a spurious `{{token}}` + is worse than none. Knobs may be run-specific (period, region, top-N — + usually `ask`) or dataset-specific (a domain value/column — usually `keep`, + and may be skipped in the Abstract workflow). Every `{{token}}` in the steps + must be listed in `## Parameters` and vice versa. +- Steps in both sections must be ordered and reproducible. +- Be as long as the analysis needs — do not omit meaningful steps, questions, + or charts just to stay short. Stay focused, but completeness matters more + than brevity. - No raw data, PII, secrets, or specific values unless essential to a request. - Write the subtitle, headings, and body in {output_language}. YAML front-matter keys stay in English. diff --git a/py-src/data_formulator/data_loader/sample_datasets_loader.py b/py-src/data_formulator/data_loader/sample_datasets_loader.py index 6c3267cf..a84b8302 100644 --- a/py-src/data_formulator/data_loader/sample_datasets_loader.py +++ b/py-src/data_formulator/data_loader/sample_datasets_loader.py @@ -25,7 +25,10 @@ import pyarrow as pa from data_formulator.data_loader.external_data_loader import ExternalDataLoader -from data_formulator.datalake.parquet_utils import df_to_safe_records +from data_formulator.datalake.parquet_utils import ( + df_to_safe_records, + sanitize_dataframe_for_arrow, +) logger = logging.getLogger(__name__) @@ -231,7 +234,13 @@ def fetch_data_as_arrow( logger.info("Returning %d / %d rows from sample dataset: %s", len(df), self._last_total_rows, source_table) - return pa.Table.from_pandas(df, preserve_index=False) + # Public sample JSON/CSV files frequently contain mixed-type object + # columns (e.g. movies.json's ``Title`` holds both strings and + # numeric values), which makes ``pa.Table.from_pandas`` raise + # ArrowTypeError. Coerce such columns to a consistent type first. + return pa.Table.from_pandas( + sanitize_dataframe_for_arrow(df), preserve_index=False + ) # ------------------------------------------------------------------ # Internal: cached full-dataset fetch diff --git a/src/app/dfSlice.tsx b/src/app/dfSlice.tsx index 89b075ab..2ceb55aa 100644 --- a/src/app/dfSlice.tsx +++ b/src/app/dfSlice.tsx @@ -245,6 +245,9 @@ export interface DataFormulatorState { /** Whether the data source sidebar is expanded (true) or collapsed to rail (false) */ dataSourceSidebarOpen: boolean; + /** Which data source sidebar tab is active. Persisted so it survives session refresh. */ + dataSourceSidebarTab: 'sources' | 'sessions' | 'knowledge'; + /** * One-shot signal asking the sidebar to focus a specific connector * (open the sidebar, switch to sources tab, expand + scroll-into-view @@ -322,6 +325,8 @@ const initialState: DataFormulatorState = { dataSourceSidebarOpen: false, + dataSourceSidebarTab: 'sources', + focusedConnectorId: undefined, } @@ -762,12 +767,16 @@ export const dataFormulatorSlice = createSlice({ viewMode: state.viewMode, dataLoaderConnectParams: state.dataLoaderConnectParams, dataSourceSidebarOpen: state.dataSourceSidebarOpen, + dataSourceSidebarTab: state.dataSourceSidebarTab, activeWorkspace: action.payload, }; }, setDataSourceSidebarOpen: (state, action: PayloadAction) => { state.dataSourceSidebarOpen = action.payload; }, + setDataSourceSidebarTab: (state, action: PayloadAction<'sources' | 'sessions' | 'knowledge'>) => { + state.dataSourceSidebarTab = action.payload; + }, /** * Ask the data-source sidebar to focus a specific connector. * Opens the sidebar (if collapsed) and stores the target id; the @@ -870,6 +879,7 @@ export const dataFormulatorSlice = createSlice({ activeWorkspace: saved.activeWorkspace ?? state.activeWorkspace ?? null, dataSourceSidebarOpen: state.dataSourceSidebarOpen, + dataSourceSidebarTab: state.dataSourceSidebarTab, // Reset display-rows tick so dependent components re-fetch. displayRowsTick: 0, diff --git a/src/components/LoadPlanCard.tsx b/src/components/LoadPlanCard.tsx index 9b60effd..b91e54ab 100644 --- a/src/components/LoadPlanCard.tsx +++ b/src/components/LoadPlanCard.tsx @@ -16,8 +16,14 @@ import type { LoadPlan, LoadPlanCandidate, PendingTableLoad } from './ComponentT interface LoadPlanCardProps { plan: LoadPlan; - onConfirm: (selected: LoadPlanCandidate[]) => void; + onConfirm: (selected: LoadPlanCandidate[], opts?: { newWorkspace?: boolean }) => void; confirmed?: boolean; + /** When true, a workspace with existing data is already open, so the + * destination of the load is ambiguous. We then offer two explicit + * actions: add to the current workspace, or load into a fresh one. + * When false (empty/new workspace), a single "Load selected" button + * loads directly with no ambiguity. */ + canLoadInNewWorkspace?: boolean; } // Plans this small auto-expand each row's preview on first render so the @@ -48,7 +54,7 @@ const formatFilterValue = (value: any) => { return Array.isArray(value) ? value.join(', ') : String(value); }; -export const LoadPlanCard: React.FC = ({ plan, onConfirm, confirmed }) => { +export const LoadPlanCard: React.FC = ({ plan, onConfirm, confirmed, canLoadInNewWorkspace }) => { const theme = useTheme(); const { t } = useTranslation(); const [selection, setSelection] = useState>( @@ -143,12 +149,12 @@ export const LoadPlanCard: React.FC = ({ plan, onConfirm, con fetchPreview(candidate, idx); }; - const handleConfirm = async () => { + const handleConfirm = async (newWorkspace = false) => { const selected = plan.candidates.filter((c, i) => selection[i] && !c.resolutionError); if (selected.length === 0) return; setLoading(true); try { - await onConfirm(selected); + await onConfirm(selected, { newWorkspace }); } finally { setLoading(false); } @@ -257,12 +263,47 @@ export const LoadPlanCard: React.FC = ({ plan, onConfirm, con defaultValue: '✓ Loaded', })} + ) : canLoadInNewWorkspace ? ( + // A workspace with data is already open — make the load + // destination explicit rather than silently appending. + <> + + + ) : ( - {activeWorkspace && ( - - - - )} + + + {activeWorkspace && ( + + )} )} diff --git a/src/views/DataView.tsx b/src/views/DataView.tsx index f6c4a79f..98217775 100644 --- a/src/views/DataView.tsx +++ b/src/views/DataView.tsx @@ -6,11 +6,13 @@ import ReactDOM from 'react-dom'; import _ from 'lodash'; -import { Typography, Box, Link, Breadcrumbs, useTheme, Fade, IconButton, Tooltip } from '@mui/material'; +import { Typography, Box, Link, Breadcrumbs, useTheme, Fade, IconButton, Tooltip, TextField, InputAdornment, Chip } from '@mui/material'; import { alpha } from '@mui/material/styles'; import { useTranslation } from 'react-i18next'; import OpenInFullIcon from '@mui/icons-material/OpenInFull'; import CloseFullscreenIcon from '@mui/icons-material/CloseFullscreen'; +import SearchIcon from '@mui/icons-material/Search'; +import ClearIcon from '@mui/icons-material/Clear'; import '../scss/DataView.scss'; @@ -27,12 +29,33 @@ export interface FreeDataViewProps { // full-canvas overlay. Used wherever the grid is shown inline (under a // chart, or as the focused-table preview). maximizable?: boolean; + // Explicit table to render. When omitted the view derives the table from + // `focusedId` (single-focus behavior). Set by the multi-table canvas to + // render each highlighted table in a stack. + tableId?: string; + // Render the Numbers-style title/metadata + search header inside the grid. + // Used by the focused-table canvas. + showHeaderBar?: boolean; + // Hide the in-grid footer widget; the focused-table canvas surfaces those + // actions (row count / random / download) in its bottom toolbar instead. + hideFooter?: boolean; + // Controlled random-rows trigger + virtual-state report, forwarded to the + // grid so the external toolbar can drive/observe them. + randomizeToken?: number; + onStateReport?: (s: { loadedCount: number; rowCount: number; virtual: boolean; canRandomize: boolean }) => void; } -export const FreeDataViewFC: FC = function DataView({ maximizable }) { +export const FreeDataViewFC: FC = function DataView({ maximizable, tableId, showHeaderBar, hideFooter, randomizeToken, onStateReport }) { const { t } = useTranslation(); const [maximized, setMaximized] = React.useState(false); + // Draft holds the live input; query is the committed term the grid/server + // actually searches. Search runs only when the user submits (Enter or the + // search icon), never on every keystroke. + const [searchDraft, setSearchDraft] = React.useState(''); + const [searchQuery, setSearchQuery] = React.useState(''); + const submitSearch = React.useCallback(() => setSearchQuery(searchDraft.trim()), [searchDraft]); + const clearSearch = React.useCallback(() => { setSearchDraft(''); setSearchQuery(''); }, []); const dispatch = useDispatch(); @@ -42,13 +65,20 @@ export const FreeDataViewFC: FC = function DataView({ maximiz // Derive the table to display based on focusedId const focusedTableId = useMemo(() => { + if (tableId) return tableId; if (!focusedId) return undefined; if (focusedId.type === 'table') return focusedId.tableId; if (focusedId.type !== 'chart') return undefined; const chartId = focusedId.chartId; const chart = allCharts.find(c => c.id === chartId); return chart?.tableRef; - }, [focusedId, allCharts]); + }, [focusedId, allCharts, tableId]); + + // The search term is temporary/per-table: clear it when switching tables. + React.useEffect(() => { + setSearchDraft(''); + setSearchQuery(''); + }, [focusedTableId]); // Only subscribe to the focused table and table count — NOT the full tables array. // This prevents re-rendering the entire data grid when the agent adds unrelated tables. @@ -132,14 +162,114 @@ export const FreeDataViewFC: FC = function DataView({ maximiz columnDefs={colDefs} rowCount={targetTable?.virtual?.rowCount || targetTable?.rows.length || 0} virtual={targetTable?.virtual ? true : false} + searchText={searchQuery} + hideFooter={hideFooter} + randomizeToken={randomizeToken} + onStateReport={onStateReport} /> ); + // Numbers-style header rendered OUTSIDE the table card: table title + + // row/column metadata on the left, a quick search box on the right. + // Sort & filter remain in the column headers/kebab menus. + const headerRowCount = targetTable?.virtual?.rowCount || targetTable?.rows.length || 0; + const headerColCount = targetTable?.names.length || 0; + const headerBar = showHeaderBar ? ( + + + + + {targetTable?.displayId || targetTable?.id || 'table'} + + {searchQuery ? ( + } + label={`"${searchQuery}"`} + onDelete={clearSearch} + deleteIcon={} + sx={{ + height: 22, maxWidth: 220, flexShrink: 0, + borderRadius: '6px', + backgroundColor: (theme) => alpha(theme.palette.primary.main, 0.08), + color: 'primary.main', + '& .MuiChip-label': { fontSize: 11.5, px: 0.75, overflow: 'hidden', textOverflow: 'ellipsis' }, + '& .MuiChip-icon': { color: 'primary.main', ml: 0.5 }, + '& .MuiChip-deleteIcon': { color: 'primary.main', '&:hover': { color: 'primary.dark' } }, + }} + /> + ) : null} + + + {t('dataGrid.rowCount', { count: headerRowCount })} · {headerColCount} {headerColCount === 1 ? 'column' : 'columns'} + + + + setSearchDraft(e.target.value)} + autoComplete="off" + inputProps={{ autoComplete: 'off', autoCorrect: 'off', autoCapitalize: 'off', spellCheck: false }} + onKeyDown={(e) => { + if (e.key === 'Enter') { e.preventDefault(); submitSearch(); } + else if (e.key === 'Escape' && searchDraft) { e.preventDefault(); clearSearch(); } + }} + placeholder={t('dataGrid.searchPlaceholder', { defaultValue: 'Search table…' })} + InputProps={{ + startAdornment: ( + + + + + + + + ), + endAdornment: (searchDraft || searchQuery) ? ( + + + + + + ) : undefined, + }} + sx={{ + width: 220, + '& .MuiOutlinedInput-root': { borderRadius: '8px', fontSize: 12, backgroundColor: 'background.paper' }, + '& .MuiOutlinedInput-input': { py: '6px' }, + }} + /> + + ) : null; + + // Wrap any table content with the header above it (when enabled), so the + // title sits outside the card frame. + const withHeader = (content: React.ReactNode) => showHeaderBar ? ( + + {headerBar} + {content} + + ) : content; + + // Focused-table canvas: the table already fills the page, so there's no + // maximize button — just a flat bordered card with the title above it. + if (showHeaderBar) { + return withHeader( + + {grid} + + ); + } + if (!maximizable) { - return grid; + return withHeader(grid); } const toggleButton = ( @@ -160,12 +290,16 @@ export const FreeDataViewFC: FC = function DataView({ maximiz // The toggle button sits just outside the table to the right (a slim panel), // so it never overlaps the column headers and the card keeps its original look. // In maximized mode the surrounding overlay already provides the card frame. + // When the header bar is shown (focused-table canvas) the card stays flat — + // no hover glow/elevation — since the title lives outside the card. const cardSx = maximized ? { overflow: 'hidden' } : { overflow: 'hidden', borderRadius: '8px', border: `1px solid ${borderColor.divider}`, - transition: 'box-shadow 0.2s ease', - '&:hover': { boxShadow: '0 0 8px rgba(25, 118, 210, 0.25)' }, + ...(showHeaderBar ? {} : { + transition: 'box-shadow 0.2s ease', + '&:hover': { boxShadow: '0 0 8px rgba(25, 118, 210, 0.25)' }, + }), }; const framed = ( @@ -209,5 +343,5 @@ export const FreeDataViewFC: FC = function DataView({ maximiz ); } - return framed; + return withHeader(framed); } \ No newline at end of file diff --git a/src/views/SelectableDataGrid.tsx b/src/views/SelectableDataGrid.tsx index 4869ef77..222a883e 100644 --- a/src/views/SelectableDataGrid.tsx +++ b/src/views/SelectableDataGrid.tsx @@ -58,8 +58,20 @@ interface SelectableDataGridProps { rowCount: number; virtual: boolean; columnDefs: ColumnDef[]; + // Controlled quick-search text (owned by the focused-table canvas). Filters + // the loaded rows client-side across all data columns. + searchText?: string; + // Hide the in-grid footer widget (row count / random / download). The + // focused-table canvas surfaces these actions in its bottom toolbar. + hideFooter?: boolean; + // Bumping this number triggers a "random rows" refetch (virtual tables). + randomizeToken?: number; + // Report virtual-pagination state up so an external toolbar can render the + // loaded/total count and enable the random-rows action. + onStateReport?: (s: { loadedCount: number; rowCount: number; virtual: boolean; canRandomize: boolean }) => void; } + function descendingComparator(a: T, b: T, orderBy: keyof T) { if (b[orderBy] < a[orderBy]) { return -1; @@ -322,7 +334,7 @@ const VirtuosoTableBody = React.forwardRef((props, ref) const PAGE_SIZE = 500; export const SelectableDataGrid: React.FC = React.memo(({ - tableId, rows, tableName, columnDefs, rowCount, virtual }) => { + tableId, rows, tableName, columnDefs, rowCount, virtual, searchText, hideFooter, randomizeToken, onStateReport }) => { const { t } = useTranslation(); const [orderBy, setOrderBy] = React.useState(undefined); @@ -360,6 +372,9 @@ export const SelectableDataGrid: React.FC = React.memo( const [isLoadingMore, setIsLoadingMore] = React.useState(false); const [isDownloading, setIsDownloading] = React.useState(false); const [hasMore, setHasMore] = React.useState(virtual ? rows.length < rowCount : false); + // Filtered total reported by the server (reflects active search/filters for + // virtual tables). Falls back to the unfiltered prop count. + const [serverRowCount, setServerRowCount] = React.useState(rowCount); const fetchIdRef = React.useRef(0); React.useEffect(() => { @@ -377,6 +392,29 @@ export const SelectableDataGrid: React.FC = React.memo( } }, [rows, order, orderBy]) + // Report virtual-pagination state up to an external toolbar (focused-table + // canvas) so it can render the loaded/total count and the random-rows dice. + React.useEffect(() => { + const effectiveCount = virtual ? serverRowCount : rowCount; + onStateReport?.({ + loadedCount: rowsToDisplay.length, + rowCount: effectiveCount, + virtual, + canRandomize: virtual && effectiveCount > 10000, + }); + }, [rowsToDisplay.length, rowCount, serverRowCount, virtual, onStateReport]); + + // Bumping `randomizeToken` (from the external toolbar's dice) resets sort + // and refetches a fresh random page for virtual tables. + const randomizeMountRef = React.useRef(true); + React.useEffect(() => { + if (randomizeMountRef.current) { randomizeMountRef.current = false; return; } + if (!virtual) return; + setOrderBy(undefined); + setOrder('asc'); + fetchVirtualDataRef.current?.([], 'asc'); + }, [randomizeToken]); + // Only the Table component depends on columnDefs (for colgroup); memoize it // so react-virtuoso keeps a stable reference when columns haven't changed. const columnIds = columnDefs.map(c => c.id).join(','); @@ -446,6 +484,14 @@ export const SelectableDataGrid: React.FC = React.memo( filtersRef.current = Object.values(columnFilters); }, [columnFilters]); + // Committed global search term (server-side pushdown for virtual tables). + // Held in a ref so fetchVirtualData stays stable across sort/scroll while + // still carrying the latest search into every request. + const searchRef = React.useRef(''); + React.useEffect(() => { + searchRef.current = (searchText || '').trim(); + }, [searchText]); + const fetchVirtualData = React.useCallback(( sortByColumnIds: string[], sortOrder: 'asc' | 'desc', @@ -470,6 +516,7 @@ export const SelectableDataGrid: React.FC = React.memo( : 'head', order_by_fields: sortByColumnIds.length > 0 ? sortByColumnIds : ['#rowId'], ...(activeFilters.length > 0 ? { filters: activeFilters } : {}), + ...(searchRef.current ? { search: searchRef.current } : {}), }; apiRequest(getUrls().SAMPLE_TABLE, { @@ -487,6 +534,7 @@ export const SelectableDataGrid: React.FC = React.memo( } else { setRowsToDisplay(newRows); } + setServerRowCount(totalCount); setHasMore(offset + newRows.length < totalCount); setIsLoading(false); setIsLoadingMore(false); @@ -518,6 +566,20 @@ export const SelectableDataGrid: React.FC = React.memo( // eslint-disable-next-line react-hooks/exhaustive-deps }, [columnFilters]); + // Refetch from offset 0 whenever the committed search term changes + // (virtual tables only). Non-virtual tables filter client-side below. + const didMountSearchRef = React.useRef(false); + React.useEffect(() => { + if (!virtual) return; + if (!didMountSearchRef.current) { + didMountSearchRef.current = true; + return; + } + searchRef.current = (searchText || '').trim(); + fetchVirtualData(orderBy ? [orderBy] : [], order, 0, false); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [searchText]); + const handleEndReached = React.useCallback(() => { if (!virtual || !hasMore || isLoadingMore || isLoading) return; fetchVirtualData( @@ -528,6 +590,15 @@ export const SelectableDataGrid: React.FC = React.memo( ); }, [virtual, hasMore, isLoadingMore, isLoading, fetchVirtualData, orderBy, order, rowsToDisplay.length]); + // Virtual tables are already filtered server-side (search pushdown), so we + // render their rows as-is. Non-virtual tables (fully loaded) filter + // client-side over the committed search term — case-insensitive substring. + const trimmedSearch = (searchText || '').trim().toLowerCase(); + const visibleRows = (!virtual && trimmedSearch) + ? rowsToDisplay.filter(row => + columnDefs.some(c => c.id !== '#rowId' && String(row[c.id] ?? '').toLowerCase().includes(trimmedSearch))) + : rowsToDisplay; + return ( = React.memo( = React.memo( )} - {virtual && } - {virtual && rowsToDisplay.length < rowCount - ? t('dataGrid.loadedOfTotal', { loaded: rowsToDisplay.length, total: rowCount }) - : t('dataGrid.rowCount', { count: rowCount })} + {virtual && rowsToDisplay.length < serverRowCount + ? t('dataGrid.loadedOfTotal', { loaded: rowsToDisplay.length, total: serverRowCount }) + : t('dataGrid.rowCount', { count: virtual ? serverRowCount : rowCount })} - {virtual && rowCount > 10000 && ( + {virtual && serverRowCount > 10000 && ( = React.memo( - + } {/* Column filter popover — variant chosen synchronously from metadata (design-doc 31). Server-side pushdown via fetchVirtualData. */} {filterPopover && (() => { diff --git a/src/views/SimpleChartRecBox.tsx b/src/views/SimpleChartRecBox.tsx index 97135c11..c0ceaac2 100644 --- a/src/views/SimpleChartRecBox.tsx +++ b/src/views/SimpleChartRecBox.tsx @@ -1560,32 +1560,6 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ } }, [pendingClarification, dispatch, t]); - // Landing / "no thread yet" highlight: when the user has loaded data - // but hasn't started an exploration on the focused table (no real - // charts AND the table isn't part of a derivation chain), gently pulse - // a colored ring around the input card to anchor the eye here. - // Suppressed once they start typing, while an agent is running, or - // while a clarification is pending — anything that already draws focus. - const focusedTableHasCharts = !!focusedTableId && charts.some(c => - c.tableRef === focusedTableId - && c.chartType !== '?' - && c.chartType !== 'Auto' - && c.source !== 'trigger' - ); - const focusedTableObj = focusedTableId ? tables.find(t => t.id === focusedTableId) : undefined; - const focusedTableHasDerivation = !!focusedTableObj && ( - focusedTableObj.derive !== undefined - || tables.some(t => t.derive?.trigger?.tableId === focusedTableId) - ); - const isLandingHighlight = ( - !!focusedTableId - && !focusedTableHasCharts - && !focusedTableHasDerivation - && !isChatFormulating - && !pendingClarification - && chatPrompt.trim() === '' - ); - const inputBox = ( void }> = function ({ '&:hover': { boxShadow: '0 2px 10px rgba(32, 33, 36, 0.14), 0 1px 3px rgba(32, 33, 36, 0.08)', }, - ...(isLandingHighlight ? { - animation: 'df-chatinput-landing-pulse 2.4s ease-in-out infinite', - '@keyframes df-chatinput-landing-pulse': { - '0%, 100%': { - boxShadow: `0 0 0 0 ${alpha(theme.palette.primary.main, 0.0)}, 0 4px 14px ${alpha(theme.palette.common.black, 0.06)}`, - }, - '50%': { - boxShadow: `0 0 0 6px ${alpha(theme.palette.primary.main, 0.14)}, 0 4px 18px ${alpha(theme.palette.common.black, 0.08)}`, - }, - }, - } : {}), '&:focus-within': { - animation: 'none', borderColor: theme.palette.primary.main, boxShadow: `0 0 0 2px ${alpha(theme.palette.primary.main, 0.15)}, 0 2px 10px rgba(32, 33, 36, 0.14)`, }, diff --git a/src/views/UnifiedDataUploadDialog.tsx b/src/views/UnifiedDataUploadDialog.tsx index 73d325e1..c08dab82 100644 --- a/src/views/UnifiedDataUploadDialog.tsx +++ b/src/views/UnifiedDataUploadDialog.tsx @@ -31,6 +31,7 @@ import { StreamIcon, getConnectorIcon, connectorSortOrder } from '../icons'; import RestartAltIcon from '@mui/icons-material/RestartAlt'; import ArrowBackIcon from '@mui/icons-material/ArrowBack'; import AddIcon from '@mui/icons-material/Add'; +import HistoryIcon from '@mui/icons-material/History'; import Paper from '@mui/material/Paper'; import CircularProgress from '@mui/material/CircularProgress'; @@ -210,6 +211,85 @@ const DataSourceCard: React.FC = ({ : card; }; +// Compact pill variant of DataSourceCard. Used by the chat-focused landing +// so data sources read as lightweight affordances orbiting the composer, +// rather than a grid of blocks competing with it. Same click behavior as +// DataSourceCard — only the visual weight differs. The description is +// demoted to a hover tooltip so the row stays dense. +const SourcePill: React.FC = ({ + icon, + title, + description, + onClick, + disabled = false, + variant = 'data', + badge, + tooltip, +}) => { + const theme = useTheme(); + const isAction = variant === 'action'; + + const pill = ( + + + {icon} + + + {title} + + {badge} + + ); + + const tip = tooltip ?? (description || null); + return tip + ? {pill} + : pill; +}; + const getUniqueTableName = (baseName: string, existingNames: Set): string => { let uniqueName = baseName; let counter = 1; @@ -523,15 +603,10 @@ export const DataLoadMenu: React.FC = ({ }, { value: 'url' as UploadTabType, - title: t('upload.loadFromUrlTitle', { defaultValue: 'Load from URL (live)' }), + title: t('upload.loadFromUrlTitle', { defaultValue: 'Load from URL' }), description: t('upload.loadFromUrlDesc'), icon: , disabled: false, - badge: , }, ]; @@ -649,63 +724,21 @@ export const DataLoadMenu: React.FC = ({ // eslint-disable-next-line react-hooks/exhaustive-deps }), [t, onStartChat]); const agentChatBox = ( - - - - - {t('upload.dataLoadingAgent', { defaultValue: 'Data Loading Agent' })} - - {hasPriorConversation && onResumeChat && ( - - - {t('upload.resumePreviousConversation', { defaultValue: 'Previous conversation →' })} - - - )} - + + + + + + ) : undefined} onNonImageFile={(file) => { // Upload non-image files (Excel, CSV, JSON, …) to the // session scratch space. The filename is shown as a @@ -727,7 +760,7 @@ export const DataLoadMenu: React.FC = ({ }} attachments={agentAttachments} onAttachmentsChange={setAgentAttachments} - minRows={1} + minRows={4} tabSuggestion={t('upload.agentChatTabSuggestion', { defaultValue: 'What dataset do we have here?', })} @@ -746,85 +779,63 @@ export const DataLoadMenu: React.FC = ({ width: '100%', display: 'flex', flexDirection: 'column', - gap: 3, + gap: 2.5, mx: 0, textAlign: 'left', }}> - {/* Data Loading Agent quick-chat */} + {/* Data Loading Agent quick-chat — the hero of this surface */} {agentChatBox} - {/* Upload data */} - - - {t('upload.uploadData', { defaultValue: 'Upload data' })} - + {/* Sources — same width as the chat box so they read as part of it */} + + {/* Connected sources status bar — "Connected: xxx, xxx + connect + link" */} - {regularDataSources.map((source) => ( - + {t('upload.connectedLabel', { defaultValue: 'Connected:' })} + + {connectionSources.map((source) => ( + onSelectTab(source.value)} + onClick={() => handleConnectionClick(source.value)} disabled={source.disabled} - badge={source.badge} + variant={source.variant} + tooltip={source.tooltip} /> ))} - - {/* Data Connections */} - - - - {t('upload.dataConnections')} - + {/* Manual upload — one-off sources (file, paste, URL) */} - {connectionSources.map((source) => ( - + {t('upload.loadDirectlyLabel', { defaultValue: 'or load data directly:' })} + + {regularDataSources.map((source) => ( + handleConnectionClick(source.value)} + onClick={() => onSelectTab(source.value)} disabled={source.disabled} - variant={source.variant} - tooltip={source.tooltip} /> ))} diff --git a/src/views/VisualizationView.tsx b/src/views/VisualizationView.tsx index 698b5a72..e886b049 100644 --- a/src/views/VisualizationView.tsx +++ b/src/views/VisualizationView.tsx @@ -32,6 +32,7 @@ import { Alert, Fade, Grow, + CircularProgress, } from '@mui/material'; import _ from 'lodash'; @@ -62,6 +63,8 @@ import CasinoIcon from '@mui/icons-material/Casino'; import SaveAltIcon from '@mui/icons-material/SaveAlt'; import OpenInNewIcon from '@mui/icons-material/OpenInNew'; import CloseIcon from '@mui/icons-material/Close'; +import AddchartIcon from '@mui/icons-material/Addchart'; +import * as d3dsv from 'd3-dsv'; import { AgentToyIcon, AnimatedAgentToyIcon } from './AgentToyIcon'; import { CHART_TEMPLATES, getChartTemplate } from '../components/ChartTemplates'; @@ -1161,6 +1164,131 @@ const EmptyStateHero: FC<{ chartSelectionBox: React.ReactNode }> = ({ chartSelec ); }; +// Content-first action dock shown when a *table* (not a chart) is focused. +// The table becomes the primary content at the top of the canvas; this +// sticky bottom bar keeps a lightweight "pick a chart" affordance and a +// pointer to the chat available without a wall of buttons dominating the +// page. The full CHART_TEMPLATES palette lives inside the popover. +// NOTE: designed to extend to multi-table selection — when several tables +// are highlighted this dock can summarize the selection and offer actions +// that span them (e.g. "combine", "chart each"). +const TableActionDock: FC<{ + chartSelectionBox: React.ReactNode; + tableId: string; + tableName: string; + rows: any[]; + virtual: boolean; + gridReport?: { loadedCount: number; rowCount: number; virtual: boolean; canRandomize: boolean } | null; + onRandomize?: () => void; +}> = ({ chartSelectionBox, tableId, tableName, rows, virtual, gridReport, onRandomize }) => { + const { t } = useTranslation(); + const [anchorEl, setAnchorEl] = useState(null); + const [isDownloading, setIsDownloading] = useState(false); + const open = Boolean(anchorEl); + + const handleDownload = async () => { + if (isDownloading) return; + setIsDownloading(true); + try { + if (virtual) { + const response = await fetchWithIdentity(getUrls().EXPORT_TABLE_CSV, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ table_name: tableId, delimiter: ',' }), + }); + if (!response.ok) throw new Error('Export failed'); + const blob = await response.blob(); + const a = document.createElement('a'); + a.href = URL.createObjectURL(blob); + a.download = `${tableName}.csv`; + a.click(); + URL.revokeObjectURL(a.href); + } else { + const csvContent = d3dsv.dsvFormat(',').format(rows); + const blob = new Blob([csvContent], { type: 'text/csv' }); + const a = document.createElement('a'); + a.href = URL.createObjectURL(blob); + a.download = `${tableName}.csv`; + a.click(); + URL.revokeObjectURL(a.href); + } + } catch (error) { + console.error('Error downloading table:', error); + } finally { + setIsDownloading(false); + } + }; + + return ( + + + + + + {gridReport?.virtual && ( + <> + + + {gridReport.loadedCount < gridReport.rowCount + ? t('dataGrid.loadedOfTotal', { loaded: gridReport.loadedCount, total: gridReport.rowCount }) + : t('dataGrid.rowCount', { count: gridReport.rowCount })} + + {gridReport.canRandomize && ( + + + + + + )} + + )} + + setAnchorEl(null)} + anchorOrigin={{ vertical: 'top', horizontal: 'left' }} + transformOrigin={{ vertical: 'bottom', horizontal: 'left' }} + slotProps={{ paper: { sx: { p: 2, maxWidth: 'min(90vw, 1140px)' } } }} + > + {/* Clicking a template creates the chart and this whole + empty-state unmounts; closing the popover here keeps + things tidy in the transient frame. */} + setAnchorEl(null)}> + {chartSelectionBox} + + + + ); +}; + export const VisualizationViewFC: FC = function VisualizationView({ }) { const { t } = useTranslation(); @@ -1180,6 +1308,11 @@ export const VisualizationViewFC: FC = function VisualizationView let tables = useSelector((state: DataFormulatorState) => state.tables); + // Virtual-pagination state reported up from the focused-table grid, so the + // bottom toolbar can show the loaded/total count and drive the random dice. + const [tableGridReport, setTableGridReport] = React.useState<{ loadedCount: number; rowCount: number; virtual: boolean; canRandomize: boolean } | null>(null); + const [tableRandomizeToken, setTableRandomizeToken] = React.useState(0); + let focusedChart = allCharts.find(c => c.id == focusedChartId) as Chart; let synthesisRunning = focusedChartId ? chartSynthesisInProgress.includes(focusedChartId) : false; @@ -1231,10 +1364,12 @@ export const VisualizationViewFC: FC = function VisualizationView )) } + const isTableFocus = focusedId?.type === 'table' && !!focusedTableId; return ( + {!isTableFocus && ( {(() => { @@ -1267,6 +1402,7 @@ export const VisualizationViewFC: FC = function VisualizationView })()} + )} {focusedId?.type === 'table' && focusedTableId && (() => { const focusedTable = tables.find(t => t.id === focusedTableId); if (!focusedTable) return null; @@ -1274,12 +1410,10 @@ export const VisualizationViewFC: FC = function VisualizationView const HEADER_HEIGHT = 32; const FOOTER_HEIGHT = 32; const MIN_TABLE_HEIGHT = 150; - const MAX_TABLE_HEIGHT = 400; const MIN_TABLE_WIDTH = 300; const MAX_TABLE_WIDTH = 900; const rowCount = focusedTable.virtual?.rowCount || focusedTable.rows?.length || 0; const contentHeight = HEADER_HEIGHT + rowCount * ROW_HEIGHT + FOOTER_HEIGHT; - const adaptiveHeight = Math.max(MIN_TABLE_HEIGHT, Math.min(MAX_TABLE_HEIGHT, contentHeight)); const ROW_ID_COL_WIDTH = 56; const sampleSize = Math.min(29, focusedTable.rows.length); const step = focusedTable.rows.length > sampleSize ? focusedTable.rows.length / sampleSize : 1; @@ -1295,16 +1429,40 @@ export const VisualizationViewFC: FC = function VisualizationView const SCROLLBAR_WIDTH = 17; // +34px gutter so the maximize button can sit just outside the table on the right. const adaptiveWidth = Math.max(MIN_TABLE_WIDTH, Math.min(MAX_TABLE_WIDTH, totalColWidth + SCROLLBAR_WIDTH + 16)) + 34; + // Single card that fills the available canvas height as much + // as possible (tall tables scroll inside), but never grows past + // its own content so short tables stay compact and centered. + // Width fills up to adaptiveWidth with an 80% floor. +48px is + // the focused-view header bar. + const HEADER_BAR_HEIGHT = 48; + const maxCardHeight = contentHeight + HEADER_BAR_HEIGHT; return ( - + + + ); })()} + {isTableFocus && focusedTableId && (() => { + const ft = tables.find(t => t.id === focusedTableId); + return ( + setTableRandomizeToken(x => x + 1)} + /> + ); + })()} diff --git a/tests/backend/agents/test_data_loading_discovery_tools.py b/tests/backend/agents/test_data_loading_discovery_tools.py index 61e65de2..7c9be80f 100644 --- a/tests/backend/agents/test_data_loading_discovery_tools.py +++ b/tests/backend/agents/test_data_loading_discovery_tools.py @@ -417,3 +417,107 @@ def test_includes_current_date_and_time(self) -> None: now = datetime.now() assert f"Current date and time: {now.strftime('%Y-%m-%d')}" in prompt assert f"({now.strftime('%A')})" in prompt + + +# ------------------------------------------------------------------ +# probe_data (design 37 §4.2 / §7) +# ------------------------------------------------------------------ + +class _StubLoader: + """Records the probe call and returns a canned result.""" + + def __init__(self, result=None): + self.result = result if result is not None else { + "rows": [{"n": 3}], "columns": ["n"], "row_count": 1, "exact": True, + } + self.calls = [] + + def probe(self, path, query): + self.calls.append((path, query)) + return self.result + + +class TestProbeData: + def test_missing_ids_returns_error(self, tmp_path: Path) -> None: + agent = DataLoadingAgent(client=None, workspace=_FakeWorkspace(tmp_path)) + agent._probe_budget = 5 + assert "error" in agent._tool_probe_data({"table_key": "k_orders"}) + assert "error" in agent._tool_probe_data({"source_id": "pg_prod"}) + + def test_unknown_table_key_returns_error(self, tmp_path: Path) -> None: + save_catalog(tmp_path, "pg_prod", _SAMPLE_TABLES) + agent = DataLoadingAgent(client=None, workspace=_FakeWorkspace(tmp_path)) + agent._probe_budget = 5 + + result = agent._tool_probe_data({ + "source_id": "pg_prod", "table_key": "nope", "query": {}, + }) + assert "error" in result + assert "not found" in result["error"] + + def test_budget_exhaustion_returns_error(self, tmp_path: Path) -> None: + save_catalog(tmp_path, "pg_prod", _SAMPLE_TABLES) + agent = DataLoadingAgent(client=None, workspace=_FakeWorkspace(tmp_path)) + agent._probe_budget = 0 + + result = agent._tool_probe_data({ + "source_id": "pg_prod", "table_key": "k_orders", "query": {}, + }) + assert "error" in result + assert "budget" in result["error"].lower() + + def test_resolves_path_and_delegates_to_loader(self, tmp_path: Path) -> None: + save_catalog(tmp_path, "pg_prod", _SAMPLE_TABLES) + agent = DataLoadingAgent(client=None, workspace=_FakeWorkspace(tmp_path)) + agent._probe_budget = 5 + stub = _StubLoader() + + with patch( + "data_formulator.data_connector.resolve_live_loader", + return_value=stub, + ): + result = agent._tool_probe_data({ + "source_id": "pg_prod", + "table_key": "k_orders", + "query": {"aggregates": [{"op": "count", "as": "n"}]}, + }) + + # The model-facing table_key is mapped to the catalog path. + assert stub.calls == [(["Sales", "monthly_orders"], + {"aggregates": [{"op": "count", "as": "n"}]})] + assert result["rows"] == [{"n": 3}] + assert "note" in result # row-cap guidance attached + assert agent._probe_budget == 4 # decremented once + + def test_not_connected_source_returns_error(self, tmp_path: Path) -> None: + save_catalog(tmp_path, "pg_prod", _SAMPLE_TABLES) + agent = DataLoadingAgent(client=None, workspace=_FakeWorkspace(tmp_path)) + agent._probe_budget = 5 + + with patch( + "data_formulator.data_connector.resolve_live_loader", + side_effect=RuntimeError("no such connector"), + ): + result = agent._tool_probe_data({ + "source_id": "pg_prod", "table_key": "k_orders", "query": {}, + }) + assert "error" in result + assert "not connected" in result["error"] + # Budget is not consumed when the loader can't be resolved. + assert agent._probe_budget == 5 + + def test_loader_error_result_passes_through(self, tmp_path: Path) -> None: + save_catalog(tmp_path, "pg_prod", _SAMPLE_TABLES) + agent = DataLoadingAgent(client=None, workspace=_FakeWorkspace(tmp_path)) + agent._probe_budget = 5 + stub = _StubLoader(result={"error": "bad column"}) + + with patch( + "data_formulator.data_connector.resolve_live_loader", + return_value=stub, + ): + result = agent._tool_probe_data({ + "source_id": "pg_prod", "table_key": "k_orders", "query": {}, + }) + assert result == {"error": "bad column"} + assert "note" not in result # no cap note on error results From 932626aac28e397347bdabbdf08ef5d9cded2b43 Mon Sep 17 00:00:00 2001 From: Chenglong Wang Date: Wed, 8 Jul 2026 16:57:22 -0700 Subject: [PATCH 35/40] improvements --- .../agents/agent_starter_questions.py | 119 ++++++++++++ py-src/data_formulator/routes/agents.py | 29 +++ src/app/dfSlice.tsx | 98 +++++++++- src/app/store.ts | 2 +- src/app/useAutoSave.tsx | 3 + src/app/utils.tsx | 3 + src/i18n/index.ts | 3 + src/i18n/locales/en/chart.json | 3 + src/i18n/locales/en/common.json | 3 + src/i18n/locales/zh/chart.json | 3 + src/i18n/locales/zh/common.json | 3 + src/views/AgentChatInput.tsx | 18 +- src/views/ChartQuickConfig.tsx | 103 +++++++++-- src/views/DataThread.tsx | 2 +- src/views/DataView.tsx | 8 +- src/views/SelectableDataGrid.tsx | 48 +++-- src/views/SimpleChartRecBox.tsx | 171 +++++++++++++++++- src/views/UnifiedDataUploadDialog.tsx | 2 +- src/views/VisualizationView.tsx | 155 ++++++++++------ 19 files changed, 665 insertions(+), 111 deletions(-) create mode 100644 py-src/data_formulator/agents/agent_starter_questions.py diff --git a/py-src/data_formulator/agents/agent_starter_questions.py b/py-src/data_formulator/agents/agent_starter_questions.py new file mode 100644 index 00000000..54e704e9 --- /dev/null +++ b/py-src/data_formulator/agents/agent_starter_questions.py @@ -0,0 +1,119 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import json +from data_formulator.agent_config import reasoning_effort_for +from data_formulator.agents.agent_utils import extract_json_objects +from data_formulator.agents.agent_language import inject_language_instruction + +import logging + +logger = logging.getLogger(__name__) + +_AGENT_ID = "starter_questions" + + +SYSTEM_PROMPT = '''You are a data analyst helping a user get started exploring a freshly loaded dataset. +You are given a summary of the available tables (their names, columns, and a few sample rows) and one designated "primary_table". +Propose a small number of short, concrete starter questions the user could ask to explore the data. + +Guidelines: +- Center the questions on the primary_table (about its own columns / trends / comparisons / distributions / top-N). +- If other tables are present and share a plausible key with the primary table, you MAY include ONE cross-table question that relates the primary table to another table. +- Each question must be answerable by charting or analyzing the provided data (do not invent columns that are not present). +- Keep each question short and natural — under 12 words, phrased as a request (e.g. "Compare sales across regions"). +- Make the questions diverse and prefer referencing specific column names so they feel tailored. +- Do NOT include a generic "show high-level trends" question — that one is already provided separately. + +Return ONLY a json object of the following form: + +{ + "questions": ["", ""] +} + +Example: + +[INPUT] + +{ + "primary_table": "sales", + "tables": [ + { + "name": "sales", + "columns": ["date", "region", "product", "revenue", "units"], + "sample_rows": [ + {"date": "2023-01-01", "region": "West", "product": "A", "revenue": 1200, "units": 30}, + {"date": "2023-01-02", "region": "East", "product": "B", "revenue": 800, "units": 20} + ] + } + ] +} + +[OUTPUT] + +{ + "questions": ["Compare revenue across regions", "Which products sell the most units?"] +} +''' + + +class StarterQuestionsAgent(object): + + def __init__(self, client, language_instruction: str = ""): + self.client = client + self.language_instruction = language_instruction + + def run(self, tables, primary_table=None, n=2): + """Generate a short list of starter exploration questions. + + ``tables`` is a list of dicts with ``name``, optional ``description`` + and either ``columns`` and/or ``sample_rows``. ``primary_table`` is + the name of the table the questions should center on. Returns a list + of question strings (best effort, may be empty on failure). + """ + + input_obj = {"primary_table": primary_table, "tables": tables, "num_questions": n} + + user_query = f"[INPUT]\n\n{json.dumps(input_obj, ensure_ascii=False, default=str)}\n\n[OUTPUT]" + + logger.info("[StarterQuestionsAgent] run start") + + system_prompt = inject_language_instruction( + SYSTEM_PROMPT, self.language_instruction, + ) + + messages = [{"role": "system", "content": system_prompt}, + {"role": "user", "content": user_query}] + + response = self.client.get_completion( + messages=messages, + reasoning_effort=reasoning_effort_for(_AGENT_ID, self.client.model), + ) + + for choice in response.choices: + logger.debug("\n=== Starter questions agent ===>\n") + logger.debug(choice.message.content + "\n") + + content = choice.message.content or "" + + questions = [] + json_blocks = extract_json_objects(content + "\n") + candidate = None + if len(json_blocks) > 0: + candidate = json_blocks[0] + else: + try: + candidate = json.loads(content + "\n") + except (json.JSONDecodeError, ValueError, TypeError): + candidate = None + + if isinstance(candidate, dict): + raw = candidate.get("questions", []) + if isinstance(raw, list): + questions = [str(q).strip() for q in raw if str(q).strip()] + elif isinstance(candidate, list): + questions = [str(q).strip() for q in candidate if str(q).strip()] + + return questions[:n] + + return [] diff --git a/py-src/data_formulator/routes/agents.py b/py-src/data_formulator/routes/agents.py index ab77f6e5..950feeaa 100644 --- a/py-src/data_formulator/routes/agents.py +++ b/py-src/data_formulator/routes/agents.py @@ -19,6 +19,7 @@ import pandas as pd from data_formulator.agents.agent_sort_data import SortDataAgent +from data_formulator.agents.agent_starter_questions import StarterQuestionsAgent from data_formulator.agents.agent_simple import SimpleAgents from data_formulator.auth.identity import get_identity_id from data_formulator.security.code_signing import sign_result, verify_code, MAX_CODE_SIZE @@ -289,6 +290,34 @@ def sort_data_request(): logger.error("Error in sort-data", exc_info=e) raise classify_and_wrap_llm_error(e) from e +@agent_bp.route('/derive-starter-questions', methods=['GET', 'POST']) +def derive_starter_questions_request(): + """Generate a few short, data-tailored starter exploration questions. + + Called once when a workspace's set of root tables changes (e.g. after + data is loaded). Input: ``input_tables`` (list of {name, columns, + sample_rows, description}) and ``model``. Returns ``{"result": [..]}``. + """ + if not request.is_json: + raise AppError(ErrorCode.INVALID_REQUEST, "Invalid request format") + + logger.info("# derive-starter-questions request") + content = request.get_json() + + try: + client = get_client(content['model']) + + n = content.get('n', 2) + language_instruction = get_language_instruction(mode="compact") + agent = StarterQuestionsAgent(client=client, language_instruction=language_instruction) + questions = agent.run(content.get('input_tables', []), primary_table=content.get('primary_table'), n=n) + + questions = questions if questions is not None else [] + return json_ok({"result": questions}) + except Exception as e: + logger.error("Error in derive-starter-questions", exc_info=e) + raise classify_and_wrap_llm_error(e) from e + @agent_bp.route('/analyst-streaming', methods=['GET', 'POST']) def analyst_streaming(): """Unified AnalystAgent streaming endpoint (design-docs/35 + /36). diff --git a/src/app/dfSlice.tsx b/src/app/dfSlice.tsx index fdc24d72..8264dba5 100644 --- a/src/app/dfSlice.tsx +++ b/src/app/dfSlice.tsx @@ -272,6 +272,17 @@ export interface DataFormulatorState { * + briefly highlight). Cleared by the sidebar after consumption. */ focusedConnectorId?: string; + + /** + * AI-generated starter exploration questions, stored per root table. + * Keyed by table id; each entry's `questions` are tailored to that table + * (plus optional cross-table questions when other tables exist). + * `signature` is the root-table-set signature the questions were generated + * for (so they refresh when the set of tables changes). Persisted; + * `starterQuestionsStatus` is transient. + */ + starterQuestions: { [tableId: string]: { questions: string[]; signature: string } }; + starterQuestionsStatus: { [tableId: string]: 'idle' | 'loading' | 'error' }; } // Define the initial state using that type @@ -346,8 +357,10 @@ const initialState: DataFormulatorState = { dataSourceSidebarTab: 'sources', focusedConnectorId: undefined, -} + starterQuestions: {}, + starterQuestionsStatus: {}, +} /** * Non-memoized equivalent of `dfSelectors.getAllCharts` for use inside * reducers. Reducers receive an Immer draft `state`; passing a draft into @@ -530,6 +543,10 @@ let removeTableStateRoutine = (state: DataFormulatorState, tableId: string) => { // Delete reports triggered from this table state.generatedReports = state.generatedReports.filter(r => r.triggerTableId !== tableId); + // Drop this table's starter questions / generation status + delete state.starterQuestions[tableId]; + delete state.starterQuestionsStatus[tableId]; + if (state.focusedId?.type === 'table' && state.focusedId.tableId === tableId) { state.focusedId = state.tables.length > 0 ? { type: 'table', tableId: state.tables[0].id } : undefined; } @@ -564,6 +581,60 @@ export const fetchFieldSemanticType = createAsyncThunk( } ); +/** + * Generate a few short, table-tailored starter exploration questions for the + * currently focused root table. Called (debounced) when a root table is + * focused and it has no fresh questions for the current table-set signature. + * The focused table is the "primary" table; all root tables are passed as + * context so the agent can add a cross-table question when relevant. + * Dispatches `startStarterQuestions` up front, then `setStarterQuestions` / + * `setStarterQuestionsError`. Results are only applied if the signature is + * still current (guards against stale responses when data changed mid-flight). + */ +export const generateStarterQuestions = createAsyncThunk( + "dataFormulatorSlice/generateStarterQuestions", + async (arg: { tableId: string; signature: string; tableIds: string[] }, { getState, dispatch }) => { + const state = getState() as DataFormulatorState; + + dispatch(dfActions.startStarterQuestions({ tableId: arg.tableId, signature: arg.signature })); + + const inputTables = arg.tableIds + .map(id => state.tables.find(t => t.id === id)) + .filter((t): t is DictTable => !!t) + .map(t => ({ + name: t.id, + columns: t.names, + sample_rows: (t.rows || []).slice(0, 10), + description: typeof t.description === 'string' ? t.description : '', + })); + + if (inputTables.length === 0) { + dispatch(dfActions.setStarterQuestions({ tableId: arg.tableId, signature: arg.signature, questions: [] })); + return; + } + + try { + const { data } = await apiRequest(getUrls().DERIVE_STARTER_QUESTIONS, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + input_tables: inputTables, + primary_table: arg.tableId, + model: dfSelectors.getActiveModel(state), + n: 2, + }), + }); + const questions: string[] = Array.isArray(data?.result) + ? data.result.map((q: any) => String(q)).filter((q: string) => q.trim() !== '') + : []; + dispatch(dfActions.setStarterQuestions({ tableId: arg.tableId, signature: arg.signature, questions: questions.slice(0, 2) })); + } catch (err) { + console.warn('generateStarterQuestions failed', err); + dispatch(dfActions.setStarterQuestionsError({ tableId: arg.tableId, signature: arg.signature })); + } + } +); + /** * Fetch backend-computed per-column statistics for a workspace-stored * (virtual) table and merge them into ``table.metadata``. Powers the @@ -737,6 +808,27 @@ export const dataFormulatorSlice = createSlice({ clearFocusedConnector: (state) => { state.focusedConnectorId = undefined; }, + /** Mark a starter-questions generation as started for a table. */ + startStarterQuestions: (state, action: PayloadAction<{ tableId: string; signature: string }>) => { + state.starterQuestions[action.payload.tableId] = { + questions: [], + signature: action.payload.signature, + }; + state.starterQuestionsStatus[action.payload.tableId] = 'loading'; + }, + /** Store generated starter questions (only if signature still current). */ + setStarterQuestions: (state, action: PayloadAction<{ tableId: string; signature: string; questions: string[] }>) => { + const cur = state.starterQuestions[action.payload.tableId]; + if (!cur || cur.signature !== action.payload.signature) return; + cur.questions = action.payload.questions; + state.starterQuestionsStatus[action.payload.tableId] = 'idle'; + }, + /** Mark starter-questions generation as failed (only if signature current). */ + setStarterQuestionsError: (state, action: PayloadAction<{ tableId: string; signature: string }>) => { + const cur = state.starterQuestions[action.payload.tableId]; + if (!cur || cur.signature !== action.payload.signature) return; + state.starterQuestionsStatus[action.payload.tableId] = 'error'; + }, loadState: (state, action: PayloadAction) => { const saved = action.payload; @@ -834,6 +926,10 @@ export const dataFormulatorSlice = createSlice({ // repopulates this slice from the module cache / fresh // renders after load. chartThumbnails: {}, + + // Restore persisted starter questions (status is transient). + starterQuestions: saved.starterQuestions ?? {}, + starterQuestionsStatus: {}, }; }, setServerConfig: (state, action: PayloadAction) => { diff --git a/src/app/store.ts b/src/app/store.ts index c230f5a6..a4a0828b 100644 --- a/src/app/store.ts +++ b/src/app/store.ts @@ -16,7 +16,7 @@ const persistConfig = { // globalModels are always fetched fresh from the server on each app start, // so there is no need (and it would cause stale-data issues) to persist them. // In-progress flags are transient and should not survive page refreshes. - blacklist: ['serverConfig', 'globalModels', 'chartSynthesisInProgress'], + blacklist: ['serverConfig', 'globalModels', 'chartSynthesisInProgress', 'starterQuestionsStatus'], } const persistedReducer = persistReducer(persistConfig, dataFormulatorReducer) diff --git a/src/app/useAutoSave.tsx b/src/app/useAutoSave.tsx index 3a167f14..e2888a17 100644 --- a/src/app/useAutoSave.tsx +++ b/src/app/useAutoSave.tsx @@ -17,6 +17,9 @@ const EXCLUDED_FIELDS = new Set([ // Transient fields that shouldn't trigger or be included in saves 'chartSynthesisInProgress', 'cleanInProgress', 'sessionLoading', 'sessionLoadingLabel', + // Starter-questions status is transient (loading/error); the questions + // themselves are persisted, but the fetch status should reset on reload. + 'starterQuestionsStatus', // Thumbnails are derived from chart specs + table data; re-rendered // from the module cache on reload, so don't waste bandwidth saving them. 'chartThumbnails', diff --git a/src/app/utils.tsx b/src/app/utils.tsx index 196c403d..e7de0b56 100644 --- a/src/app/utils.tsx +++ b/src/app/utils.tsx @@ -49,6 +49,9 @@ export function getUrls() { GET_RECOMMENDATION_QUESTIONS: `/api/agent/get-recommendation-questions`, + // Starter exploration questions (generated on data load) + DERIVE_STARTER_QUESTIONS: `/api/agent/derive-starter-questions`, + // Workspace display name (auto-naming) WORKSPACE_NAME: `/api/agent/workspace-name`, diff --git a/src/i18n/index.ts b/src/i18n/index.ts index ea75a071..2912d1d5 100644 --- a/src/i18n/index.ts +++ b/src/i18n/index.ts @@ -6,6 +6,9 @@ import { initReactI18next } from 'react-i18next'; import LanguageDetector from 'i18next-browser-languagedetector'; import { en, zh } from './locales'; +// NOTE: locale JSON is ingested into the i18next store once, here, at init(). +// Adding keys to a locale file requires a full page reload (not just HMR) for +// the running app to pick them up, since HMR won't re-run this init(). const resources = { en: { translation: en }, zh: { translation: zh }, diff --git a/src/i18n/locales/en/chart.json b/src/i18n/locales/en/chart.json index 68dd50c2..60e4c96b 100644 --- a/src/i18n/locales/en/chart.json +++ b/src/i18n/locales/en/chart.json @@ -20,6 +20,9 @@ "duplicate": "duplicate the chart", "delete": "delete", "deleteChart": "delete chart", + "deleteChartConfirm": "Delete this chart?", + "deleteChartCancel": "cancel", + "deleteChartYes": "delete", "sampleSize": "Sample size", "sampleSizeAria": "Sample size", "sampleAgain": "sample again!", diff --git a/src/i18n/locales/en/common.json b/src/i18n/locales/en/common.json index 7357326d..12999d79 100644 --- a/src/i18n/locales/en/common.json +++ b/src/i18n/locales/en/common.json @@ -540,6 +540,8 @@ "generateReport": "Generate a report", "reportPrompt": "Write a report summarizing the key findings from this exploration.", "askedForReport": "Write a report summarizing the exploration.", + "expandStarters": "Show suggestions", + "collapseStarters": "Hide suggestions", "endConversation": "End conversation", "sendReply": "Send reply", "explore": "Explore", @@ -590,6 +592,7 @@ "rowCount": "{{count}} rows", "loadedOfTotal": "{{loaded}} / {{total}} rows", "viewRandomRows": "view 10000 random rows from this table", + "restoreOrder": "Restore original order", "downloadAsCsv": "Download as CSV", "downloading": "Downloading...", "columnMenu": { diff --git a/src/i18n/locales/zh/chart.json b/src/i18n/locales/zh/chart.json index d08ce521..3b9f02d5 100644 --- a/src/i18n/locales/zh/chart.json +++ b/src/i18n/locales/zh/chart.json @@ -20,6 +20,9 @@ "duplicate": "复制图表", "delete": "删除", "deleteChart": "删除图表", + "deleteChartConfirm": "确定删除此图表？", + "deleteChartCancel": "取消", + "deleteChartYes": "删除", "sampleSize": "样本大小", "sampleSizeAria": "样本大小", "sampleAgain": "重新采样！", diff --git a/src/i18n/locales/zh/common.json b/src/i18n/locales/zh/common.json index e08983fb..07d2e376 100644 --- a/src/i18n/locales/zh/common.json +++ b/src/i18n/locales/zh/common.json @@ -476,6 +476,7 @@ "rowCount": "{{count}} 行", "loadedOfTotal": "已加载 {{loaded}} / {{total}} 行", "viewRandomRows": "查看此表的 10000 行随机数据", + "restoreOrder": "恢复原始顺序", "downloadAsCsv": "下载为 CSV", "downloading": "下载中...", "columnMenu": { @@ -591,6 +592,8 @@ "generateReport": "生成报告", "reportPrompt": "撰写一份报告，总结本次探索的主要发现。", "askedForReport": "撰写一份报告，总结本次探索。", + "expandStarters": "显示建议", + "collapseStarters": "隐藏建议", "endConversation": "结束对话", "sendReply": "发送回复", "explore": "探索", diff --git a/src/views/AgentChatInput.tsx b/src/views/AgentChatInput.tsx index b4d9ea0e..6a0598ba 100644 --- a/src/views/AgentChatInput.tsx +++ b/src/views/AgentChatInput.tsx @@ -470,7 +470,7 @@ export const AgentChatInput: React.FC = ({ px: 1.5, py: 0.5, cursor: 'pointer', - color: 'text.secondary', + color: 'text.primary', '&:hover': { bgcolor: 'action.hover', color: 'text.primary' }, }} > @@ -482,7 +482,7 @@ export const AgentChatInput: React.FC = ({ alignItems: 'center', justifyContent: 'center', width: 16, flexShrink: 0, - color: 'text.disabled', + color: 'text.secondary', }} > {s.icon} @@ -509,20 +509,6 @@ export const AgentChatInput: React.FC = ({ > {s.label} - {s.kind && ( - - {s.kind} - - )} ))} diff --git a/src/views/ChartQuickConfig.tsx b/src/views/ChartQuickConfig.tsx index 570a6b89..b4b4b968 100644 --- a/src/views/ChartQuickConfig.tsx +++ b/src/views/ChartQuickConfig.tsx @@ -11,7 +11,7 @@ import { FC } from 'react'; import React from 'react'; import { useTranslation } from 'react-i18next'; import { useSelector, useDispatch } from 'react-redux'; -import { Box, Typography, Select, MenuItem, useTheme, Tooltip, IconButton, Divider } from '@mui/material'; +import { Box, Typography, Select, MenuItem, useTheme, Tooltip, IconButton, Divider, Popover, Button } from '@mui/material'; import DeleteIcon from '@mui/icons-material/Delete'; import { DataFormulatorState, dfActions, dfSelectors } from '../app/dfSlice'; @@ -124,10 +124,15 @@ type QuickControl = { }; export const ChartQuickConfig: FC = function ({ chartId, tableMetadata, options, deleteDisabled }) { - const { t } = useTranslation('chart'); + const { t } = useTranslation(); const theme = useTheme(); const dispatch = useDispatch(); + // Confirmation popover for the destructive delete action: clicking the + // trash opens a small "delete?" popover so an accidental click can't remove + // the chart. Anchored to the button; explicit confirm/dismiss. + const [deleteAnchor, setDeleteAnchor] = React.useState(null); + const allCharts = useSelector(dfSelectors.getAllCharts); const conceptShelfItems = useSelector((state: DataFormulatorState) => state.conceptShelfItems); const chart = allCharts.find((c: Chart) => c.id == chartId) as Chart | undefined; @@ -326,22 +331,84 @@ export const ChartQuickConfig: FC = function ({ chartId, })} {/* Built-in chart-level action: delete this chart. Lives in the property-config bar so the chart's controls and its delete sit - together; a hairline divider sets it apart from the property - controls. Only shown when there are controls to set it apart - from — otherwise it stands alone in the bar. */} - - - dispatch(dfActions.deleteChartById(chartId))} - sx={{ color: 'text.disabled','&:hover': { color: 'error.main', backgroundColor: 'rgba(211, 47, 47, 0.08)' } }} - > - - - - + together; a hairline divider + extra spacing sets it apart from + the property controls so it isn't easy to mis-fire. Clicking + opens a confirmation popover since it's destructive. */} + + + + + setDeleteAnchor(e.currentTarget)} + sx={{ + color: deleteAnchor ? 'error.main' : 'text.disabled', + backgroundColor: deleteAnchor ? 'rgba(211, 47, 47, 0.08)' : 'transparent', + '&:hover': { color: 'error.main', backgroundColor: 'rgba(211, 47, 47, 0.08)' }, + }} + > + + + + + setDeleteAnchor(null)} + anchorOrigin={{ vertical: 'top', horizontal: 'center' }} + transformOrigin={{ vertical: 'bottom', horizontal: 'left' }} + slotProps={{ paper: { sx: { + p: 1.25, + borderRadius: '8px', + mt: '-10px', + ml: '-14px', + overflow: 'visible', + boxShadow: '0 4px 16px rgba(0,0,0,0.16)', + // Callout arrow pointing straight down at the delete button + // (button center sits under this arrow via the ml offset). + '&::before': { + content: '""', + position: 'absolute', + bottom: -5, + left: 14, + width: 10, + height: 10, + backgroundColor: 'background.paper', + transform: 'rotate(45deg)', + boxShadow: '2px 2px 3px rgba(0,0,0,0.06)', + }, + } } }} + > + + {t('chart.deleteChartConfirm')} + + + + + + + ); diff --git a/src/views/DataThread.tsx b/src/views/DataThread.tsx index fff265d2..8a194a66 100644 --- a/src/views/DataThread.tsx +++ b/src/views/DataThread.tsx @@ -3443,7 +3443,7 @@ export const DataThread: FC<{sx?: SxProps}> = function ({ sx }) { display: 'flex', flexDirection: 'row', flexWrap: 'nowrap', - justifyContent: 'center', + justifyContent: 'flex-start', gap: `${CARD_GAP}px`, py: 1, // Bottom padding leaves room so the scroll handler can position diff --git a/src/views/DataView.tsx b/src/views/DataView.tsx index 98217775..f5dd9214 100644 --- a/src/views/DataView.tsx +++ b/src/views/DataView.tsx @@ -42,10 +42,13 @@ export interface FreeDataViewProps { // Controlled random-rows trigger + virtual-state report, forwarded to the // grid so the external toolbar can drive/observe them. randomizeToken?: number; - onStateReport?: (s: { loadedCount: number; rowCount: number; virtual: boolean; canRandomize: boolean }) => void; + // Controlled "restore natural order" trigger, forwarded to the grid so the + // toolbar can undo a random sample. + resetOrderToken?: number; + onStateReport?: (s: { loadedCount: number; rowCount: number; virtual: boolean; canRandomize: boolean; isRandom: boolean }) => void; } -export const FreeDataViewFC: FC = function DataView({ maximizable, tableId, showHeaderBar, hideFooter, randomizeToken, onStateReport }) { +export const FreeDataViewFC: FC = function DataView({ maximizable, tableId, showHeaderBar, hideFooter, randomizeToken, resetOrderToken, onStateReport }) { const { t } = useTranslation(); const [maximized, setMaximized] = React.useState(false); @@ -165,6 +168,7 @@ export const FreeDataViewFC: FC = function DataView({ maximiz searchText={searchQuery} hideFooter={hideFooter} randomizeToken={randomizeToken} + resetOrderToken={resetOrderToken} onStateReport={onStateReport} /> diff --git a/src/views/SelectableDataGrid.tsx b/src/views/SelectableDataGrid.tsx index 222a883e..62b77633 100644 --- a/src/views/SelectableDataGrid.tsx +++ b/src/views/SelectableDataGrid.tsx @@ -66,9 +66,12 @@ interface SelectableDataGridProps { hideFooter?: boolean; // Bumping this number triggers a "random rows" refetch (virtual tables). randomizeToken?: number; + // Bumping this number restores the natural (#rowId head) order after a + // random sample (virtual tables). + resetOrderToken?: number; // Report virtual-pagination state up so an external toolbar can render the // loaded/total count and enable the random-rows action. - onStateReport?: (s: { loadedCount: number; rowCount: number; virtual: boolean; canRandomize: boolean }) => void; + onStateReport?: (s: { loadedCount: number; rowCount: number; virtual: boolean; canRandomize: boolean; isRandom: boolean }) => void; } @@ -334,7 +337,7 @@ const VirtuosoTableBody = React.forwardRef((props, ref) const PAGE_SIZE = 500; export const SelectableDataGrid: React.FC = React.memo(({ - tableId, rows, tableName, columnDefs, rowCount, virtual, searchText, hideFooter, randomizeToken, onStateReport }) => { + tableId, rows, tableName, columnDefs, rowCount, virtual, searchText, hideFooter, randomizeToken, resetOrderToken, onStateReport }) => { const { t } = useTranslation(); const [orderBy, setOrderBy] = React.useState(undefined); @@ -354,11 +357,12 @@ export const SelectableDataGrid: React.FC = React.memo( // Ref-based bridge to fetchVirtualData (declared further below); lets stable // sort handlers call into the latest fetch function without re-creating themselves. - const fetchVirtualDataRef = React.useRef<((sortByColumnIds: string[], sortOrder: 'asc' | 'desc', offset?: number, append?: boolean) => void) | null>(null); + const fetchVirtualDataRef = React.useRef<((sortByColumnIds: string[], sortOrder: 'asc' | 'desc', offset?: number, append?: boolean, random?: boolean) => void) | null>(null); const applySort = React.useCallback((newOrderBy: string | undefined, newOrder: 'asc' | 'desc') => { setOrder(newOrder); setOrderBy(newOrderBy); + setIsRandom(false); if (virtual) { fetchVirtualDataRef.current?.(newOrderBy ? [newOrderBy] : [], newOrder); } @@ -367,7 +371,9 @@ export const SelectableDataGrid: React.FC = React.memo( let theme = useTheme(); const [rowsToDisplay, setRowsToDisplay] = React.useState(rows); - + // True while the grid is showing a random sample (virtual tables). Lets the + // external toolbar offer a "restore order" action alongside the dice. + const [isRandom, setIsRandom] = React.useState(false); const [isLoading, setIsLoading] = React.useState(true); const [isLoadingMore, setIsLoadingMore] = React.useState(false); const [isDownloading, setIsDownloading] = React.useState(false); @@ -401,20 +407,35 @@ export const SelectableDataGrid: React.FC = React.memo( rowCount: effectiveCount, virtual, canRandomize: virtual && effectiveCount > 10000, + isRandom: virtual && isRandom, }); - }, [rowsToDisplay.length, rowCount, serverRowCount, virtual, onStateReport]); + }, [rowsToDisplay.length, rowCount, serverRowCount, virtual, isRandom, onStateReport]); // Bumping `randomizeToken` (from the external toolbar's dice) resets sort - // and refetches a fresh random page for virtual tables. + // and refetches a fresh random sample (ORDER BY RANDOM() on the server, + // which also respects any active filters/search) for virtual tables. const randomizeMountRef = React.useRef(true); React.useEffect(() => { if (randomizeMountRef.current) { randomizeMountRef.current = false; return; } if (!virtual) return; setOrderBy(undefined); setOrder('asc'); - fetchVirtualDataRef.current?.([], 'asc'); + setIsRandom(true); + fetchVirtualDataRef.current?.([], 'asc', 0, false, true); }, [randomizeToken]); + // Bumping `resetOrderToken` (from the toolbar's "restore order" button) + // returns a randomized virtual table to its natural #rowId head order. + const resetOrderMountRef = React.useRef(true); + React.useEffect(() => { + if (resetOrderMountRef.current) { resetOrderMountRef.current = false; return; } + if (!virtual) return; + setOrderBy(undefined); + setOrder('asc'); + setIsRandom(false); + fetchVirtualDataRef.current?.([], 'asc', 0, false, false); + }, [resetOrderToken]); + // Only the Table component depends on columnDefs (for colgroup); memoize it // so react-virtuoso keeps a stable reference when columns haven't changed. const columnIds = columnDefs.map(c => c.id).join(','); @@ -497,6 +518,7 @@ export const SelectableDataGrid: React.FC = React.memo( sortOrder: 'asc' | 'desc', offset: number = 0, append: boolean = false, + random: boolean = false, ) => { if (!append) { setIsLoading(true); @@ -511,9 +533,11 @@ export const SelectableDataGrid: React.FC = React.memo( table: tableId, size: PAGE_SIZE, offset, - method: sortByColumnIds.length > 0 - ? (sortOrder === 'asc' ? 'head' : 'bottom') - : 'head', + method: random + ? 'random' + : (sortByColumnIds.length > 0 + ? (sortOrder === 'asc' ? 'head' : 'bottom') + : 'head'), order_by_fields: sortByColumnIds.length > 0 ? sortByColumnIds : ['#rowId'], ...(activeFilters.length > 0 ? { filters: activeFilters } : {}), ...(searchRef.current ? { search: searchRef.current } : {}), @@ -535,7 +559,7 @@ export const SelectableDataGrid: React.FC = React.memo( setRowsToDisplay(newRows); } setServerRowCount(totalCount); - setHasMore(offset + newRows.length < totalCount); + setHasMore(!random && offset + newRows.length < totalCount); setIsLoading(false); setIsLoadingMore(false); }) @@ -562,6 +586,7 @@ export const SelectableDataGrid: React.FC = React.memo( didMountFiltersRef.current = true; return; } + setIsRandom(false); fetchVirtualData(orderBy ? [orderBy] : [], order, 0, false); // eslint-disable-next-line react-hooks/exhaustive-deps }, [columnFilters]); @@ -576,6 +601,7 @@ export const SelectableDataGrid: React.FC = React.memo( return; } searchRef.current = (searchText || '').trim(); + setIsRandom(false); fetchVirtualData(orderBy ? [orderBy] : [], order, 0, false); // eslint-disable-next-line react-hooks/exhaustive-deps }, [searchText]); diff --git a/src/views/SimpleChartRecBox.tsx b/src/views/SimpleChartRecBox.tsx index c0ceaac2..3fe84f18 100644 --- a/src/views/SimpleChartRecBox.tsx +++ b/src/views/SimpleChartRecBox.tsx @@ -14,6 +14,7 @@ import { Card, LinearProgress, Chip, + Collapse, Popper, Paper, MenuList, @@ -21,7 +22,7 @@ import { } from '@mui/material'; import { useDispatch, useSelector } from 'react-redux'; -import { DataFormulatorState, dfActions, dfSelectors, fetchCodeExpl, fetchFieldSemanticType, generateFreshChart, GeneratedReport } from '../app/dfSlice'; +import { DataFormulatorState, dfActions, dfSelectors, fetchCodeExpl, fetchFieldSemanticType, generateFreshChart, generateStarterQuestions, GeneratedReport } from '../app/dfSlice'; import { AppDispatch } from '../app/store'; import { resolveRecommendedChart, getUrls, getTriggers, translateBackend } from '../app/utils'; import { streamRequest } from '../app/apiClient'; @@ -33,9 +34,9 @@ import { normalizeClarifyEvent, formatClarificationResponses } from '../app/clar import { alpha } from '@mui/material/styles'; import { WritingPencil } from '../components/FunComponents'; import ArrowUpwardRoundedIcon from '@mui/icons-material/ArrowUpwardRounded'; -import CloseIcon from '@mui/icons-material/Close'; import AddIcon from '@mui/icons-material/Add'; import TipsAndUpdatesIcon from '@mui/icons-material/TipsAndUpdates'; +import BoltIcon from '@mui/icons-material/Bolt'; import EditOutlinedIcon from '@mui/icons-material/EditOutlined'; import StopIcon from '@mui/icons-material/Stop'; @@ -45,12 +46,43 @@ import { Theme } from '@mui/material/styles'; import { useTranslation } from 'react-i18next'; import { shouldAutoFocusGeneratedChart } from '../app/agentInteractionPolicy'; import { ClarificationPanel, DelegatePanel, ExplanationPanel } from './AgentPausePanel'; +import { CARD_WIDTH } from './threadLayout'; + +// Approx footprint of the leading lightning-bolt IconButton (size small, +// p:0.5 + 16px icon). Used to cap a starter chip so a single chip fits +// within one thread-column width alongside the toggle icon. +const STARTER_ICON_WIDTH = 28; // Seed prompt used when the user invokes "report" mode (or a report hand-off) // without typing an explicit instruction. The unified analyst loads its // `report` skill and emits `write_report` within a normal explore run. const REPORT_SEED_PROMPT = 'Write a report summarizing the exploration.'; +// A starter-question chip that only shows a tooltip when its label is +// truncated (i.e. the text is too long to fit within the capped width). +const StarterChip: FC<{ label: string; onClick: () => void; sx: any }> = ({ label, onClick, sx }) => { + const labelRef = useRef(null); + const [isClipped, setIsClipped] = useState(false); + const checkClipped = () => { + // The MUI `.MuiChip-label` element (parent of our span) is the one that + // applies overflow:hidden + ellipsis, so measure that container. + const el = labelRef.current?.parentElement; + if (el) setIsClipped(el.scrollWidth > el.clientWidth); + }; + return ( + + {label}} + onClick={onClick} + onMouseEnter={checkClipped} + sx={sx} + /> + + ); +}; + const AgentWorkingOverlay: FC<{ message?: string; elapsed?: number; theme: Theme; onCancel?: () => void; color?: 'primary' | 'warning' }> = ({ message, elapsed, theme, onCancel, color = 'primary' }) => { const { t } = useTranslation(); // `message` is the running plan: steps joined by the STEP_SEP control char @@ -134,7 +166,8 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ const tables = useSelector((state: DataFormulatorState) => state.tables); const focusedId = useSelector((state: DataFormulatorState) => state.focusedId); const charts = useSelector(dfSelectors.getAllCharts); - const conceptShelfItems = useSelector((state: DataFormulatorState) => state.conceptShelfItems); + const starterQuestions = useSelector((state: DataFormulatorState) => state.starterQuestions); + const starterQuestionsStatus = useSelector((state: DataFormulatorState) => state.starterQuestionsStatus); const conceptShelfItems = useSelector((state: DataFormulatorState) => state.conceptShelfItems); const config = useSelector((state: DataFormulatorState) => state.config); const activeModel = useSelector(dfSelectors.getActiveModel); const workspaceBackend = useSelector((state: DataFormulatorState) => state.serverConfig.WORKSPACE_BACKEND); @@ -160,9 +193,11 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ // clears). Tracks the draftId we've already auto-submitted for. const clarifySubmittedRef = useRef(null); const [isChatFormulating, setIsChatFormulating] = useState(false); + // Whether the getting-started starter questions are collapsed (click the + // lightning bolt to expand/collapse). + const [starterCollapsed, setStarterCollapsed] = useState(false); const [mentionedTableIds, setMentionedTableIds] = useState([]); - const [mentionDropdownOpen, setMentionDropdownOpen] = useState(false); - const [mentionHighlightIdx, setMentionHighlightIdx] = useState(0); + const [mentionDropdownOpen, setMentionDropdownOpen] = useState(false); const [mentionHighlightIdx, setMentionHighlightIdx] = useState(0); const [attachedImages, setAttachedImages] = useState([]); const [attachedFiles, setAttachedFiles] = useState<{ name: string; content: string }[]>([]); const fileInputRef = useRef(null); @@ -198,6 +233,14 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ // Stale draft detection is handled by loadState in dfSlice (marks running/clarifying drafts as interrupted) const inputCardRef = useRef(null); + // Ref to the chat textarea so getting-started starter prompts can seed + // the input and immediately focus it for editing. + const chatInputRef = useRef(null); + + const seedChatPrompt = useCallback((text: string) => { + setChatPrompt(text); + requestAnimationFrame(() => chatInputRef.current?.focus()); + }, []); const generatedReports = useSelector((state: DataFormulatorState) => state.generatedReports); @@ -270,6 +313,35 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ setMentionHighlightIdx(0); }, [mentionFilter]); + // ── Starter-questions generation trigger ───────────────────────── + // Questions are stored per root table (each table has its own, plus an + // optional cross-table question). When a root table is focused and it has + // no fresh questions for the current table set, generate them lazily. The + // signature (all root table ids) refreshes questions when tables change; + // the 500ms debounce collapses batch loads into a single call. + const rootTableSignature = React.useMemo( + () => rootTables.map(t => t.id).sort().join('|'), + [rootTables] + ); + const focusedRootTableId = (focusedTableId && rootTables.some(t => t.id === focusedTableId)) + ? focusedTableId + : undefined; + React.useEffect(() => { + if (!focusedRootTableId) return; + const entry = starterQuestions[focusedRootTableId]; + if (entry && entry.signature === rootTableSignature) return; // already fresh + if (starterQuestionsStatus[focusedRootTableId] === 'loading') return; // in flight + const timer = setTimeout(() => { + dispatch(generateStarterQuestions({ + tableId: focusedRootTableId, + signature: rootTableSignature, + tableIds: rootTableSignature.split('|'), + })); + }, 500); + return () => clearTimeout(timer); + }, [focusedRootTableId, rootTableSignature, starterQuestions, starterQuestionsStatus, dispatch]); + + // Helper: confirm selection of a mention (table only) const confirmMention = (optionId: string) => { const tbl = tables.find(t => t.id === optionId); @@ -1799,6 +1871,7 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ inputLabel: { shrink: true }, input: { readOnly: isChatFormulating }, }} + inputRef={chatInputRef} value={chatPrompt} placeholder={ pendingClarification @@ -1920,8 +1993,96 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ ); + // ── Getting-started guidance ───────────────────────────────────── + // When a root table is focused, show a muted row of AI-generated starter + // questions tailored to that table (see the trigger effect above — each + // table has its own set, plus an optional cross-table question). Clicking + // a chip runs it; clicking the lightning bolt collapses/expands the row. + const focusedStarterEntry = focusedRootTableId ? starterQuestions[focusedRootTableId] : undefined; + const focusedStarterStatus = focusedRootTableId ? starterQuestionsStatus[focusedRootTableId] : undefined; + const focusedStarterFresh = !!focusedStarterEntry && focusedStarterEntry.signature === rootTableSignature; + const starterLoading = !!focusedRootTableId && (!focusedStarterFresh || focusedStarterStatus === 'loading'); + + const showGettingStarted = !!focusedRootTableId + && !isChatFormulating + && !pendingClarification + && (starterLoading || (focusedStarterEntry?.questions?.length ?? 0) > 0); + + const starterChipSx = { + height: 24, borderRadius: '6px', fontSize: 11, + color: 'text.secondary', + backgroundColor: 'transparent', + border: `1px solid ${borderColor.divider}`, + // Cap a single chip to one column width minus the toggle icon, and + // allow it to shrink (flex) when the row would otherwise overflow. + maxWidth: CARD_WIDTH - STARTER_ICON_WIDTH, + minWidth: 0, + flexShrink: 1, + '& .MuiChip-label': { + px: '8px', + overflow: 'hidden', + textOverflow: 'ellipsis', + whiteSpace: 'nowrap', + }, + '&:hover': { + backgroundColor: alpha(theme.palette.text.primary, 0.04), + color: 'text.primary', + borderColor: alpha(theme.palette.text.primary, 0.24), + }, + } as const; + + const gettingStartedBlock = showGettingStarted ? ( + + + setStarterCollapsed(c => !c)} + sx={{ + flexShrink: 0, + p: 0.5, borderRadius: '6px', color: 'text.disabled', + transition: 'background-color 0.15s, color 0.15s', + '&:hover': { color: 'text.secondary', backgroundColor: alpha(theme.palette.text.primary, 0.06) }, + }} + > + + + + {starterCollapsed && ( + setStarterCollapsed(false)} + sx={{ fontSize: 11, color: 'text.disabled', cursor: 'pointer', '&:hover': { color: 'text.secondary' } }} + > + {t('chartRec.expandStarters', { defaultValue: 'Show suggestions' })} + + )} + + {starterLoading + ? + : (focusedStarterEntry?.questions ?? []).map((q, i) => ( + submitChat(q)} + sx={starterChipSx} + /> + )) + } + + + ) : null; + return ( + {gettingStartedBlock} {/* The input box */} {inputBox} diff --git a/src/views/UnifiedDataUploadDialog.tsx b/src/views/UnifiedDataUploadDialog.tsx index c08dab82..32061b33 100644 --- a/src/views/UnifiedDataUploadDialog.tsx +++ b/src/views/UnifiedDataUploadDialog.tsx @@ -259,7 +259,7 @@ const SourcePill: React.FC = ({ justifyContent: 'center', width: 22, height: 22, - borderRadius: 0.75, + borderRadius: '50%', backgroundColor: alpha(theme.palette.primary.main, 0.08), flexShrink: 0, '& .MuiSvgIcon-root': { fontSize: 15 }, diff --git a/src/views/VisualizationView.tsx b/src/views/VisualizationView.tsx index e886b049..3161b094 100644 --- a/src/views/VisualizationView.tsx +++ b/src/views/VisualizationView.tsx @@ -532,6 +532,12 @@ export const ChartEditorFC: FC<{}> = function ChartEditorFC({}) { const [encodingOpen, setEncodingOpen] = useState(false); const editButtonRef = useRef(null); + // State for the compact action dock that sits below the chart-mode data + // table (mirrors the table-focus dock; replaces the grid's inline footer). + const [chartTableGridReport, setChartTableGridReport] = useState<{ loadedCount: number; rowCount: number; virtual: boolean; canRandomize: boolean; isRandom: boolean } | null>(null); + const [chartRandomizeToken, setChartRandomizeToken] = useState(0); + const [chartResetOrderToken, setChartResetOrderToken] = useState(0); + // Reset local UI state when focused chart changes useEffect(() => { setCodeDialogOpen(false); @@ -883,13 +889,13 @@ export const ChartEditorFC: FC<{}> = function ChartEditorFC({}) { {(() => { const ROW_HEIGHT = 25; const HEADER_HEIGHT = 32; - const FOOTER_HEIGHT = 32; - const MIN_TABLE_HEIGHT = 150; + const MIN_TABLE_HEIGHT = 60; const MAX_TABLE_HEIGHT = 400; const MIN_TABLE_WIDTH = 300; const MAX_TABLE_WIDTH = 900; const rowCount = table.virtual?.rowCount || table.rows?.length || 0; - const contentHeight = HEADER_HEIGHT + rowCount * ROW_HEIGHT + FOOTER_HEIGHT; + // Footer is hidden in chart mode (hideFooter), so don't reserve its height. + const contentHeight = HEADER_HEIGHT + rowCount * ROW_HEIGHT + 12; const adaptiveHeight = Math.max(MIN_TABLE_HEIGHT, Math.min(MAX_TABLE_HEIGHT, contentHeight)); // Estimate total width from columns (generous: account for type icons, sort arrows, padding) @@ -912,11 +918,27 @@ export const ChartEditorFC: FC<{}> = function ChartEditorFC({}) { const adaptiveWidth = Math.max(MIN_TABLE_WIDTH, Math.min(MAX_TABLE_WIDTH, totalColWidth + SCROLLBAR_WIDTH + 16)) + 34; return ( - - + + ); })()} + setChartRandomizeToken(x => x + 1)} + onResetOrder={() => setChartResetOrderToken(x => x + 1)} + /> ; })()} , @@ -1173,18 +1195,23 @@ const EmptyStateHero: FC<{ chartSelectionBox: React.ReactNode }> = ({ chartSelec // are highlighted this dock can summarize the selection and offer actions // that span them (e.g. "combine", "chart each"). const TableActionDock: FC<{ - chartSelectionBox: React.ReactNode; + chartSelectionBox?: React.ReactNode; tableId: string; tableName: string; rows: any[]; virtual: boolean; - gridReport?: { loadedCount: number; rowCount: number; virtual: boolean; canRandomize: boolean } | null; + // Compact variant (smaller paddings) used below the chart-mode data table. + compact?: boolean; + gridReport?: { loadedCount: number; rowCount: number; virtual: boolean; canRandomize: boolean; isRandom: boolean } | null; onRandomize?: () => void; -}> = ({ chartSelectionBox, tableId, tableName, rows, virtual, gridReport, onRandomize }) => { + onResetOrder?: () => void; +}> = ({ chartSelectionBox, tableId, tableName, rows, virtual, compact, gridReport, onRandomize, onResetOrder }) => { const { t } = useTranslation(); const [anchorEl, setAnchorEl] = useState(null); const [isDownloading, setIsDownloading] = useState(false); const open = Boolean(anchorEl); + // The Quick chart action only renders when a template picker is supplied. + const showQuickChart = !!chartSelectionBox; const handleDownload = async () => { if (isDownloading) return; @@ -1221,70 +1248,86 @@ const TableActionDock: FC<{ return ( - - + {showQuickChart && ( + <> + + + + )} - {gridReport?.virtual && ( - <> - - - {gridReport.loadedCount < gridReport.rowCount - ? t('dataGrid.loadedOfTotal', { loaded: gridReport.loadedCount, total: gridReport.rowCount }) - : t('dataGrid.rowCount', { count: gridReport.rowCount })} - - {gridReport.canRandomize && ( - - - - - - )} - + + + {gridReport?.virtual + ? (gridReport.loadedCount < gridReport.rowCount + ? t('dataGrid.loadedOfTotal', { loaded: gridReport.loadedCount, total: gridReport.rowCount }) + : t('dataGrid.rowCount', { count: gridReport.rowCount })) + : t('dataGrid.rowCount', { count: rows.length })} + + {gridReport?.virtual && gridReport.canRandomize && ( + + + + + )} - setAnchorEl(null)} - anchorOrigin={{ vertical: 'top', horizontal: 'left' }} - transformOrigin={{ vertical: 'bottom', horizontal: 'left' }} - slotProps={{ paper: { sx: { p: 2, maxWidth: 'min(90vw, 1140px)' } } }} - > - {/* Clicking a template creates the chart and this whole - empty-state unmounts; closing the popover here keeps - things tidy in the transient frame. */} - setAnchorEl(null)}> - {chartSelectionBox} - - + {showQuickChart && ( + setAnchorEl(null)} + anchorOrigin={{ vertical: 'top', horizontal: 'left' }} + transformOrigin={{ vertical: 'bottom', horizontal: 'left' }} + slotProps={{ paper: { sx: { p: 2, maxWidth: 'min(90vw, 1140px)' } } }} + > + {/* Clicking a template creates the chart and this whole + empty-state unmounts; closing the popover here keeps + things tidy in the transient frame. */} + setAnchorEl(null)}> + {chartSelectionBox} + + + )} ); }; @@ -1310,8 +1353,9 @@ export const VisualizationViewFC: FC = function VisualizationView // Virtual-pagination state reported up from the focused-table grid, so the // bottom toolbar can show the loaded/total count and drive the random dice. - const [tableGridReport, setTableGridReport] = React.useState<{ loadedCount: number; rowCount: number; virtual: boolean; canRandomize: boolean } | null>(null); + const [tableGridReport, setTableGridReport] = React.useState<{ loadedCount: number; rowCount: number; virtual: boolean; canRandomize: boolean; isRandom: boolean } | null>(null); const [tableRandomizeToken, setTableRandomizeToken] = React.useState(0); + const [tableResetOrderToken, setTableResetOrderToken] = React.useState(0); let focusedChart = allCharts.find(c => c.id == focusedChartId) as Chart; let synthesisRunning = focusedChartId ? chartSynthesisInProgress.includes(focusedChartId) : false; @@ -1444,7 +1488,7 @@ export const VisualizationViewFC: FC = function VisualizationView px: 3, pt: 2, pb: 2, boxSizing: 'border-box', }}> - + ); @@ -1460,6 +1504,7 @@ export const VisualizationViewFC: FC = function VisualizationView virtual={!!ft?.virtual} gridReport={tableGridReport} onRandomize={() => setTableRandomizeToken(x => x + 1)} + onResetOrder={() => setTableResetOrderToken(x => x + 1)} /> ); })()} From 1a6530d980969230af56ecf78f08fa450392b911 Mon Sep 17 00:00:00 2001 From: Chenglong Wang Date: Wed, 8 Jul 2026 22:29:39 -0700 Subject: [PATCH 36/40] fixes --- py-src/data_formulator/datalake/__init__.py | 4 - .../datalake/azure_blob_workspace.py | 223 +++-- .../datalake/blob_disk_cache.py | 238 ++++++ .../data_formulator/datalake/cache_manager.py | 372 -------- .../datalake/cached_azure_blob_workspace.py | 794 ------------------ pyproject.toml | 8 + .../backend/benchmarks/benchmark_workspace.py | 10 +- .../security/test_confined_dir_migration.py | 48 +- 8 files changed, 408 insertions(+), 1289 deletions(-) create mode 100644 py-src/data_formulator/datalake/blob_disk_cache.py delete mode 100644 py-src/data_formulator/datalake/cache_manager.py delete mode 100644 py-src/data_formulator/datalake/cached_azure_blob_workspace.py diff --git a/py-src/data_formulator/datalake/__init__.py b/py-src/data_formulator/datalake/__init__.py index 6199804a..1dc9a0cf 100644 --- a/py-src/data_formulator/datalake/__init__.py +++ b/py-src/data_formulator/datalake/__init__.py @@ -43,8 +43,6 @@ ) from data_formulator.datalake.workspace_manager import WorkspaceManager from data_formulator.datalake.azure_blob_workspace import AzureBlobWorkspace -from data_formulator.datalake.cached_azure_blob_workspace import CachedAzureBlobWorkspace -from data_formulator.datalake.cache_manager import GlobalCacheManager # Metadata types and operations from data_formulator.datalake.workspace_metadata import ( @@ -92,8 +90,6 @@ "Workspace", "WorkspaceWithTempData", "AzureBlobWorkspace", - "CachedAzureBlobWorkspace", - "GlobalCacheManager", "get_data_formulator_home", "get_default_workspace_root", "get_user_home", diff --git a/py-src/data_formulator/datalake/azure_blob_workspace.py b/py-src/data_formulator/datalake/azure_blob_workspace.py index 52fc9400..245a7fab 100644 --- a/py-src/data_formulator/datalake/azure_blob_workspace.py +++ b/py-src/data_formulator/datalake/azure_blob_workspace.py @@ -64,6 +64,21 @@ logger = logging.getLogger(__name__) +def _data_cache_ttl() -> float: + """Seconds a cached *data* blob may be served without re-validating. + + Metadata is always re-validated (TTL 0). Data blobs (parquet) are + effectively immutable per table version, so a small TTL lets rapid repeat + reads (agent tool loops, UI refreshes) skip Azure entirely. Override with + ``AZURE_BLOB_CACHE_TTL_SECONDS``. + """ + try: + return max(0.0, float(os.getenv("AZURE_BLOB_CACHE_TTL_SECONDS", "3"))) + except (TypeError, ValueError): + return 3.0 + + + class AzureBlobWorkspace(Workspace): """ Workspace backed by Azure Blob Storage. @@ -111,6 +126,7 @@ def __init__( # --- blob storage ---------------------------------------------------- self._container: ContainerClient = container_client + self._container_name = getattr(container_client, "container_name", "") or "" if blob_prefix is not None: # Direct prefix mode (used by AzureBlobWorkspaceManager) self._datalake_root = "" @@ -141,6 +157,13 @@ def __init__( scratch_base.mkdir(parents=True, exist_ok=True) self._scratch_dir = scratch_base self._confined_scratch = ConfinedDir(scratch_base, mkdir=False) + # Blob storage has no local workspace root, so ``confined_root`` + # (inherited from :class:`Workspace`, returns ``self._confined_root``) + # would otherwise be undefined. Point it at the same local scratch jail + # so agents that read/list from the workspace root (e.g. the + # data-loading chat's read_file/list_directory tools) operate on the + # local working dir instead of raising AttributeError. + self._confined_root = ConfinedDir(scratch_base, mkdir=False) # --- in-memory metadata cache ---------------------------------------- # Avoids re-downloading workspace.yaml on every method call. @@ -153,16 +176,22 @@ def __init__( self._metadata_lock = threading.Lock() # --- blob data cache ------------------------------------------------- - # Caches downloaded blob bytes keyed by filename. Avoids repeated - # downloads of the same data file within one request (e.g. - # analyze_table calls run_parquet_sql once per column, each of - # which would otherwise re-download the entire parquet blob). + # Request-local in-memory cache of downloaded blob bytes keyed by + # filename. Avoids repeated downloads of the same data file within one + # request (e.g. analyze_table calls run_parquet_sql once per column). + # Backed by the process-global :mod:`blob_disk_cache`, which persists + # bytes + ETag across the short-lived per-request workspace instances. # Invalidated per-file on upload/delete, cleared on cleanup. self._blob_data_cache: dict[str, bytes] = {} # --- metadata -------------------------------------------------------- - if not self._blob_exists(METADATA_FILENAME): - self._init_metadata() + # Skip the existence HEAD when the metadata blob is already cached on + # disk (warm container) — this runs on every request-scoped construction. + from data_formulator.datalake.blob_disk_cache import get_blob_disk_cache + + if get_blob_disk_cache().get(self._cache_key(METADATA_FILENAME)) is None: + if not self._blob_exists(METADATA_FILENAME): + self._init_metadata() logger.debug("Initialized AzureBlobWorkspace at %s", self._prefix) @@ -178,6 +207,10 @@ def _data_blob_key(self, filename: str) -> str: """Blob-internal key for a data file (under data/ subdirectory).""" return f"data/{filename}" + def _cache_key(self, filename: str) -> str: + """Globally-unique key for the disk cache: container + full blob name.""" + return f"{self._container_name}/{self._blob_name(filename)}" + def _get_blob(self, filename: str): """Return a ``BlobClient`` for *filename*.""" return self._container.get_blob_client(self._blob_name(filename)) @@ -193,25 +226,79 @@ def _blob_exists(self, filename: str) -> bool: def _upload_bytes( self, filename: str, data: bytes | str, *, overwrite: bool = True ) -> int: - """Upload *data* to blob. Returns size in bytes.""" + """Upload *data* to blob. Returns size in bytes. + + Write-through: the disk cache is updated with the new bytes + ETag so + this worker serves the fresh copy immediately without a round trip. + """ + from data_formulator.datalake.blob_disk_cache import get_blob_disk_cache + raw = data.encode("utf-8") if isinstance(data, str) else data - self._get_blob(filename).upload_blob(raw, overwrite=overwrite) - # Invalidate cached copy of this file + resp = self._get_blob(filename).upload_blob(raw, overwrite=overwrite) + cache = get_blob_disk_cache() + key = self._cache_key(filename) + etag = resp.get("etag") if isinstance(resp, dict) else None + if etag: + cache.put(key, raw, etag) + else: + cache.invalidate(key) + # Invalidate request-local copy of this file self._blob_data_cache.pop(filename, None) if hasattr(self, "_temp_file_cache") and filename in self._temp_file_cache: self._temp_file_cache.pop(filename).unlink(missing_ok=True) return len(raw) + def _ensure_cached(self, filename: str): + """Return a disk-cache :class:`CacheEntry` for *filename*, fetching or + re-validating from Azure only when necessary. + + - Fresh within TTL (data blobs only) → served from disk, no Azure call. + - Otherwise revalidate via a cheap ``get_blob_properties`` HEAD (no data + transfer): matching ETag → keep the cached bytes; changed ETag → full + download to refresh the cache. + - Cold cache → full download. + Raises ``ResourceNotFoundError`` if the blob does not exist. + + Note: we deliberately avoid a conditional ``download_blob`` here. The + ``StorageStreamDownloader`` injects an ``If-Match`` on multi-chunk + continuation requests, which combined with an ``If-None-Match`` makes + large (multi-chunk) downloads fail with ``ResourceModifiedError``. + Comparing ETags ourselves after a HEAD is equally cheap and robust. + """ + from data_formulator.datalake.blob_disk_cache import get_blob_disk_cache + + cache = get_blob_disk_cache() + key = self._cache_key(filename) + ttl = 0.0 if filename == METADATA_FILENAME else _data_cache_ttl() + blob = self._get_blob(filename) + + entry = cache.get(key) + if entry is not None: + if cache.is_fresh(key, ttl): + return entry + # Cheap revalidation: compare ETags via a HEAD (no data transfer). + if blob.get_blob_properties().etag == entry.etag: + cache.mark_validated(key) + return entry + + # Cold cache or changed blob — full download. + stream = blob.download_blob() + data = stream.readall() + return cache.put(key, data, stream.properties.etag) + def _download_bytes(self, filename: str) -> bytes: cached = self._blob_data_cache.get(filename) if cached is not None: return cached - data = self._get_blob(filename).download_blob().readall() + data = self._ensure_cached(filename).read_bytes() self._blob_data_cache[filename] = data return data def _delete_blob(self, filename: str) -> None: + from data_formulator.datalake.blob_disk_cache import get_blob_disk_cache + self._get_blob(filename).delete_blob() + get_blob_disk_cache().invalidate(self._cache_key(filename)) self._blob_data_cache.pop(filename, None) if hasattr(self, "_temp_file_cache") and filename in self._temp_file_cache: self._temp_file_cache.pop(filename).unlink(missing_ok=True) @@ -220,27 +307,14 @@ def _delete_blob(self, filename: str) -> None: def _temp_local_copy(self, filename: str): """Yield a local file path containing the blob's data. - The file is cached on disk for the lifetime of this workspace - instance so that repeated calls (e.g. ``run_parquet_sql`` once - per column in ``analyze_table``) don't re-write the temp file. - The cache is keyed by filename and cleaned up when the instance - is garbage-collected or when :meth:`cleanup` is called. + Backed by the process-global disk cache: the yielded path points at the + cached ``.bin`` file (ETag-validated against Azure), so repeated calls + across requests reuse the same local file with no re-download. Callers + only read the file (DuckDB / pyarrow), so sharing the cached path is + safe. """ - if not hasattr(self, "_temp_file_cache"): - self._temp_file_cache: dict[str, Path] = {} - - tmp_path = self._temp_file_cache.get(filename) - if tmp_path is None or not tmp_path.exists(): - data = self._download_bytes(filename) - suffix = Path(filename).suffix - tmp = tempfile.NamedTemporaryFile(suffix=suffix, delete=False) - tmp.write(data) - tmp.close() - tmp_path = Path(tmp.name) - self._temp_file_cache[filename] = tmp_path - - yield tmp_path - # Don't delete — reused across calls, cleaned up on GC / cleanup() + entry = self._ensure_cached(filename) + yield entry.path def _cleanup_temp_files(self) -> None: """Remove all cached temp files from disk.""" @@ -384,8 +458,12 @@ def _cleanup(m: WorkspaceMetadata) -> None: def cleanup(self) -> None: """Delete **all** blobs under this workspace's prefix.""" + from data_formulator.datalake.blob_disk_cache import get_blob_disk_cache + + cache = get_blob_disk_cache() for blob in self._container.list_blobs(name_starts_with=self._prefix): self._container.delete_blob(blob.name) + cache.invalidate(f"{self._container_name}/{blob.name}") self._metadata_cache = None self._blob_data_cache.clear() self._cleanup_temp_files() @@ -397,19 +475,25 @@ def cleanup(self) -> None: # ------------------------------------------------------------------ def read_data_as_df(self, table_name: str) -> pd.DataFrame: + from azure.core.exceptions import ResourceNotFoundError + meta = self.get_table_metadata(table_name) if meta is None: raise FileNotFoundError(f"Table not found: {table_name}") - if not self._blob_exists(self._data_blob_key(meta.filename)): + # Read straight from the ETag-validated disk cache path (no redundant + # existence HEAD, no full in-memory copy); the download validates + # existence and raises ResourceNotFoundError if the blob is gone. + try: + entry = self._ensure_cached(self._data_blob_key(meta.filename)) + except ResourceNotFoundError: raise FileNotFoundError(f"Blob not found: {meta.filename}") - buf = io.BytesIO(self._download_bytes(self._data_blob_key(meta.filename))) readers = { - "parquet": lambda b: pd.read_parquet(b), - "csv": lambda b: pd.read_csv(b), - "excel": lambda b: pd.read_excel(b), - "json": lambda b: pd.read_json(b), - "txt": lambda b: pd.read_csv(b, sep="\t"), + "parquet": lambda p: pd.read_parquet(p), + "csv": lambda p: pd.read_csv(p), + "excel": lambda p: pd.read_excel(p), + "json": lambda p: pd.read_json(p), + "txt": lambda p: pd.read_csv(p, sep="\t"), } reader = readers.get(meta.file_type) if reader is None: @@ -417,7 +501,7 @@ def read_data_as_df(self, table_name: str) -> pd.DataFrame: f"Unsupported file type '{meta.file_type}' for table '{table_name}'. " f"Supported: {', '.join(readers)}." ) - return reader(buf) + return reader(entry.path) # ------------------------------------------------------------------ # Parquet write @@ -531,31 +615,34 @@ def write_parquet( # ------------------------------------------------------------------ def get_parquet_schema(self, table_name: str) -> dict: + from azure.core.exceptions import ResourceNotFoundError + meta = self.get_table_metadata(table_name) if meta is None: raise FileNotFoundError(f"Table not found: {table_name}") if meta.file_type != "parquet": raise ValueError(f"Table {table_name} is not a parquet file") - if not self._blob_exists(self._data_blob_key(meta.filename)): - raise FileNotFoundError(f"Parquet blob not found: {meta.filename}") - with self._temp_local_copy(self._data_blob_key(meta.filename)) as tmp_path: - pf = pq.ParquetFile(tmp_path) - schema = pf.schema_arrow - return { - "table_name": table_name, - "filename": meta.filename, - "num_rows": pf.metadata.num_rows, - "num_columns": len(schema), - "columns": [ - {"name": f.name, "type": str(f.type), "nullable": f.nullable} - for f in schema - ], - "created_at": meta.created_at.isoformat(), - "last_synced": ( - meta.last_synced.isoformat() if meta.last_synced else None - ), - } + try: + entry = self._ensure_cached(self._data_blob_key(meta.filename)) + except ResourceNotFoundError: + raise FileNotFoundError(f"Parquet blob not found: {meta.filename}") + pf = pq.ParquetFile(entry.path) + schema = pf.schema_arrow + return { + "table_name": table_name, + "filename": meta.filename, + "num_rows": pf.metadata.num_rows, + "num_columns": len(schema), + "columns": [ + {"name": f.name, "type": str(f.type), "nullable": f.nullable} + for f in schema + ], + "created_at": meta.created_at.isoformat(), + "last_synced": ( + meta.last_synced.isoformat() if meta.last_synced else None + ), + } def get_parquet_path(self, table_name: str) -> str: # type: ignore[override] """Return the full blob name for the parquet file. @@ -580,25 +667,27 @@ def run_parquet_sql(self, table_name: str, sql: str) -> pd.DataFrame: the query, so DuckDB can use its native parquet reader. """ import duckdb + from azure.core.exceptions import ResourceNotFoundError meta = self.get_table_metadata(table_name) if meta is None: raise FileNotFoundError(f"Table not found: {table_name}") if meta.file_type != "parquet": raise ValueError(f"Table {table_name} is not a parquet file") - if not self._blob_exists(self._data_blob_key(meta.filename)): - raise FileNotFoundError(f"Parquet blob not found: {meta.filename}") if "{parquet}" not in sql: raise ValueError("SQL must contain {parquet} placeholder") - with self._temp_local_copy(self._data_blob_key(meta.filename)) as tmp_path: - escaped = str(tmp_path).replace("\\", "\\\\").replace("'", "''") - full_sql = sql.format(parquet=f"read_parquet('{escaped}')") - conn = duckdb.connect(":memory:") - try: - return conn.execute(full_sql).fetchdf() - finally: - conn.close() + try: + entry = self._ensure_cached(self._data_blob_key(meta.filename)) + except ResourceNotFoundError: + raise FileNotFoundError(f"Parquet blob not found: {meta.filename}") + escaped = str(entry.path).replace("\\", "\\\\").replace("'", "''") + full_sql = sql.format(parquet=f"read_parquet('{escaped}')") + conn = duckdb.connect(":memory:") + try: + return conn.execute(full_sql).fetchdf() + finally: + conn.close() # ------------------------------------------------------------------ # Local directory materialisation (for sandbox execution) diff --git a/py-src/data_formulator/datalake/blob_disk_cache.py b/py-src/data_formulator/datalake/blob_disk_cache.py new file mode 100644 index 00000000..c4090b31 --- /dev/null +++ b/py-src/data_formulator/datalake/blob_disk_cache.py @@ -0,0 +1,238 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Process-persistent, ETag-validated local disk cache for Azure Blob reads. + +Azure blob workspaces build a *fresh* :class:`AzureBlobWorkspace` on every +request, so their per-instance in-memory caches are always cold and every +request re-downloads ``workspace.yaml`` and data blobs (parquet files can be +many megabytes). This module provides a single, process-global cache that +survives across those short-lived instances (and across requests within a +worker container), backed by real files on local disk. + +Layout under ``/blob_cache/``:: + + .bin # the blob bytes + .meta.json # {"key", "blob_name", "container", "etag", "size", "cached_at"} + +The cache stores ``bytes + etag``. Freshness (whether a conditional GET is +needed) is tracked *in memory per process* via a monotonic timestamp, so we +never write to disk just to record a read. Callers (the workspace) decide the +TTL and issue conditional GETs; this module only stores/serves bytes and etags +and tracks last-validation times. + +Eviction is best-effort LRU by total bytes, capped by +``AZURE_BLOB_CACHE_MAX_BYTES`` (default 2 GiB). +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +import threading +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +from data_formulator.datalake.workspace import get_data_formulator_home + +logger = logging.getLogger(__name__) + +CACHE_DIR_NAME = "blob_cache" +_DEFAULT_MAX_BYTES = 2 * 1024 * 1024 * 1024 # 2 GiB + + +@dataclass +class CacheEntry: + """A cached blob: its bytes live at ``path``, validated by ``etag``.""" + + key: str + path: Path + etag: str + size: int + + def read_bytes(self) -> bytes: + return self.path.read_bytes() + + +class BlobDiskCache: + """Thread-safe on-disk cache of blob bytes keyed by ``container/blob_name``. + + Safe to share across threads. Multiple *processes* (gunicorn workers) + share the same directory; writes are atomic (temp + ``os.replace``) so a + concurrent reader never sees a half-written file. In-memory bookkeeping + (index, validation timestamps, total size) is per-process — that only + affects eviction accounting and TTL freshness, both of which are + best-effort and remain correct via ETag validation. + """ + + def __init__(self, root: Path, max_bytes: int = _DEFAULT_MAX_BYTES) -> None: + self._root = root + self._max_bytes = max_bytes + self._lock = threading.RLock() + self._index: dict[str, CacheEntry] = {} + self._validated_at: dict[str, float] = {} + self._total_bytes = 0 + self._root.mkdir(parents=True, exist_ok=True) + self._load_index() + + # ------------------------------------------------------------------ + # Paths / keys + # ------------------------------------------------------------------ + + @staticmethod + def _stem(key: str) -> str: + return hashlib.sha256(key.encode("utf-8")).hexdigest() + + def _bin_path(self, key: str) -> Path: + return self._root / f"{self._stem(key)}.bin" + + def _meta_path(self, key: str) -> Path: + return self._root / f"{self._stem(key)}.meta.json" + + # ------------------------------------------------------------------ + # Index bootstrap + # ------------------------------------------------------------------ + + def _load_index(self) -> None: + """Populate the in-memory index by scanning existing meta files.""" + try: + meta_files = list(self._root.glob("*.meta.json")) + except OSError: + return + for meta_file in meta_files: + try: + meta = json.loads(meta_file.read_text(encoding="utf-8")) + key = meta["key"] + bin_path = self._bin_path(key) + if not bin_path.exists(): + continue + size = int(meta.get("size", bin_path.stat().st_size)) + self._index[key] = CacheEntry( + key=key, path=bin_path, etag=meta["etag"], size=size + ) + self._total_bytes += size + except Exception: + logger.debug("blob cache: skipping bad meta %s", meta_file, exc_info=True) + + # ------------------------------------------------------------------ + # Read side + # ------------------------------------------------------------------ + + def get(self, key: str) -> Optional[CacheEntry]: + """Return the cached entry for *key*, or ``None`` if absent.""" + with self._lock: + entry = self._index.get(key) + if entry is not None and entry.path.exists(): + return entry + if entry is not None: + # bin vanished underneath us — drop the stale index record + self._drop_locked(key) + return None + + def is_fresh(self, key: str, ttl_seconds: float) -> bool: + """Whether *key* was validated within the last ``ttl_seconds``.""" + if ttl_seconds <= 0: + return False + with self._lock: + last = self._validated_at.get(key) + return last is not None and (time.monotonic() - last) < ttl_seconds + + def mark_validated(self, key: str) -> None: + """Record that *key* was just confirmed up-to-date against Azure.""" + with self._lock: + self._validated_at[key] = time.monotonic() + + # ------------------------------------------------------------------ + # Write side + # ------------------------------------------------------------------ + + def put(self, key: str, data: bytes, etag: str) -> CacheEntry: + """Store *data*/*etag* for *key* and return the resulting entry.""" + bin_path = self._bin_path(key) + meta_path = self._meta_path(key) + self._atomic_write(bin_path, data) + meta = { + "key": key, + "etag": etag, + "size": len(data), + "cached_at": time.time(), + } + self._atomic_write( + meta_path, json.dumps(meta, ensure_ascii=False).encode("utf-8") + ) + with self._lock: + old = self._index.get(key) + if old is not None: + self._total_bytes -= old.size + entry = CacheEntry(key=key, path=bin_path, etag=etag, size=len(data)) + self._index[key] = entry + self._total_bytes += entry.size + self._validated_at[key] = time.monotonic() + self._evict_if_needed_locked() + return entry + + def invalidate(self, key: str) -> None: + """Remove *key* from the cache (disk + memory).""" + with self._lock: + self._drop_locked(key) + + # ------------------------------------------------------------------ + # Internals + # ------------------------------------------------------------------ + + @staticmethod + def _atomic_write(path: Path, data: bytes) -> None: + tmp = path.with_name(f"{path.name}.{os.getpid()}.{threading.get_ident()}.tmp") + try: + tmp.write_bytes(data) + os.replace(tmp, path) + finally: + if tmp.exists(): + tmp.unlink(missing_ok=True) + + def _drop_locked(self, key: str) -> None: + entry = self._index.pop(key, None) + if entry is not None: + self._total_bytes -= entry.size + self._validated_at.pop(key, None) + self._bin_path(key).unlink(missing_ok=True) + self._meta_path(key).unlink(missing_ok=True) + + def _evict_if_needed_locked(self) -> None: + if self._total_bytes <= self._max_bytes: + return + # Evict least-recently-validated first; entries never validated this + # process (timestamp 0) go first. + candidates = sorted( + self._index.keys(), + key=lambda k: self._validated_at.get(k, 0.0), + ) + for key in candidates: + if self._total_bytes <= self._max_bytes: + break + self._drop_locked(key) + + +_cache_singleton: Optional[BlobDiskCache] = None +_singleton_lock = threading.Lock() + + +def get_blob_disk_cache() -> BlobDiskCache: + """Return the process-global :class:`BlobDiskCache` (created on first use).""" + global _cache_singleton + if _cache_singleton is None: + with _singleton_lock: + if _cache_singleton is None: + try: + max_bytes = int( + os.getenv("AZURE_BLOB_CACHE_MAX_BYTES", str(_DEFAULT_MAX_BYTES)) + ) + except ValueError: + max_bytes = _DEFAULT_MAX_BYTES + root = get_data_formulator_home() / CACHE_DIR_NAME + _cache_singleton = BlobDiskCache(root, max_bytes=max_bytes) + return _cache_singleton diff --git a/py-src/data_formulator/datalake/cache_manager.py b/py-src/data_formulator/datalake/cache_manager.py deleted file mode 100644 index b7cc8948..00000000 --- a/py-src/data_formulator/datalake/cache_manager.py +++ /dev/null @@ -1,372 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. - -""" -Global cache manager for multi-user deployments. - -While each :class:`CachedAzureBlobWorkspace` enforces its own per-workspace -limit (default 1 GB), this module enforces a **server-wide ceiling** across -ALL user caches, preventing the aggregate local cache from consuming all -available disk space when many users are active. - -Architecture -~~~~~~~~~~~~ -``GlobalCacheManager`` is a **thread-safe singleton**. The first call to -:meth:`get_instance` configures it (cache root, max bytes, scan interval); -subsequent calls return the same object. - -Cross-user eviction -~~~~~~~~~~~~~~~~~~~ -When the global limit is exceeded, files are evicted across *all* user -cache directories using **LRU by mtime** (oldest files first, regardless -of which user owns them). Protected files (``workspace.yaml``) are never -evicted. Eviction targets 80 % of the global max to avoid thrashing. - -Graceful degradation -~~~~~~~~~~~~~~~~~~~~ -When the global cache is full and eviction cannot free enough space, -individual workspaces fall through to direct Azure reads. User-initiated -writes always succeed locally (correctness), but read-path caching is -skipped so the disk isn't filled further. - -Disk scanning -~~~~~~~~~~~~~ -Total disk usage is computed via ``os.walk()`` over the cache root. -To avoid excessive I/O on servers with many files, the scan is -**debounced** — at most once per ``scan_interval`` seconds (default 10 s). -""" - -from __future__ import annotations - -import logging -import os -import threading -import time -from pathlib import Path -from typing import Any - -from data_formulator.datalake.workspace_metadata import METADATA_FILENAME - -logger = logging.getLogger(__name__) - -# Default global cache limit: 50 GB -_DEFAULT_GLOBAL_MAX_BYTES = 50 * 1024**3 - -# Default interval between filesystem scans (seconds) -_DEFAULT_SCAN_INTERVAL = 10.0 - - -class GlobalCacheManager: - """Thread-safe singleton managing total cache disk usage across all users. - - Usage:: - - mgr = GlobalCacheManager.get_instance( - cache_root=Path("~/.data_formulator/cache"), - max_global_bytes=10 * 1024**3, # 10 GB - ) - - # Before caching a downloaded file (optional write): - if mgr.try_acquire_space(len(data)): - cache_file.write_bytes(data) - - # After a mandatory write (e.g. user upload): - cache_file.write_bytes(data) - mgr.notify_write(len(data)) - - # Monitoring: - stats = mgr.get_global_stats() - """ - - _instance: GlobalCacheManager | None = None - _init_lock = threading.Lock() - - # ------------------------------------------------------------------ - # Singleton access - # ------------------------------------------------------------------ - - @classmethod - def get_instance( - cls, - cache_root: Path | str | None = None, - max_global_bytes: int = _DEFAULT_GLOBAL_MAX_BYTES, - scan_interval: float = _DEFAULT_SCAN_INTERVAL, - ) -> GlobalCacheManager: - """Return the singleton, creating it on first call. - - Args: - cache_root: Root of the local cache tree. Defaults to - ``~/.data_formulator/cache``. - max_global_bytes: Global ceiling in bytes (default 10 GB). - scan_interval: Min seconds between full filesystem scans - (default 10). - - Subsequent calls return the existing singleton — arguments are - ignored after the first call. - """ - if cls._instance is not None: - return cls._instance - with cls._init_lock: - if cls._instance is not None: - return cls._instance - if cache_root is None: - from data_formulator.datalake.workspace import ( - get_data_formulator_home, - ) - - cache_root = get_data_formulator_home() / "cache" - cls._instance = cls( - Path(cache_root), max_global_bytes, scan_interval - ) - return cls._instance - - @classmethod - def reset_instance(cls) -> None: - """Discard the singleton (for testing only).""" - with cls._init_lock: - cls._instance = None - - # ------------------------------------------------------------------ - # Construction (private — use get_instance) - # ------------------------------------------------------------------ - - def __init__( - self, - cache_root: Path, - max_global_bytes: int, - scan_interval: float, - ): - self._cache_root = cache_root - self._max_global_bytes = max_global_bytes - self._scan_interval = scan_interval - - self._lock = threading.Lock() - self._last_scan_time: float = 0.0 - self._cached_total_bytes: int = 0 - - logger.info( - "GlobalCacheManager: root=%s max=%d MB scan_interval=%.1fs", - cache_root, - max_global_bytes // (1024 * 1024), - scan_interval, - ) - - # ------------------------------------------------------------------ - # Properties - # ------------------------------------------------------------------ - - @property - def max_global_bytes(self) -> int: - return self._max_global_bytes - - @property - def cache_root(self) -> Path: - return self._cache_root - - # ------------------------------------------------------------------ - # Disk scanning (debounced) - # ------------------------------------------------------------------ - - def _scan_total_size(self) -> int: - """Walk the cache root and sum file sizes. - - Debounced: returns the cached value if the last scan was less - than ``scan_interval`` seconds ago. - - **Must be called with ``self._lock`` held.** - """ - now = time.monotonic() - if now - self._last_scan_time < self._scan_interval: - return self._cached_total_bytes - - total = 0 - try: - for dirpath, _dirnames, filenames in os.walk(self._cache_root): - for fn in filenames: - try: - total += os.path.getsize(os.path.join(dirpath, fn)) - except OSError: - pass - except OSError: - pass - - self._cached_total_bytes = total - self._last_scan_time = now - return total - - # ------------------------------------------------------------------ - # Space management - # ------------------------------------------------------------------ - - def try_acquire_space(self, needed_bytes: int) -> bool: - """Try to make room for *needed_bytes* of new cache data. - - 1. If total + needed is under the limit, return ``True``. - 2. Otherwise, run cross-user LRU eviction. - 3. If still insufficient, return ``False`` (caller should skip - local caching and serve from Azure directly). - - This is intended for **optional** cache writes (e.g. caching a - download). For **mandatory** writes (user uploads), use - :meth:`notify_write` after writing instead. - """ - with self._lock: - total = self._scan_total_size() - if total + needed_bytes <= self._max_global_bytes: - self._cached_total_bytes = total + needed_bytes - return True - - # Try cross-user eviction - freed = self._evict_global_unlocked( - target_free=needed_bytes + int(self._max_global_bytes * 0.1) - ) - - # Re-scan after eviction - if freed > 0: - self._last_scan_time = 0.0 # force fresh scan - total = self._scan_total_size() - if total + needed_bytes <= self._max_global_bytes: - self._cached_total_bytes = total + needed_bytes - return True - - return False - - def notify_write(self, nbytes: int) -> None: - """Notify the manager of a mandatory write (e.g. user upload). - - The write has already happened. This bumps the cached counter - and triggers global eviction if the limit is exceeded. - """ - with self._lock: - self._cached_total_bytes += nbytes - if self._cached_total_bytes > self._max_global_bytes: - self._evict_global_unlocked( - target_free=int( - self._cached_total_bytes - - self._max_global_bytes * 0.8 - ) - ) - - def maybe_evict_global(self) -> int: - """Run cross-user eviction if total exceeds the global limit. - - Returns bytes freed. - """ - with self._lock: - total = self._scan_total_size() - if total <= self._max_global_bytes: - return 0 - return self._evict_global_unlocked( - target_free=int(total - self._max_global_bytes * 0.8) - ) - - # ------------------------------------------------------------------ - # Cross-user LRU eviction (internal) - # ------------------------------------------------------------------ - - def _evict_global_unlocked(self, target_free: int) -> int: - """Evict files across all user caches, LRU by mtime. - - **Must be called with ``self._lock`` held.** - - Skips: - * ``workspace.yaml`` — correctness-critical metadata. - * Hidden files (starting with ``.``). - - Args: - target_free: Bytes to free. - - Returns: - Total bytes actually freed. - """ - if target_free <= 0: - return 0 - - # Collect all candidate files across all user caches - candidates: list[tuple[str, float, int]] = [] - try: - for dirpath, _, filenames in os.walk(self._cache_root): - for fn in filenames: - if fn == METADATA_FILENAME: - continue - if fn.startswith("."): - continue - full = os.path.join(dirpath, fn) - try: - st = os.stat(full) - candidates.append((full, st.st_mtime, st.st_size)) - except OSError: - pass - except OSError: - return 0 - - # Sort by mtime ascending (oldest first = evict first) - candidates.sort(key=lambda x: x[1]) - - freed = 0 - evicted = 0 - for full_path, _mtime, size in candidates: - if freed >= target_free: - break - try: - os.unlink(full_path) - freed += size - evicted += 1 - except OSError: - pass - - if evicted: - logger.info( - "Global cache eviction: removed %d file(s), freed %.1f MB " - "(target was %.1f MB)", - evicted, - freed / (1024 * 1024), - target_free / (1024 * 1024), - ) - # Invalidate cached scan so next check is accurate - self._last_scan_time = 0.0 - - return freed - - # ------------------------------------------------------------------ - # Monitoring - # ------------------------------------------------------------------ - - def get_global_stats(self) -> dict[str, Any]: - """Return global cache statistics for monitoring / debugging.""" - with self._lock: - total = self._scan_total_size() - - # Count user-level cache directories (root/datalake_root/user_id/) - user_dirs = 0 - try: - for entry in os.scandir(self._cache_root): - if entry.is_dir(): - for sub in os.scandir(entry.path): - if sub.is_dir(): - user_dirs += 1 - except OSError: - pass - - return { - "cache_root": str(self._cache_root), - "total_size_bytes": total, - "total_size_mb": round(total / (1024 * 1024), 2), - "max_size_mb": round(self._max_global_bytes / (1024 * 1024), 2), - "utilization_pct": ( - round(total / self._max_global_bytes * 100, 1) - if self._max_global_bytes > 0 - else 0 - ), - "user_cache_count": user_dirs, - } - - # ------------------------------------------------------------------ - # Representation - # ------------------------------------------------------------------ - - def __repr__(self) -> str: - return ( - f"GlobalCacheManager(root={self._cache_root!r}, " - f"max={self._max_global_bytes // (1024**2)} MB)" - ) diff --git a/py-src/data_formulator/datalake/cached_azure_blob_workspace.py b/py-src/data_formulator/datalake/cached_azure_blob_workspace.py deleted file mode 100644 index 3a121bf4..00000000 --- a/py-src/data_formulator/datalake/cached_azure_blob_workspace.py +++ /dev/null @@ -1,794 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. - -""" -Cached Azure Blob workspace with persistent local file mirror. - -Wraps :class:`AzureBlobWorkspace` with a **write-through local cache** -under ``~/.data_formulator/cache/``. Reads come from the local mirror -(filesystem speed), writes go to the local mirror immediately and are -uploaded to Azure Blob Storage in a background thread. - -Key performance improvements over plain ``AzureBlobWorkspace``: - -* ``read_data_as_df()`` — reads local parquet directly (no blob download) -* ``local_dir()`` — yields the cache directory (no temp-dir downloads) -* ``run_parquet_sql()`` — runs DuckDB against local parquet (no temp copy) -* ``write_parquet()`` — writes local file immediately, Azure upload in bg - -Cache eviction --------------- -An LRU eviction mechanism prevents unbounded disk growth: - -* **Max size** — configurable, default 1 GB per workspace. -* **Trigger** — checked after every write. -* **Policy** — evict least-recently-used files (oldest ``mtime``) until - total cache size drops to 80 % of the max. -* **Protected** — ``workspace.yaml`` and files with pending background - uploads are never evicted. - -Usage:: - - from azure.storage.blob import ContainerClient - from data_formulator.datalake.cached_azure_blob_workspace import ( - CachedAzureBlobWorkspace, - ) - - container = ContainerClient.from_connection_string(conn_str, "my-container") - ws = CachedAzureBlobWorkspace( - "user:42", container, - datalake_root="workspaces", - max_cache_bytes=2 * 1024**3, # 2 GB cache - ) -""" - -from __future__ import annotations - -import atexit -import collections -import io -import logging -import os -import shutil -import threading -import time -from concurrent.futures import Future, ThreadPoolExecutor -from contextlib import contextmanager -from datetime import datetime, timezone -from pathlib import Path -from typing import Any, Optional, TYPE_CHECKING - -import pandas as pd -import pyarrow as pa -import pyarrow.parquet as pq -import yaml - -from data_formulator.datalake.azure_blob_workspace import AzureBlobWorkspace -from data_formulator.datalake.cache_manager import GlobalCacheManager -from data_formulator.datalake.workspace_metadata import ( - METADATA_FILENAME, - WorkspaceMetadata, -) -from data_formulator.datalake.parquet_utils import sanitize_table_name -from data_formulator.datalake.workspace import Workspace, get_data_formulator_home -from data_formulator.security.path_safety import ConfinedDir - -if TYPE_CHECKING: - from azure.storage.blob import ContainerClient - -logger = logging.getLogger(__name__) - -# Default cache size limit per workspace (1 GB). -_DEFAULT_MAX_CACHE_BYTES = 1 * 1024 ** 3 - - -class CachedAzureBlobWorkspace(AzureBlobWorkspace): - """Azure Blob workspace with a persistent local file cache. - - Every file written to Azure is also written to a local cache directory. - Reads are served from the local cache whenever possible — falling back - to Azure only for files that have been evicted or written by another - process. - - Background uploads - ~~~~~~~~~~~~~~~~~~ - Data-file uploads are submitted to a :class:`ThreadPoolExecutor` so - they don't block the request thread. Metadata (``workspace.yaml``) - is always uploaded **synchronously** because it is small and - correctness-critical. Call :meth:`wait_for_uploads` to block until - all pending uploads finish (e.g. before shutdown or tests). - - Multi-instance safety - ~~~~~~~~~~~~~~~~~~~~~~ - When the same user is served by multiple server instances (e.g. - behind a load balancer), each instance has its own local cache. - Stale-cache detection compares the local ``workspace.yaml``'s - ``updated_at`` timestamp against Azure on a configurable interval - (default: 30 s). If Azure is newer, the local metadata and any - changed data files are re-downloaded. - - Global cache budget - ~~~~~~~~~~~~~~~~~~~~ - A :class:`GlobalCacheManager` singleton enforces a server-wide - ceiling (default 10 GB) across **all** user caches. When the - global limit is exceeded, cross-user LRU eviction removes the - oldest files server-wide. If eviction cannot free enough space, - download-path caching is skipped (graceful degradation) so reads - fall through to Azure Blob Storage directly. - - Thread safety - ~~~~~~~~~~~~~ - The local cache directory is per-user so there are no cross-user - conflicts. Per-file upload locks prevent concurrent background - uploads from racing on the same blob. The ``_pending_uploads`` - set is protected by a global lock. - """ - - # ------------------------------------------------------------------ - # Construction - # ------------------------------------------------------------------ - - def __init__( - self, - identity_id: str, - container_client: "ContainerClient", - datalake_root: str = "", - *, - blob_prefix: str | None = None, - cache_root: str | Path | None = None, - max_cache_bytes: int = _DEFAULT_MAX_CACHE_BYTES, - max_global_cache_bytes: int | None = None, - bg_upload_workers: int = 2, - staleness_check_interval: float = 30.0, - ): - """ - Args: - identity_id: Unique user identifier. - container_client: Azure ``ContainerClient``. - datalake_root: Path prefix inside the blob container. - cache_root: Root of the local cache tree. Defaults to - ``~/.data_formulator/cache``. - max_cache_bytes: Maximum total size of cached files for this - workspace before LRU eviction kicks in. - max_global_cache_bytes: Server-wide ceiling across **all** - user caches (default: 10 GB). ``None`` uses the - :class:`GlobalCacheManager` default. - bg_upload_workers: Thread pool size for background uploads. - staleness_check_interval: Seconds between Azure metadata - freshness checks (default: 30). Set to 0 to check on - every ``get_metadata()`` call. - """ - # We must set up the cache directory *before* calling - # super().__init__ because the parent's __init__ may call - # _upload_bytes (via _init_metadata / save_metadata). - safe_id = Workspace._sanitize_identity_id(identity_id) - - if cache_root is None: - cache_root = get_data_formulator_home() / "cache" - base = Path(cache_root) - - if blob_prefix is not None: - # Multi-workspace mode: cache dir based on blob prefix - safe_prefix = blob_prefix.strip("/").replace("/", os.sep) - self._cache_dir = base / safe_prefix - else: - root = datalake_root.strip("/") - if root: - self._cache_dir = base / root / safe_id - else: - self._cache_dir = base / safe_id - self._cache_dir.mkdir(parents=True, exist_ok=True) - self._cache_jail = ConfinedDir(self._cache_dir, mkdir=False) - - self._max_cache_bytes = max_cache_bytes - - # Background upload machinery - self._pending_uploads: set[str] = set() - self._upload_lock = threading.Lock() - # Per-file locks to serialise concurrent uploads to the same blob - self._file_locks: dict[str, threading.Lock] = collections.defaultdict( - threading.Lock - ) - self._upload_executor = ThreadPoolExecutor( - max_workers=bg_upload_workers, - thread_name_prefix="df_cache_upload", - ) - self._upload_futures: list[Future] = [] - # Register atexit hook to flush pending uploads on interpreter exit - atexit.register(self._atexit_flush) - - # Staleness detection for multi-instance deployments - self._staleness_check_interval = staleness_check_interval - self._last_staleness_check: float = 0.0 # epoch seconds - self._local_metadata_updated_at: Optional[datetime] = None - - # Initialise the global cache manager (singleton) — must be - # before super().__init__ because _upload_bytes references it. - gcm_kwargs: dict[str, Any] = {"cache_root": base} - if max_global_cache_bytes is not None: - gcm_kwargs["max_global_bytes"] = max_global_cache_bytes - self._global_cache = GlobalCacheManager.get_instance(**gcm_kwargs) - - # Now safe to call super().__init__ — our _upload_bytes / etc. - # overrides are in place and _cache_dir is ready. - super().__init__(identity_id, container_client, datalake_root, blob_prefix=blob_prefix) - - # Run initial eviction if cache is over-sized (e.g. from prev run) - self._maybe_evict() - - logger.info( - "Initialized CachedAzureBlobWorkspace: prefix=%s cache=%s " - "max=%d MB global_max=%d MB", - self._prefix, - self._cache_dir, - self._max_cache_bytes // (1024 * 1024), - self._global_cache.max_global_bytes // (1024 * 1024), - ) - - # ------------------------------------------------------------------ - # Cache path helper - # ------------------------------------------------------------------ - - def _cache_path(self, filename: str) -> Path: - """Return the local cache path for *filename*. - - Delegates to :class:`ConfinedDir` which raises ``ValueError`` - if the resolved path escapes the cache directory. - """ - return self._cache_jail.resolve(filename) - - # ------------------------------------------------------------------ - # Low-level blob overrides (write-through cache) - # ------------------------------------------------------------------ - - def _upload_bytes( - self, - filename: str, - data: bytes | str, - *, - overwrite: bool = True, - ) -> int: - """Write *data* to local cache **and** Azure. - - Metadata (``workspace.yaml``) is uploaded synchronously. - Data files are uploaded in a background thread. - """ - raw = data.encode("utf-8") if isinstance(data, str) else data - - # 1. Write to local cache immediately - cache_file = self._cache_path(filename) - cache_file.parent.mkdir(parents=True, exist_ok=True) - cache_file.write_bytes(raw) - - # 2. Invalidate in-memory caches (inherited from AzureBlobWorkspace) - self._blob_data_cache.pop(filename, None) - if hasattr(self, "_temp_file_cache") and filename in self._temp_file_cache: - self._temp_file_cache.pop(filename).unlink(missing_ok=True) - - # 3. Upload to Azure - if filename == METADATA_FILENAME: - # Metadata: synchronous (small, correctness-critical) - self._get_blob(filename).upload_blob(raw, overwrite=overwrite) - else: - # Data files: background upload with per-file lock - with self._upload_lock: - self._pending_uploads.add(filename) - - def _bg_upload(fn: str, payload: bytes) -> None: - # Per-file lock ensures concurrent writes to the same - # blob are serialised — last write always wins. - file_lock = self._file_locks[fn] - file_lock.acquire() - try: - self._get_blob(fn).upload_blob(payload, overwrite=True) - logger.debug("Background upload complete: %s", fn) - except Exception: - logger.warning( - "Background upload FAILED for %s — data is safe in " - "local cache and will be retried on next write.", - fn, - exc_info=True, - ) - finally: - file_lock.release() - with self._upload_lock: - self._pending_uploads.discard(fn) - - fut = self._upload_executor.submit(_bg_upload, filename, raw) - self._upload_futures.append(fut) - - # 4. Evict if needed (per-workspace, then global) - self._maybe_evict() - # Notify global manager of the mandatory write - self._global_cache.notify_write(len(raw)) - - return len(raw) - - def _download_bytes(self, filename: str) -> bytes: - """Read from local cache first, then Azure, then populate cache.""" - # 1. Local cache hit - cache_file = self._cache_path(filename) - if cache_file.exists(): - data = cache_file.read_bytes() - # Touch to update mtime for LRU tracking - try: - os.utime(cache_file, None) - except OSError: - pass - # Also populate in-memory cache for extra speed on hot paths - self._blob_data_cache[filename] = data - return data - - # 2. In-memory cache (shouldn't happen if local cache is warm) - cached = self._blob_data_cache.get(filename) - if cached is not None: - # Backfill local cache — only if global budget allows - if self._global_cache.try_acquire_space(len(cached)): - try: - cache_file.parent.mkdir(parents=True, exist_ok=True) - cache_file.write_bytes(cached) - except OSError: - logger.debug("Failed to backfill cache for %s", filename) - return cached - - # 3. Azure download (cache miss — evicted or written by other instance) - data = self._get_blob(filename).download_blob().readall() - self._blob_data_cache[filename] = data - - # Persist to local cache only if global budget allows - if self._global_cache.try_acquire_space(len(data)): - try: - cache_file.parent.mkdir(parents=True, exist_ok=True) - cache_file.write_bytes(data) - except OSError: - logger.debug( - "Global cache full — serving %s from Azure without " - "local caching", - filename, - ) - else: - logger.debug( - "Global cache full — serving %s from Azure without " - "local caching", - filename, - ) - return data - - def _blob_exists(self, filename: str) -> bool: - """Check local cache first, then Azure.""" - if self._cache_path(filename).exists(): - return True - # Fall back to Azure (file may have been evicted) - return super()._blob_exists(filename) - - # ------------------------------------------------------------------ - # Staleness detection (multi-instance safety) - # ------------------------------------------------------------------ - - def _check_staleness(self) -> None: - """Compare local metadata timestamp with Azure's. - - If Azure has a newer ``updated_at``, invalidate local metadata - and any data files whose table metadata has changed. - Called by ``get_metadata()`` at most once per - ``staleness_check_interval``. - """ - now = time.monotonic() - if now - self._last_staleness_check < self._staleness_check_interval: - return - self._last_staleness_check = now - - try: - # Fetch fresh metadata from Azure (bypass all caches) - raw = self._get_blob(METADATA_FILENAME).download_blob().readall() - parsed = yaml.safe_load(raw) - if parsed is None: - return - remote_meta = WorkspaceMetadata.from_dict(parsed) - except Exception: - # If Azure is unreachable, use local cache silently - logger.debug("Staleness check: Azure unreachable, using local cache") - return - - local_meta = self._metadata_cache - if local_meta is None: - # No local metadata cached yet — will be loaded fresh anyway - return - - if remote_meta.updated_at <= local_meta.updated_at: - return # local is up to date - - logger.info( - "Stale cache detected: local=%s remote=%s — refreshing", - local_meta.updated_at.isoformat(), - remote_meta.updated_at.isoformat(), - ) - - # Find data files that changed (different hash or new tables) - for table_name, remote_table in remote_meta.tables.items(): - local_table = local_meta.tables.get(table_name) - if ( - local_table is None - or local_table.content_hash != remote_table.content_hash - ): - # Invalidate cached file so next read re-downloads - self._cache_path(remote_table.filename).unlink(missing_ok=True) - self._blob_data_cache.pop(remote_table.filename, None) - logger.debug("Invalidated stale cached file: %s", remote_table.filename) - - # Find tables deleted remotely - for table_name in list(local_meta.tables.keys()): - if table_name not in remote_meta.tables: - old_fn = local_meta.tables[table_name].filename - self._cache_path(old_fn).unlink(missing_ok=True) - self._blob_data_cache.pop(old_fn, None) - - # Update local metadata cache file and in-memory cache - self._cache_path(METADATA_FILENAME).write_bytes(raw) - self._metadata_cache = remote_meta - self._blob_data_cache[METADATA_FILENAME] = raw - - def _delete_blob(self, filename: str) -> None: - """Delete from local cache, in-memory caches, and Azure.""" - # Local cache - self._cache_path(filename).unlink(missing_ok=True) - - # In-memory caches - self._blob_data_cache.pop(filename, None) - if hasattr(self, "_temp_file_cache") and filename in self._temp_file_cache: - self._temp_file_cache.pop(filename).unlink(missing_ok=True) - - # Azure - try: - self._get_blob(filename).delete_blob() - except Exception: - logger.debug("Azure delete failed for %s (may not exist)", filename) - - # ------------------------------------------------------------------ - # Temp-local-copy override (use cache file directly) - # ------------------------------------------------------------------ - - # ------------------------------------------------------------------ - # Metadata override with staleness check - # ------------------------------------------------------------------ - - def get_metadata(self) -> WorkspaceMetadata: - """Return workspace metadata, checking Azure for staleness.""" - self._check_staleness() - return super().get_metadata() - - # ------------------------------------------------------------------ - # Temp-local-copy override (use cache file directly) - # ------------------------------------------------------------------ - - @contextmanager - def _temp_local_copy(self, filename: str): - """Yield the local cache path directly — no temp file needed.""" - cache_file = self._cache_path(filename) - if not cache_file.exists(): - # Ensure file is in cache - self._download_bytes(filename) - yield cache_file - - # ------------------------------------------------------------------ - # Read overrides (read directly from local cache files) - # ------------------------------------------------------------------ - - def read_data_as_df(self, table_name: str) -> pd.DataFrame: - """Read table from local cache (fast!) with Azure fallback.""" - meta = self.get_table_metadata(table_name) - if meta is None: - raise FileNotFoundError(f"Table not found: {table_name}") - - cache_file = self._cache_path(meta.filename) - - # Ensure file is in cache - if not cache_file.exists(): - # Download from Azure and populate cache - self._download_bytes(meta.filename) - - # Update mtime for LRU - try: - os.utime(cache_file, None) - except OSError: - pass - - # Read directly from local file — fastest path - readers = { - "parquet": lambda p: pd.read_parquet(p), - "csv": lambda p: pd.read_csv(p), - "excel": lambda p: pd.read_excel(p), - "json": lambda p: pd.read_json(p), - "txt": lambda p: pd.read_csv(p, sep="\t"), - } - reader = readers.get(meta.file_type) - if reader is None: - raise ValueError( - f"Unsupported file type '{meta.file_type}' for table " - f"'{table_name}'. Supported: {', '.join(readers)}." - ) - return reader(cache_file) - - def run_parquet_sql(self, table_name: str, sql: str) -> pd.DataFrame: - """Run DuckDB SQL against local cache file (no temp copy needed).""" - import duckdb - - meta = self.get_table_metadata(table_name) - if meta is None: - raise FileNotFoundError(f"Table not found: {table_name}") - if meta.file_type != "parquet": - raise ValueError(f"Table {table_name} is not a parquet file") - if "{parquet}" not in sql: - raise ValueError("SQL must contain {parquet} placeholder") - - cache_file = self._cache_path(meta.filename) - if not cache_file.exists(): - self._download_bytes(meta.filename) - - # Update mtime for LRU - try: - os.utime(cache_file, None) - except OSError: - pass - - escaped = str(cache_file).replace("\\", "\\\\").replace("'", "''") - full_sql = sql.format(parquet=f"read_parquet('{escaped}')") - conn = duckdb.connect(":memory:") - try: - return conn.execute(full_sql).fetchdf() - finally: - conn.close() - - # ------------------------------------------------------------------ - # local_dir — the biggest win - # ------------------------------------------------------------------ - - @contextmanager - def local_dir(self): - """Yield the cache directory — no temp dir, no mass downloads. - - Verifies that all workspace data files are present in the local - cache before yielding. Any missing files (evicted or written by - another instance) are downloaded on demand. - """ - self._ensure_all_cached() - yield self._cache_dir - - def _ensure_all_cached(self) -> None: - """Download any workspace files not present in local cache.""" - for blob in self._container.list_blobs(name_starts_with=self._prefix): - rel = blob.name[len(self._prefix) :] - if not rel or rel == METADATA_FILENAME: - continue - cache_file = self._cache_path(rel) - if not cache_file.exists(): - data = self._container.download_blob(blob.name).readall() - cache_file.parent.mkdir(parents=True, exist_ok=True) - cache_file.write_bytes(data) - logger.debug("local_dir: downloaded missing file %s", rel) - - # ------------------------------------------------------------------ - # Cache eviction (LRU by mtime) - # ------------------------------------------------------------------ - - def _get_cache_size(self) -> int: - """Total bytes of all files in the cache directory.""" - total = 0 - for f in self._cache_dir.iterdir(): - if f.is_file(): - try: - total += f.stat().st_size - except OSError: - pass - return total - - def _maybe_evict(self) -> None: - """Evict LRU files if cache exceeds ``max_cache_bytes``. - - Evicts down to 80 % of max. Never evicts ``workspace.yaml`` - or files with pending background uploads. - """ - total = self._get_cache_size() - if total <= self._max_cache_bytes: - return - - target = int(self._max_cache_bytes * 0.8) - - # Collect eviction candidates (sorted oldest mtime first) - candidates: list[tuple[Path, float, int]] = [] - with self._upload_lock: - pending = set(self._pending_uploads) - - for f in self._cache_dir.iterdir(): - if not f.is_file(): - continue - if f.name == METADATA_FILENAME: - continue # never evict metadata - if f.name in pending: - continue # never evict files being uploaded - try: - st = f.stat() - candidates.append((f, st.st_mtime, st.st_size)) - except OSError: - pass - - # Sort by mtime ascending (oldest first = evict first) - candidates.sort(key=lambda x: x[1]) - - evicted = 0 - freed = 0 - for path, mtime, size in candidates: - if total <= target: - break - try: - path.unlink(missing_ok=True) - total -= size - freed += size - evicted += 1 - except OSError: - pass - - if evicted: - logger.info( - "Cache eviction: removed %d file(s), freed %.1f MB, " - "remaining %.1f / %.1f MB", - evicted, - freed / (1024 * 1024), - total / (1024 * 1024), - self._max_cache_bytes / (1024 * 1024), - ) - - # Also run global cross-user eviction if needed - self._global_cache.maybe_evict_global() - - def get_cache_stats(self) -> dict[str, Any]: - """Return cache statistics for monitoring / debugging.""" - files = [] - total_size = 0 - for f in self._cache_dir.iterdir(): - if f.is_file(): - try: - st = f.stat() - files.append({"name": f.name, "size": st.st_size, "mtime": st.st_mtime}) - total_size += st.st_size - except OSError: - pass - - with self._upload_lock: - pending = list(self._pending_uploads) - - return { - "cache_dir": str(self._cache_dir), - "file_count": len(files), - "total_size_bytes": total_size, - "total_size_mb": round(total_size / (1024 * 1024), 2), - "max_size_mb": round(self._max_cache_bytes / (1024 * 1024), 2), - "utilization_pct": round(total_size / self._max_cache_bytes * 100, 1) - if self._max_cache_bytes > 0 - else 0, - "pending_uploads": pending, - "files": sorted(files, key=lambda f: f["mtime"], reverse=True), - } - - # ------------------------------------------------------------------ - # Lifecycle - # ------------------------------------------------------------------ - - def wait_for_uploads(self, timeout: float | None = 30) -> bool: - """Block until all pending background uploads complete. - - Args: - timeout: Max seconds to wait. ``None`` = wait forever. - - Returns: - ``True`` if all uploads finished, ``False`` if timeout was hit. - """ - # Collect outstanding futures - futures = [f for f in self._upload_futures if not f.done()] - if not futures: - return True - - from concurrent.futures import wait, FIRST_EXCEPTION - - done, not_done = wait(futures, timeout=timeout) - # Clean up completed futures - self._upload_futures = [f for f in self._upload_futures if not f.done()] - - if not_done: - logger.warning( - "%d background upload(s) did not complete within %.1fs", - len(not_done), - timeout or 0, - ) - return False - return True - - def _atexit_flush(self) -> None: - """Best-effort flush of pending uploads on interpreter shutdown.""" - with self._upload_lock: - if not self._pending_uploads: - return - pending_count = len(self._pending_uploads) - - logger.info("Flushing %d pending upload(s) on shutdown...", pending_count) - self.wait_for_uploads(timeout=30) - - def cleanup(self) -> None: - """Remove local cache immediately, delete Azure blobs in background. - - The local cache is cleared **synchronously** so the workspace is - immediately reusable. Azure blob deletion is submitted to the - background thread pool so the caller isn't blocked by many - sequential Azure API calls. - """ - # 1. Wait for any in-flight uploads (so we don't race with them) - self.wait_for_uploads(timeout=60) - - # 2. Clear local caches immediately (non-blocking) - if self._cache_dir.exists(): - shutil.rmtree(self._cache_dir, ignore_errors=True) - self._cache_dir.mkdir(parents=True, exist_ok=True) - self._metadata_cache = None - self._blob_data_cache.clear() - self._cleanup_temp_files() - self._cleanup_scratch() - - # 3. Delete Azure blobs in background - prefix = self._prefix - container = self._container - - def _bg_cleanup() -> None: - try: - for blob in container.list_blobs(name_starts_with=prefix): - try: - container.delete_blob(blob.name) - except Exception: - logger.debug("Failed to delete blob %s", blob.name) - logger.info("Background cleanup finished for %s", prefix) - except Exception: - logger.warning( - "Background Azure cleanup failed for %s", - prefix, - exc_info=True, - ) - - self._upload_executor.submit(_bg_cleanup) - logger.info("Cleanup: local cache cleared, Azure deletion queued for %s", self._safe_id) - - # ------------------------------------------------------------------ - # snapshot / session overrides: ensure cache consistency - # ------------------------------------------------------------------ - - def restore_workspace_snapshot(self, src: Path) -> None: - """Restore snapshot and repopulate local cache.""" - # Clear local cache first - if self._cache_dir.exists(): - shutil.rmtree(self._cache_dir, ignore_errors=True) - self._cache_dir.mkdir(parents=True, exist_ok=True) - - # Delegate to parent (uploads blobs to Azure) - super().restore_workspace_snapshot(src) - - # Repopulate cache from the source snapshot - if src.exists(): - for f in src.rglob("*"): - if f.is_file(): - rel = str(f.relative_to(src)) - cache_file = self._cache_path(rel) - cache_file.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(f, cache_file) - - def invalidate_metadata_cache(self) -> None: - """Force re-read of metadata from Azure (clears local + in-memory).""" - self._cache_path(METADATA_FILENAME).unlink(missing_ok=True) - super().invalidate_metadata_cache() - - # ------------------------------------------------------------------ - # Representation - # ------------------------------------------------------------------ - - def __repr__(self) -> str: - return ( - f"CachedAzureBlobWorkspace(identity_id={self._identity_id!r}, " - f"prefix={self._prefix!r}, cache={self._cache_dir!r})" - ) diff --git a/pyproject.toml b/pyproject.toml index 780e8f98..87254ce2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,6 +67,14 @@ Repository = "https://github.com/microsoft/data-formulator.git" package-dir = {"" = "py-src"} include-package-data = true +# Non-Python resources that must ship inside the installed package. The Azure +# build runs `pip install .` from a zip (no VCS), so include-package-data alone +# does not pick these up — declare them explicitly. Analyst skills load their +# `SKILL.md` body and `tools.json` schemas at runtime from the package dir. +# Scoped to the exact skill filenames so unrelated data JSON is not bundled. +[tool.setuptools.package-data] +"*" = ["SKILL.md", "tools.json"] + [project.scripts] data_formulator = "data_formulator:run_app" diff --git a/tests/backend/benchmarks/benchmark_workspace.py b/tests/backend/benchmarks/benchmark_workspace.py index ed271deb..83c26936 100644 --- a/tests/backend/benchmarks/benchmark_workspace.py +++ b/tests/backend/benchmarks/benchmark_workspace.py @@ -308,7 +308,7 @@ def print_report(all_results, latency_ms=None): print(" * full_derive_data -- the two above combined (dominates latency)") print(" * warm cache -- blob_data_cache avoids re-downloads for reads") print(" * local_dir always bypasses the blob_data_cache") - print(" * CachedAzureBlobWorkspace keeps a local mirror => local_dir is free") + print(" * blob_disk_cache keeps an ETag-validated local copy => reads avoid re-download") print() @@ -418,11 +418,11 @@ def _run_simulated(args, all_results): except Exception: pass - # -- 5. CachedAzureBlobWorkspace simulation ------------------------------ - # The CachedAzureBlobWorkspace uses a LOCAL file mirror so reads - # are at filesystem speed. We simulate it here by wrapping the + # -- 5. Local-mirror cache simulation ------------------------------------ + # AzureBlobWorkspace + blob_disk_cache keeps an ETag-validated local copy + # so reads are at filesystem speed. We simulate it here by wrapping the # SimulatedBlobWorkspace with the same write-through-to-cache pattern. - print(f"\n[5/5] CachedAzureBlobWorkspace pattern (local mirror)") + print(f"\n[5/5] Local-mirror cache pattern (blob_disk_cache)") with tempfile.TemporaryDirectory(prefix="df_bench_cached_") as tmpdir: ws_cached = Workspace("bench_cached", root_dir=tmpdir) all_results["Cached Azure"] = run_benchmark( diff --git a/tests/backend/security/test_confined_dir_migration.py b/tests/backend/security/test_confined_dir_migration.py index 8a620669..0ce69d61 100644 --- a/tests/backend/security/test_confined_dir_migration.py +++ b/tests/backend/security/test_confined_dir_migration.py @@ -4,7 +4,7 @@ """Regression tests for Phase 5 ConfinedDir migration. Verifies that Workspace.confined_* properties, agent tool path safety, -scratch route path safety, and CachedAzureBlobWorkspace._cache_path all +and scratch route path safety all correctly delegate to ConfinedDir after the migration from hand-written resolve+relative_to checks. """ @@ -162,52 +162,6 @@ def test_preview_scratch_traversal_blocked(self, agent, workspace_path): assert "error" in actions[0] -# ── CachedAzureBlobWorkspace._cache_path uses ConfinedDir ────────────── - - -class TestCachedAzureBlobCachePath: - - def test_cache_path_rejects_traversal(self, tmp_path): - from data_formulator.datalake.cached_azure_blob_workspace import ( - CachedAzureBlobWorkspace, - ) - jail = ConfinedDir(tmp_path / "cache", mkdir=True) - - instance = CachedAzureBlobWorkspace.__new__(CachedAzureBlobWorkspace) - instance._cache_dir = jail.root - instance._cache_jail = jail - - with pytest.raises(ValueError): - instance._cache_path("../../etc/passwd") - - def test_cache_path_normal_file(self, tmp_path): - from data_formulator.datalake.cached_azure_blob_workspace import ( - CachedAzureBlobWorkspace, - ) - cache_dir = tmp_path / "cache" - jail = ConfinedDir(cache_dir, mkdir=True) - - instance = CachedAzureBlobWorkspace.__new__(CachedAzureBlobWorkspace) - instance._cache_dir = jail.root - instance._cache_jail = jail - - result = instance._cache_path("data.parquet") - assert result == (cache_dir / "data.parquet").resolve() - - def test_cache_path_absolute_path_rejected(self, tmp_path): - from data_formulator.datalake.cached_azure_blob_workspace import ( - CachedAzureBlobWorkspace, - ) - jail = ConfinedDir(tmp_path / "cache", mkdir=True) - - instance = CachedAzureBlobWorkspace.__new__(CachedAzureBlobWorkspace) - instance._cache_dir = jail.root - instance._cache_jail = jail - - with pytest.raises(ValueError): - instance._cache_path("/etc/passwd") - - # ── Scratch routes use workspace.confined_scratch ─────────────────────── From eb293e44e431c4ee15a4d6dc57a7a6cbfbe6c17f Mon Sep 17 00:00:00 2001 From: Chenglong Wang Date: Thu, 9 Jul 2026 21:32:16 -0700 Subject: [PATCH 37/40] minor cleanup --- .../37-connector-agent-tooling-interface.md | 211 ------------------ 1 file changed, 211 deletions(-) delete mode 100644 design-docs/37-connector-agent-tooling-interface.md diff --git a/design-docs/37-connector-agent-tooling-interface.md b/design-docs/37-connector-agent-tooling-interface.md deleted file mode 100644 index a0b2c0b1..00000000 --- a/design-docs/37-connector-agent-tooling-interface.md +++ /dev/null @@ -1,211 +0,0 @@ -# Connector-Level Agent Tooling Interface - -Status: **draft** — discussion doc, refining before implementation. -Owner: @chenwang -Scope: -- [py-src/data_formulator/data_loader/external_data_loader.py](../py-src/data_formulator/data_loader/external_data_loader.py) (the loader interface) -- [py-src/data_formulator/agents/agent_data_loading_chat.py](../py-src/data_formulator/agents/agent_data_loading_chat.py) (tools) -- [py-src/data_formulator/agents/context.py](../py-src/data_formulator/agents/context.py) (`handle_read_catalog_metadata` and friends) -- [py-src/data_formulator/data_connector.py](../py-src/data_formulator/data_connector.py) (live loader resolution) - -Related: [32-data-loading-agent-navigation.md](32-data-loading-agent-navigation.md) (find/browse/inspect tools — cache-only), [13-unified-source-filters-plan.md](13-unified-source-filters-plan.md) (source-agnostic filter vocabulary), [14-unified-source-metadata-plan.md](14-unified-source-metadata-plan.md), [22-data-source-metadata-survey.md](22-data-source-metadata-survey.md). - ---- - -## 1. Motivation - -The data-loading agent stalls on large tables. Real symptom, verbatim from a session: - -> Found the three requested Kusto tables under SampleMetrics. SQLServerLocation is tiny and safe to load fully. The two metrics tables are very large; **their column metadata is not synced, so I cannot safely add column filters or aggregations yet.** This plan loads only the platform-limited subset for exploration, not the full tables. - -The agent did the right thing given what it can see — but what it can see is a **stale, column-less cache snapshot**. It has no way to look at the live source, so it can't build the filter/aggregation that would let a big table load safely. We route oversized tables into the chat precisely so the agent can narrow them down (design 25 / the size-routing work), and then the agent has no tool to actually do it. - -The fix we want is **not** per-connector prompt tuning or one-off Kusto hacks. We want the agent to have **generic, connector-level tooling to interrogate any source**, so: - -- A new loader gets useful agent behavior "for free" from base-class defaults. -- No loader needs bespoke agent glue or instance-specific tuning. -- The agent reasons about *any* source (SQL, Kusto, Mongo, Superset, blob, …) through one stable vocabulary. - -This doc proposes what that interface looks like. - ---- - -## 2. Where we are today - -### 2.1 The loader interface already has live read methods - -`ExternalDataLoader` is richer than the agent uses. Existing (mostly live) methods: - -| Method | Live? | Returns | -|---|---|---| -| `list_tables(filter)` / `ls(path, filter)` | live | table enumeration | -| `catalog_hierarchy()` / `effective_hierarchy()` / `pinned_scope()` | pure | navigation shape | -| `get_metadata(path)` | **live** | columns, types, `row_count`, `sample_rows` | -| `get_column_types(source_table)` | **live** | source column types (widget hints) | -| `get_column_values(source_table, column, keyword, limit, offset)` | **live** | distinct values (default: empty) | -| `fetch_data_as_arrow(source_table, import_options)` | **live** | bounded rows w/ `columns`, `filters`, `source_filters`, `sort`, `size` | - -### 2.2 The agent can't reach any of them - -The three discovery tools are **cache-only** (design 32 made this an explicit non-goal: "No live connector calls during browse"): - -- `list_data` / `find_data` → read `catalog_cache/.json`. -- `describe_data` → [`handle_read_catalog_metadata`](../py-src/data_formulator/agents/context.py) just loads the cache entry and formats it. It never calls `get_metadata`. - -`execute_python` runs in a sandbox with **no connector access** (pandas/duckdb over `scratch/` files only). So the agent's only live interaction with a source is indirect and after-the-fact. - -### 2.3 Why the cache is column-less for the failing case - -Cluster-wide Kusto browse (no pinned DB) runs `list_tables(fetch_columns=False)` on purpose — the bulk `.show database schema as json` per database across a whole cluster times out. So cluster-wide cache entries carry `row_count` + description but **no columns**. `describe_data` faithfully reports "not synced". This is the correct perf tradeoff for *browsing*; it's the wrong data for *the agent deciding how to filter*. - -### 2.4 The load boundary: no raw queries — on purpose - -`fetch_data_as_arrow` is deliberately constrained: - -> Only source_table is supported (no raw query strings) to avoid security and dialect diversity issues across loaders. - -This is a core principle we should preserve. **"Give the agent direct DB access" cannot mean "let the LLM emit raw SQL/KQL".** It must mean "give the agent a rich, structured, read-only capability vocabulary that each loader compiles to its own dialect." - ---- - -## 3. Design principles - -1. **Structured, not raw.** The agent expresses intent as structured operations (filter, project, group, aggregate, count, distinct, profile). Loaders compile to SQL/KQL/pipeline. No LLM-authored query text ever reaches a driver. -2. **Connector-level & generic.** Capabilities live on `ExternalDataLoader`. The agent never special-cases a source. One tool surface for all loaders. -3. **Native-first execution; shared building blocks, not a lowest-common-denominator default.** Each loader runs a probe the way its backend does it best — DB systems compile the query to their dialect and push it down to the source engine; file/object systems read-and-compute in an embedded engine (DuckDB) over the actual files. The base class supplies *reusable strategy building blocks* (a shared SQL compiler, a DuckDB read-and-compute helper) so a loader **picks a strategy** rather than hand-rolling one, and any loader in a known family (SQL, file) is useful for free. A bounded-sample fallback exists for the rare loader that can do neither, but it is explicit and advertises `approximate` — it is **never** the silent default. We do **not** ship the old "pull a local copy of N rows and re-aggregate a sample" behaviour as the universal path: it is approximate for large tables and wastes bandwidth pulling raw rows a real backend could have aggregated itself. -4. **Read-only & bounded.** Every capability is read-only, row/time/byte-capped, and time-limited. Reuse the `notruncation`-on-bounded-fetch pattern; never unbounded. -5. **Exactness is self-describing.** Every probe result carries an `exact` flag, so the agent (and UI) knows whether a number is source-exact or a bounded-sample approximation without a separate capability-negotiation step. -6. **Stay agentic — no cache write-back.** Probe/describe results live in the agent's conversation context and drive the current turn's reasoning; they are **not** persisted back into `catalog_cache`. The cache stays owned by the browse/sync path (design 14/32); the agent just reads live when it needs more than the cache holds. This keeps the two concerns cleanly separated and avoids merge/staleness conflicts. - ---- - -## 4. Proposed interface: `describe` + `probe` - -Two capabilities on `ExternalDataLoader`, both keyed by catalog `path` (the same list `get_metadata` already takes): - -- **`describe(path)`** — schema/metadata (columns, types, `row_count`, a small `sample_rows`). Not a data query; it's the Phase-1 unblocker. -- **`probe(path, query)`** — a single bounded read expressed as a restricted **single-table Select–Project–Aggregate query** (SPJ *minus the join* — see §4.2). One function, one compiler per loader, covering count / distinct / sample / aggregate as degenerate shapes of the same object. - -### 4.1 `describe(path)` -Live schema + sample. Thin wrapper over `get_metadata(path)` returning columns, types, `row_count`, and a small `sample_rows`. When the cache lacks columns (the reported failure), the agent calls this and gets the real schema back. - -### 4.2 `probe(path, query)` — the SPJQ query object -A structured, read-only query over **one table**. It speaks the source-agnostic filter vocabulary from design 13 and expresses select (σ), project (π), and aggregate (γ) — deliberately **no join**. That "single-table only" restraint is what keeps it compilable across every backend and stops it degenerating into a raw-query hole. - -```jsonc -{ - "path": ["db", "table"], // ONE table — no joins, no subqueries - "filters": [{"column": "...", "op": "EQ|NEQ|GT|GTE|LT|LTE|IN|ILIKE|BETWEEN|IS_NULL", "value": ...}], // σ - "columns": ["colA", "colB"], // π (projection; omit = all) - "group_by": ["region"], // γ keys - "aggregates": [{"op": "count|count_distinct|sum|avg|min|max", "column": "...", "as": "..."}], // γ metrics - "order_by": [{"column": "...", "dir": "asc|desc"}], - "limit": 100 // bounded, hard-capped -} -``` - -Everything the agent needs is a degenerate shape of this one object: - -| Intent | Query shape | -|---|---| -| **sample** rows | no `group_by`, no `aggregates`, `limit=N` | -| **count** | `aggregates=[{op:count}]`, no `group_by` | -| **distinct values** (+ frequency) | `group_by=[c]`, `aggregates=[{op:count}]`, `order_by` desc | -| **profile / aggregate** | `group_by=[region]`, `aggregates=[sum(x), min(ts), max(ts)]` | -| **date range** | `aggregates=[{op:min,column:ts}, {op:max,column:ts}]` | - -The **same** SPJQ object compiles to each backend's native form; **§5** details the per-backend execution strategies (native pushdown for DBs, read-and-compute for files, an explicit sample fallback). - -Result payload: `{rows, columns, row_count?, exact: bool, compiled_note?}` — `exact=false` flags the rare sampled/approximate answer (Strategy C, §5.3). - -`limit` is always **hard-capped server-side** (e.g. `min(limit, MAX_PROBE_ROWS)`) — not for correctness but to protect the agent's context window. We deliberately never return a large table for the LLM to parse, because those rows become input tokens on the next turn. The result message **always** states the cap so the agent knows it is looking at a capped preview, e.g. *"showing 50 of N rows (capped at 50 — use aggregate shapes or the load plan for the full table)"*. When `group_by`/`aggregates` reduce the result below the cap the note simply confirms the full grouped result fit; when the cap trims raw rows it also sets `exact=false`. - ---- - -## 5. Execution strategies — native-first, per backend - -A probe is **always compiled and executed against the backend that owns the data** — never by pulling a "local copy" of raw rows back to the app and re-aggregating a sample. `describe` stays a thin live wrapper over `get_metadata` (native `information_schema` / `.show schema` where a loader overrides it). `probe` runs with one of three strategies; the first two are exact and are the expected path, the third is an explicit, advertised fallback. - -### 5.1 Strategy A — Native pushdown (DB systems) - -Postgres, MySQL, MSSQL, BigQuery, Athena, **Kusto**, Mongo. The loader compiles the SPJQ object to its own dialect and executes it **on the source engine**: - -- SQL family → `SELECT … WHERE … GROUP BY … ORDER BY … LIMIT` (shared compiler, dialect-specific quoting; alongside the existing `build_*_where_clause*` helpers). -- Kusto → `T | where … | summarize … by … | order by … | take …`. -- Mongo → single-collection pipeline `$match / $group / $sort / $limit`. - -The source filters/groups/aggregates over the **whole** table using its own indexes and returns only the small result. Always `exact=true`. This is the path that fixes the large-Kusto-table motivation: a `summarize … by …` over the full table instead of a sample of the first N rows. - -### 5.2 Strategy B — Read-and-compute (file / object systems) - -Local folder, S3/blob, and other file-backed sources (parquet/csv/json). There's no query engine on the source, so "native" here means **hand the file(s) to an embedded analytical engine (DuckDB) and run the compiled SQL there** — letting DuckDB scan the file directly (`read_parquet('…')` / `read_csv_auto('…')`) with predicate & projection pushdown into the scan. Reading the file *is* the native operation for these sources, so this is **exact**, not a sample. Because DuckDB speaks SQL, the **same compiler as Strategy A** produces the query — only the `FROM` target differs (`read_parquet('…')` vs. a table name). This is the preferred implementation for file sources: DuckDB is faster and more correct than hand-rolling scans/aggregation in Python. - -### 5.3 Strategy C — Bounded-sample (explicit fallback only) - -For the rare loader that can neither push down nor cheaply read-and-compute (e.g. a thin API loader exposing only `fetch_data_as_arrow`), the base class offers `_probe_via_sample`: fetch a bounded set of rows and compute the SPJQ over that sample in DuckDB. This is `exact=false`, and the loader must **opt in** by overriding `probe` to call it. It is **not** a silent default — we add a proper native strategy in preference to leaning on it. The `exact` flag travels back to the agent so it can phrase a sampled number honestly. - -### 5.4 Shared building blocks - -So "each backend instantiates the probe arguments to best do computation" — but they don't each reinvent it: - -- **SQL compiler** (one place): SPJQ → SQL with parameterized/escaped values and dialect quoting. Reused by every SQL DB (Strategy A) *and* the DuckDB read-and-compute path (Strategy B). This is the big reuse — one compiler, two families. -- **DuckDB engine helper**: point DuckDB at a loader's file(s), run the shared-compiled SQL, return Arrow. Serves the whole file/object family. -- **Kusto & Mongo** bring their own compilers (KQL / aggregation pipeline) but reuse the same SPJQ object and result payload. - -A new SQL or file loader gets a working, exact probe by wiring into the matching mixin; only a genuinely new *kind* of backend needs a new compiler. - -### 5.5 No separate capability negotiation - -Earlier drafts advertised an `agent_capabilities()` map (`pushdown`/`approximate`/`unavailable`). That is redundant: the probe result already carries `exact` (true for Strategy A/B, false for the Strategy C sample), and a loader that hasn't opted into any strategy returns an `{error}` from the base `probe`. The agent reads the `exact` flag to caveat approximate numbers and treats an error as "probe unavailable, fall back to describe / the load plan" — no extra negotiation method to keep in sync. `probe` stays a concrete base method (not `@abstractmethod`) so third-party plugin loaders that predate it keep working; the default simply reports the source unsupported. - ---- - -## 6. Agent integration - -1. **Live loader resolution inside a tool call.** The agent stream runs inside the Flask request context, so `DataConnector._get_identity()` + `load_connectors(identity)` + the registry lookup (mirroring [`_resolve_connector_with_key`](../py-src/data_formulator/data_connector.py)) can turn `source_id → live loader` mid-turn. Factor a small helper so the agent/context layer isn't cache-only anymore. -2. **Two tools** thin-wrap the capabilities: `describe_data` (gains a live fallback — auto when the cached entry has no columns, capped to table nodes) and `probe_data(path, query)` (the single SPJQ probe). One query tool means fewer ways for the LLM to mis-select. Both the tool description and every result message make the row cap explicit — `probe_data` returns *at most N rows* and exists for inspection/reasoning, not bulk loading — so the agent understands it will never get a huge table dumped into its context and routes full-data needs through the load plan instead. -3. **Stay agentic — results are in-context only.** `describe`/`probe` results feed the agent's reasoning for the current turn; we do **not** write them back into `catalog_cache`. If a later turn needs fresh schema again it simply re-describes (cheap, bounded, budget-capped). The cache remains owned by the browse/sync path, so there's no merge-with-`sync_catalog_metadata` problem to solve. -4. **Prompt update.** Teach the probe→plan loop: `describe_data` for schema, `probe_data` (count / distinct / aggregate shapes) to size the slice and pick real filter values, then `propose_load_plan`. Reinforce "never guess columns/values — probe first" (the prompt already says this for the cache path). -5. **Shared query object with the load path.** `probe` *without* `group_by`/`aggregates` is the same select-project-limit shape `fetch_data_as_arrow` / `ingest_to_workspace` already consume via `import_options`. Converging them on one query object means the agent probes and loads with the *same* structure, and there's one compiler per loader serving both — a net simplification, not just a new tool. - ---- - -## 7. Security & guardrails - -- **Read-only.** No DDL/DML/writes; capabilities compile only to read queries. The raw-query ban stays. -- **Bounded everything.** Row caps, byte caps (`notruncation` only on a bounded shape), and per-call timeouts. Reuse `MAX_IMPORT_ROWS` conventions. -- **Per-turn probe budget.** Cap probe calls per turn to avoid a chatty agent hammering the source; return a "budget exhausted" signal. -- **Serialize non-parallel-safe clients.** Kusto's client mutates `self.kusto_database` per query — probes must serialize like batch import does. -- **Identity scoping.** Loader resolution goes through the same per-identity registry as imports; no cross-user leakage. -- **Injection surface.** Because the agent supplies *values*, not query text, values flow through the existing parameterized/escaped builders. Column/table identifiers validated against the (freshly probed) schema, not free-form. - ---- - -## 8. Phasing - -- **Phase 1 (unblock):** live `describe_data` fallback (`get_metadata`) — auto-fires when a table node's cached columns are missing. Fixes the reported failure with the smallest surface. Low risk, high value. -- **Phase 2 (probe — exact strategies first):** ship `probe_data` with the two exact strategies — the shared SQL compiler + Strategy A pushdown for the SQL family, and Strategy B read-and-compute (DuckDB over the files) for file/object sources. These cover the common cases correctly; no approximate default. -- **Phase 3 (remaining DB natives + fallback + prompt loop):** Kusto/Mongo `probe` compilers (KQL / pipeline) for exact source-side aggregation; the explicit bounded-sample fallback (Strategy C) for any loader that can do neither; probe→plan prompt loop. -- **Phase 4 (polish):** UI reuse of the probe `exact` flag for smart filter widgets; profile caching; approximate→exact upgrade hints. - ---- - -## 9. Open questions - -1. ~~**Tool granularity for the LLM:** one polymorphic tool vs. several named tools?~~ **Resolved:** one `probe_data` tool taking the SPJQ query object, plus `describe_data` for schema. Degenerate shapes cover count / distinct / sample / aggregate, so there's nothing extra to select between. -2. ~~**Auto vs. explicit live fallback in `describe_data`:** fire `get_metadata` automatically whenever columns are missing vs. require `live=true`?~~ **Resolved:** always auto — `describe_data` fires `get_metadata` automatically whenever a table node's cached columns are missing (one query per described table), no `live` flag. The per-turn probe budget (§7) caps runaway calls. -3. ~~**Where does the local-compute default run?** Loader process vs. the agent's DuckDB sandbox.~~ **Resolved — reframed:** there is no universal local-compute *default*. DB systems don't local-compute at all (Strategy A pushes down); file sources read-and-compute **inside the loader** via DuckDB over the actual files (Strategy B), and the rare sample fallback (Strategy C) also runs loader-side. Compute never happens in the agent's `scratch/` sandbox — connector concerns stay in the loader. -4. ~~**Cache write-back conflicts:** how to merge probed columns with a later full `sync_catalog_metadata`?~~ **Resolved — dropped:** we stay agentic and do not write probe results back to the cache (§3.6 / §6.3), so there's no merge conflict to manage. -5. ~~**Cost controls for metered sources** (BigQuery/Athena bytes-scanned): should `probe` require a dry-run byte estimate and refuse above a threshold?~~ **Resolved — not for now.** No dry-run byte estimate in v1; the row/time/byte caps (§7) already bound each probe. Revisit if a metered source proves expensive in practice. -6. ~~**Do we ever want a genuine escape hatch** (a vetted, read-only, sandboxed raw-query capability), gated behind a flag?~~ **Resolved — no.** No raw-query escape hatch, not even flagged. It reintroduces the raw-query risk §3.1 forbids, multiplies the per-dialect sandbox + cost-control burden (each backend has different ways to smuggle side effects / blow up cost), and isn't needed: `probe` covers single-table reads and joins/expressions go through DuckDB `execute_python` (§10). A genuine gap the structured vocabulary can't express is closed by adding a *typed* operator to `probe` (e.g. the date-bucketing follow-up), never by opening a raw hole. - ---- - -## 10. Non-goals - -- Raw LLM-authored SQL/KQL to drivers (violates §3.1). **No escape hatch, not even flagged** (§9.6): the per-dialect read-only-sandbox + cost-control burden outweighs the benefit, and typed `probe` operators + DuckDB `execute_python` cover the real needs. -- Reworking the post-load Data Agent (analysis side) — this is the loading agent only. -- Write/ingest changes — `ingest_to_workspace` / `fetch_data_as_arrow` load path is unchanged; `probe` is a read-only sibling that shares the same query shape. -- **Joins / multi-table probes** — `probe` is single-table by design (the "J" is dropped on purpose). Cross-table work is done by loading both tables and joining in DuckDB via `execute_python`. -- **Computed expressions / raw fragments in `probe` v1** — only bare columns + the fixed aggregate ops. (Date-bucketing like `bin(ts, 1d)` / `date_trunc` is the most likely early follow-up, added as a structured field, never as raw text.) -- Persistent agent "cwd" state — calls stay stateless with explicit `(source_id, path)` (consistent with design 32). From 86b4f72722bb1e19ed45bea4c2749b4fd8d67ca8 Mon Sep 17 00:00:00 2001 From: Chenglong Wang Date: Thu, 9 Jul 2026 23:43:02 -0700 Subject: [PATCH 38/40] checkpoint --- .../agents/agent_data_loading_chat.py | 22 +- .../analyst/skills/core/SKILL.md | 7 +- .../analyst/skills/core/skill.py | 25 +- .../analyst/skills/core/tools.json | 11 +- py-src/data_formulator/data_connector.py | 34 +++ .../data_loader/external_data_loader.py | 15 ++ .../data_loader/kusto_data_loader.py | 89 +++---- src/app/dfSlice.tsx | 26 ++ src/app/utils.tsx | 1 + src/components/ComponentType.tsx | 1 + src/components/LoadPlanCard.tsx | 104 ++++---- src/components/TablePreviewRow.tsx | 35 ++- src/components/filterFormat.ts | 42 ++-- src/i18n/locales/en/dataLoading.json | 4 + src/i18n/locales/en/loader.json | 2 +- src/i18n/locales/zh/dataLoading.json | 4 + src/i18n/locales/zh/loader.json | 2 +- src/views/DBTableManager.tsx | 232 ++++++++++++------ src/views/DataFormulator.tsx | 21 +- src/views/DataFrameTable.tsx | 35 ++- src/views/DataLoadingChat.tsx | 186 ++++++++++---- src/views/SimpleChartRecBox.tsx | 71 ++---- src/views/UnifiedDataUploadDialog.tsx | 20 +- tests/backend/agents/test_client_utils.py | 4 +- .../test_data_loading_discovery_tools.py | 19 ++ .../unit/components/filterFormat.test.ts | 25 ++ 26 files changed, 717 insertions(+), 320 deletions(-) create mode 100644 tests/frontend/unit/components/filterFormat.test.ts diff --git a/py-src/data_formulator/agents/agent_data_loading_chat.py b/py-src/data_formulator/agents/agent_data_loading_chat.py index cb3d1be8..32e6401c 100644 --- a/py-src/data_formulator/agents/agent_data_loading_chat.py +++ b/py-src/data_formulator/agents/agent_data_loading_chat.py @@ -84,6 +84,10 @@ full table use propose_load_plan, never probe_data. 5. Call propose_load_plan(candidates=[...], reasoning="...") — the UI shows a confirmation card. + Set `selected=true` for every table jointly needed to satisfy the request + (for example, all members of an explicitly requested group). When candidates + are alternatives or the match is ambiguous, select only the best match and + leave the other useful alternatives unselected for the user to review. 6. Keep your text brief after propose_load_plan. The UI handles the rest. Workflow selection rubric (apply in order): @@ -410,6 +414,10 @@ "source_id": {"type": "string"}, "table_key": {"type": "string"}, "display_name": {"type": "string"}, + "selected": { + "type": "boolean", + "description": "Whether this candidate should be checked by default. Select all tables jointly needed for the request; when candidates are ambiguous alternatives, select only the best match." + }, "source_table": {"type": "string", "description": "Optional legacy import id. Prefer source_id + table_key; the backend resolves the real import id."}, "filters": { "type": "array", @@ -425,7 +433,7 @@ "sort_by": {"type": "string"}, "sort_order": {"type": "string", "enum": ["asc", "desc"]}, }, - "required": ["source_id", "table_key", "display_name"], + "required": ["source_id", "table_key", "display_name", "selected"], }, }, "reasoning": {"type": "string", "description": "Brief explanation of why these tables are recommended"}, @@ -1280,6 +1288,15 @@ def _tool_propose_load_plan(self, args): ) } + # A valid, unconfirmed plan must have at least one checked candidate. + # Besides preventing a dead-end zero-selection card, this matters + # because the persisted frontend model uses all-selected-false to mean + # "this plan was already loaded". Respect the agent's recommendation + # when it selected anything; otherwise choose the first resolvable + # candidate as the conservative fallback. + if resolvable and not any(c.get("selected") for c in resolvable): + resolvable[0]["selected"] = True + actions = [{ "type": "load_plan", "candidates": candidates, @@ -1349,6 +1366,9 @@ def _normalize_load_plan_candidate(self, candidate): result["display_name"] = str(display_name) result["source_table"] = str(import_id) result["source_table_name"] = str(source_name) + # Agent recommendation for the initial checkbox state. Default true + # for legacy callers / cached schemas that predate this field. + result["selected"] = result.get("selected") is not False result["filters"] = self._normalize_load_plan_filters(result.get("filters")) if resolution_error: result["resolution_error"] = resolution_error diff --git a/py-src/data_formulator/analyst/skills/core/SKILL.md b/py-src/data_formulator/analyst/skills/core/SKILL.md index 0be7d911..de31afa2 100644 --- a/py-src/data_formulator/analyst/skills/core/SKILL.md +++ b/py-src/data_formulator/analyst/skills/core/SKILL.md @@ -116,9 +116,10 @@ Hand off to a peer agent when the question needs work outside your scope. - `target` — `"data_loading"` (the user's question needs data not in the workspace). -- `options` — 1–2 seed prompts for the target agent; each becomes a one-click - button (label == seed prompt). If two, make them meaningfully distinct (e.g. - `'monthly orders 2024'`). +- `delegate_prompt` — a single, complete instruction for the target agent: + describe exactly what data to find/load — sources, tables, columns, filters, + and time ranges as relevant. Write a full sentence or two, not a bare search + phrase. - `message` — a short note to the user that you're handing off. Only delegate if the workspace tables genuinely can't cover the question. diff --git a/py-src/data_formulator/analyst/skills/core/skill.py b/py-src/data_formulator/analyst/skills/core/skill.py index cc816936..2889b588 100644 --- a/py-src/data_formulator/analyst/skills/core/skill.py +++ b/py-src/data_formulator/analyst/skills/core/skill.py @@ -230,7 +230,7 @@ def _handle_delegate( try: payload = self._normalize_delegate_action(action) except ValueError as exc: - msg = str(exc) or "delegate action requires target and options." + msg = str(exc) or "delegate action requires target and delegate_prompt." yield { "type": "error", "message": msg, @@ -380,15 +380,20 @@ def _normalize_delegate_action(cls, action: dict[str, Any]) -> dict[str, Any]: f"delegate action requires 'target' ∈ {_DELEGATE_TARGETS}, got {target!r}" ) message = str(action.get("message") or "").strip() - raw_options = action.get("options") - cleaned: list[str] = [] - if isinstance(raw_options, list): - for opt in raw_options: - if isinstance(opt, str) and opt.strip(): - cleaned.append(opt.strip()) - if not cleaned: - raise ValueError("delegate action requires non-empty 'options[]'") - payload: dict[str, Any] = {"target": target, "options": cleaned[:2]} + # The agent writes a single complete instruction in `delegate_prompt`. + # Fall back to the legacy `options[]` shape so older callers / cached + # specs still hand off gracefully. + delegate_prompt = str(action.get("delegate_prompt") or "").strip() + if not delegate_prompt: + raw_options = action.get("options") + if isinstance(raw_options, list): + for opt in raw_options: + if isinstance(opt, str) and opt.strip(): + delegate_prompt = opt.strip() + break + if not delegate_prompt: + raise ValueError("delegate action requires a non-empty 'delegate_prompt'") + payload: dict[str, Any] = {"target": target, "delegate_prompt": delegate_prompt} if message: payload["message"] = message return payload diff --git a/py-src/data_formulator/analyst/skills/core/tools.json b/py-src/data_formulator/analyst/skills/core/tools.json index bcb3826d..a433988a 100644 --- a/py-src/data_formulator/analyst/skills/core/tools.json +++ b/py-src/data_formulator/analyst/skills/core/tools.json @@ -148,15 +148,14 @@ }, "message": { "type": "string", - "description": "Short note to the user that you're handing off, e.g. 'I'll hand this to the data loading agent — pick a search:'." + "description": "Short note to the user that you're handing off, e.g. 'I'll hand this to the data loading agent to find the data you need.'" }, - "options": { - "type": "array", - "items": { "type": "string" }, - "description": "1–2 seed prompts for the target agent. Each becomes a one-click button (label == seed prompt); if two, make them meaningfully distinct." + "delegate_prompt": { + "type": "string", + "description": "A single, complete instruction for the target agent describing exactly what data to find/load (sources, tables, columns, filters, time ranges as relevant). Write a full sentence or two — be specific, not a short search phrase." } }, - "required": ["target", "options"] + "required": ["target", "delegate_prompt"] } } } diff --git a/py-src/data_formulator/data_connector.py b/py-src/data_formulator/data_connector.py index 5b1d03fa..7ace63b0 100644 --- a/py-src/data_formulator/data_connector.py +++ b/py-src/data_formulator/data_connector.py @@ -920,6 +920,40 @@ def list_data_loaders(): return json_ok({"loaders": loaders, "disabled": disabled, "plugins": plugins_info}) +@connectors_bp.route("/api/data-loaders/discover-options", methods=["POST"]) +def discover_data_loader_options(): + """Discover values for one loader parameter after an explicit UI action.""" + from data_formulator.data_loader import DATA_LOADERS + + data = request.get_json() or {} + loader_type = str(data.get("loader_type") or "").strip() + param_name = str(data.get("param_name") or "").strip() + loader_class = DATA_LOADERS.get(loader_type) + if loader_class is None: + raise AppError(ErrorCode.INVALID_REQUEST, f"Unknown loader type: {loader_type}") + if not param_name: + raise AppError(ErrorCode.INVALID_REQUEST, "param_name is required") + + params = dict(data.get("params") or {}) + connector_id = data.get("connector_id") + if connector_id: + source = _resolve_connector({"connector_id": connector_id}) + if source._loader_class is not loader_class: + raise AppError(ErrorCode.INVALID_REQUEST, "Connector type does not match loader type") + identity = source._get_identity() + stored = source._vault_retrieve(identity) or {} + supplied = {k: v for k, v in params.items() if v not in (None, "")} + params = {**source._default_params, **stored, **supplied} + + try: + options = loader_class.discover_param_options(param_name, params) + return json_ok({"param_name": param_name, "options": options}) + except AppError: + raise + except Exception as exc: + classify_and_raise_connector_error(exc, operation="connect") + + @connectors_bp.route("/api/local/pick-directory", methods=["POST"]) def pick_local_directory(): """Open a native OS directory picker and return the selected path. diff --git a/py-src/data_formulator/data_loader/external_data_loader.py b/py-src/data_formulator/data_loader/external_data_loader.py index 63eda549..07ce3d4d 100644 --- a/py-src/data_formulator/data_loader/external_data_loader.py +++ b/py-src/data_formulator/data_loader/external_data_loader.py @@ -564,6 +564,21 @@ def validate_params( if missing: raise ConnectorParamError(missing, cls.__name__) + @classmethod + def discover_param_options( + cls, + param_name: str, + params: dict[str, Any], + ) -> list[str]: + """Discover selectable parameter values after an explicit request. + + Loaders may override this for fields whose options require a live, + lightweight service call. Discovery is never run automatically. + """ + raise NotImplementedError( + f"{cls.__name__} does not support options for {param_name}" + ) + @staticmethod @abstractmethod def auth_instructions() -> str: diff --git a/py-src/data_formulator/data_loader/kusto_data_loader.py b/py-src/data_formulator/data_loader/kusto_data_loader.py index e4cf4433..ff9480e1 100644 --- a/py-src/data_formulator/data_loader/kusto_data_loader.py +++ b/py-src/data_formulator/data_loader/kusto_data_loader.py @@ -43,7 +43,7 @@ class KustoDataLoader(ExternalDataLoader): def list_params() -> list[dict[str, Any]]: params_list = [ {"name": "kusto_cluster", "type": "string", "required": True, "tier": "connection", "description": "e.g., https://mycluster.region.kusto.windows.net"}, - {"name": "kusto_database", "type": "string", "required": False, "tier": "filter", "description": "Database name (leave empty to browse all databases)"}, + {"name": "kusto_database", "type": "string", "required": True, "tier": "connection", "description": "Database name (required)"}, {"name": "client_id", "type": "string", "required": False, "tier": "auth", "description": "Service principal only"}, {"name": "client_secret", "type": "string", "required": False, "sensitive": True, "tier": "auth", "description": "Service principal only"}, {"name": "tenant_id", "type": "string", "required": False, "tier": "auth", "description": "Service principal only"} @@ -497,48 +497,40 @@ def _resolve_source_table(self, source_table: str) -> tuple[str | None, str]: return self.kusto_database, source_table return None, source_table - def list_tables(self, table_filter: str | None = None) -> list[dict[str, Any]]: - """List tables from the Kusto cluster. + @classmethod + def discover_param_options( + cls, + param_name: str, + params: dict[str, Any], + ) -> list[str]: + """List accessible databases only when explicitly requested.""" + if param_name != "kusto_database": + return [] + if not str(params.get("kusto_cluster") or "").strip(): + raise ValueError("kusto_cluster is required to load databases") + loader = cls(params) + result = loader.client.execute(None, ".show databases") + df = dataframe_from_result_table(result.primary_results[0]) + if "DatabaseName" not in df.columns: + return [] + return sorted({ + str(name).strip() + for name in df["DatabaseName"].dropna().tolist() + if str(name).strip() + }, key=str.casefold) - When a database is pinned (``kusto_database`` supplied at connect - time), lists tables within that database and returns ``path = - [table]``. Otherwise enumerates every database on the cluster and - returns ``path = [database, table]`` so the catalog groups tables by - database — matching ``catalog_hierarchy()``. + def list_tables(self, table_filter: str | None = None) -> list[dict[str, Any]]: + """List tables from the configured Kusto database. Uses `.show tables details` for lightweight metadata (name, DocString). - When a single database is pinned, a bulk `.show database schema as json` - also fetches every table's columns in one control command. A - full-cluster scan (no pinned database) skips columns — running a bulk - schema query against *every* database is what caused connection - timeouts on large clusters; columns load lazily per database instead. + A bulk `.show database schema as json` also fetches every table's + columns in one control command. """ - if self.kusto_database: - return self._list_tables_in_db( - self.kusto_database, table_filter, - path_prefix=[], fetch_columns=True) - - # No database pinned: enumerate all databases on the cluster. Columns - # are intentionally NOT fetched here — a bulk schema query per database - # across the whole cluster is expensive and times out. They load lazily - # per database (see ``get_metadata`` / pinned-database browsing). - tables: list[dict[str, Any]] = [] - self._report_progress("Listing databases on the cluster…") - db_df = self.query(".show databases") - db_names = [ - rec.get("DatabaseName") - for rec in db_df.to_dict(orient="records") - if rec.get("DatabaseName") - ] - total = len(db_names) - self._report_progress(f"Found {total} databases; listing tables…") - for idx, db_name in enumerate(db_names, start=1): - self._report_progress( - f"Querying database '{db_name}' ({idx}/{total})…") - tables.extend(self._list_tables_in_db( - db_name, table_filter, - path_prefix=[db_name], fetch_columns=False)) - return tables + if not self.kusto_database: + raise ValueError("kusto_database is required") + return self._list_tables_in_db( + self.kusto_database, table_filter, + path_prefix=[], fetch_columns=True) def _fetch_db_columns_bulk(self, db_name: str) -> dict[str, list[dict[str, str]]]: """Fetch column schemas for *every* table in a database with a single @@ -739,8 +731,23 @@ def get_metadata(self, path: list[str]) -> dict[str, Any]: self.kusto_database = old_db def test_connection(self) -> bool: + """Verify live cluster access without invoking result conversion. + + Connection testing must not use :meth:`query`: that method converts + the response to pandas and normalizes its columns, so a local result + conversion failure could incorrectly mark a successful Kusto request + as a failed connection. Catalog information may also exist in the disk + cache and is not evidence that this live probe succeeded. + """ try: - self.query(".show databases | take 1") + self.client.execute(self.kusto_database, ".show tables") return True - except Exception: + except Exception as exc: + logger.warning( + "Kusto connection probe failed for cluster %s (database %s): %s", + self.kusto_cluster, + self.kusto_database, + exc, + exc_info=True, + ) return False \ No newline at end of file diff --git a/src/app/dfSlice.tsx b/src/app/dfSlice.tsx index 8264dba5..0b1999ce 100644 --- a/src/app/dfSlice.tsx +++ b/src/app/dfSlice.tsx @@ -1777,6 +1777,32 @@ export const dataFormulatorSlice = createSlice({ ) => { state.dataLoadingChatPending = action.payload; }, + queueDataLoadingTask: ( + state, + action: PayloadAction<{ text: string; images: string[]; attachments: string[] }>, + ) => { + // Start a new data-loading task while PRESERVING the prior + // conversation (Option A). Retriggers (agent delegate, a fresh + // query from the menu, a sample-task click) no longer wipe the + // thread — instead, when history exists we drop a lightweight + // "new request" divider so the boundary between tasks is clear, + // then queue the submission for `DataLoadingChat` to auto-send. + // The explicit reset button (`clearChatMessages`) remains the way + // to start from a blank slate. + if (state.dataLoadingChatMessages.length > 0) { + state.dataLoadingChatMessages = [ + ...state.dataLoadingChatMessages, + { + id: `divider-${Date.now()}`, + role: 'assistant', + content: '', + divider: true, + timestamp: Date.now(), + }, + ]; + } + state.dataLoadingChatPending = action.payload; + }, clearDataLoadingChatPending: (state) => { state.dataLoadingChatPending = null; }, diff --git a/src/app/utils.tsx b/src/app/utils.tsx index e7de0b56..cadccf14 100644 --- a/src/app/utils.tsx +++ b/src/app/utils.tsx @@ -97,6 +97,7 @@ export interface SourceTableRef { * All action routes accept `connector_id` in the JSON body. */ export const CONNECTOR_ACTION_URLS = { + DISCOVER_OPTIONS: '/api/data-loaders/discover-options', CONNECT: '/api/connectors/connect', DISCONNECT: '/api/connectors/disconnect', GET_STATUS: '/api/connectors/get-status', diff --git a/src/components/ComponentType.tsx b/src/components/ComponentType.tsx index 330befb4..a3a861c3 100644 --- a/src/components/ComponentType.tsx +++ b/src/components/ComponentType.tsx @@ -194,6 +194,7 @@ export interface ChatMessage { codeBlocks?: CodeExecution[]; // executed code + results (assistant only) pendingLoads?: PendingTableLoad[]; // tables awaiting user confirmation loadPlan?: LoadPlan; // Agent-proposed data loading plan + divider?: boolean; // renders a "new request" separator instead of a bubble; excluded from agent history timestamp: number; } diff --git a/src/components/LoadPlanCard.tsx b/src/components/LoadPlanCard.tsx index c054d119..ccacdd01 100644 --- a/src/components/LoadPlanCard.tsx +++ b/src/components/LoadPlanCard.tsx @@ -3,13 +3,15 @@ import React, { useState } from 'react'; import { - Box, Button, Checkbox, Chip, CircularProgress, Typography, + Box, Button, Checkbox, Chip, CircularProgress, Tooltip, Typography, alpha, useTheme, } from '@mui/material'; import CheckIcon from '@mui/icons-material/Check'; +import FilterAltOutlinedIcon from '@mui/icons-material/FilterAltOutlined'; import { useTranslation } from 'react-i18next'; import { apiRequest } from '../app/apiClient'; import { CONNECTOR_ACTION_URLS } from '../app/utils'; +import { getConnectorIcon } from '../icons'; import { transition } from '../app/tokens'; import { TablePreviewRow, TablePreviewData } from './TablePreviewRow'; import { formatFilterChipLabel } from './filterFormat'; @@ -27,10 +29,10 @@ interface LoadPlanCardProps { canLoadInNewWorkspace?: boolean; } -// Plans this small auto-expand each row's preview on first render so the -// user can see what they're loading without an extra click. Larger plans -// (4+) stay collapsed to avoid overwhelming the chat surface. -const AUTO_PREVIEW_THRESHOLD = 3; +// Reserve a stable area while a remote preview request is in flight. Resolved +// previews return to natural height: five data rows plus a quiet row-count +// caption provide enough validation without making multi-candidate plans tall. +const LOAD_PLAN_LOADING_HEIGHT = 158; interface PreviewState { loading: boolean; @@ -54,19 +56,20 @@ export const LoadPlanCard: React.FC = ({ plan, onConfirm, con const theme = useTheme(); const { t } = useTranslation(); const [selection, setSelection] = useState>( - () => Object.fromEntries(plan.candidates.map((c, i) => [i, !c.resolutionError])) + () => Object.fromEntries(plan.candidates.map((c, i) => [ + i, + !c.resolutionError && c.selected !== false, + ])) ); const [loading, setLoading] = useState(false); - // Auto-expand previews for small plans. We seed the map with empty - // expanded entries; the actual data fetch happens lazily inside the - // Collapse's first paint via the same code path as a manual click. - const shouldAutoPreview = plan.candidates.length <= AUTO_PREVIEW_THRESHOLD; + // Every resolvable candidate preview is always open. Seed loading state on + // the first render so the fixed-height spinner area is reserved before the + // asynchronous preview requests begin. const [previews, setPreviews] = useState>(() => { - if (!shouldAutoPreview) return {}; const seed: Record = {}; plan.candidates.forEach((c, i) => { if (!c.resolutionError) { - seed[i] = { loading: false, expanded: true, rows: [], columns: [] }; + seed[i] = { loading: true, expanded: true, rows: [], columns: [] }; } }); return seed; @@ -118,10 +121,9 @@ export const LoadPlanCard: React.FC = ({ plan, onConfirm, con } }, [t]); - // Kick off auto-preview fetches once on mount. We don't await — each - // row paints its loading state and resolves independently. + // Fetch every preview once on mount. We don't await — each row already + // displays its fixed-height spinner and resolves independently. React.useEffect(() => { - if (!shouldAutoPreview) return; plan.candidates.forEach((c, i) => { if (!c.resolutionError) { fetchPreview(c, i); @@ -131,20 +133,6 @@ export const LoadPlanCard: React.FC = ({ plan, onConfirm, con // eslint-disable-next-line react-hooks/exhaustive-deps }, []); - const handlePreview = (candidate: LoadPlanCandidate, idx: number) => { - const current = previews[idx]; - // Toggle if we already have data or are currently fetching. - if (current?.expanded) { - setPreviews(prev => ({ ...prev, [idx]: { ...current, expanded: false } })); - return; - } - if (current?.rows.length) { - setPreviews(prev => ({ ...prev, [idx]: { ...current, expanded: true } })); - return; - } - fetchPreview(candidate, idx); - }; - const handleConfirm = async (newWorkspace = false) => { const selected = plan.candidates.filter((c, i) => selection[i] && !c.resolutionError); if (selected.length === 0) return; @@ -156,12 +144,6 @@ export const LoadPlanCard: React.FC = ({ plan, onConfirm, con } }; - // Whether all candidates resolve to a single source — used to decide - // if the per-row source label is informative or just redundant. - const sources = new Set(plan.candidates.map(c => c.sourceId)); - const showSourceLabel = sources.size > 1; - const sharedSourceId = sources.size === 1 ? plan.candidates[0].sourceId : undefined; - const isDark = theme.palette.mode === 'dark'; const borderColorBase = confirmed ? alpha(theme.palette.success.main, 0.3) @@ -203,18 +185,44 @@ export const LoadPlanCard: React.FC = ({ plan, onConfirm, con : { state: 'idle' }; return ( - 0 ? { + mt: 0.75, + pt: 0.75, + borderTop: '1px solid', + borderColor: 'divider', + } : {}), + }}> + : toggleItem(i)} sx={{ p: 0.25 }} />} - trailing={showSourceLabel - ? ({c.sourceId}) - : undefined} + trailing={!unresolved ? ( + + + {getConnectorIcon(c.sourceId.split(':', 1)[0], { + sx: { fontSize: 13, flexShrink: 0, color: 'text.secondary' }, + })} + + {c.sourceId} + + + + ) : undefined} filterChips={hasFilters ? ( <> + + + + {t('dataLoading.loadPlan.filtersLabel', { defaultValue: 'Filters:' })} + + {c.filters?.map((f, fi) => ( = ({ plan, onConfirm, con ) : undefined} preview={previewData} - expanded={!!preview?.expanded} - onTogglePreview={unresolved ? undefined : () => handlePreview(c, i)} + expanded={!unresolved} + loadingHeight={LOAD_PLAN_LOADING_HEIGHT} dim={unresolved} unresolved={unresolved ? { message: t('dataLoading.loadPlan.unresolved', { @@ -238,19 +246,13 @@ export const LoadPlanCard: React.FC = ({ plan, onConfirm, con }), detail: c.resolutionError, } : undefined} - /> + /> + ); })} - {/* Footer: action button (unconfirmed) or quiet caption (confirmed). - When every candidate shares one source, surface it once down - here instead of duplicating it on each row. */} + {/* Footer: action button (unconfirmed) or quiet caption (confirmed). */} - {!showSourceLabel && sharedSourceId && ( - - {t('dataLoading.loadPlan.fromSource', { defaultValue: 'from' })} {sharedSourceId} - - )} {confirmed ? ( diff --git a/src/components/TablePreviewRow.tsx b/src/components/TablePreviewRow.tsx index aff9d9e6..b91ad130 100644 --- a/src/components/TablePreviewRow.tsx +++ b/src/components/TablePreviewRow.tsx @@ -26,6 +26,9 @@ export interface TablePreviewRowProps { filterChips?: React.ReactNode; // optional chip row under header preview: TablePreviewData; expanded: boolean; + /** Optional height reserved only while a remote preview is loading. + * Ready/error/empty states return to their natural content height. */ + loadingHeight?: number; onTogglePreview?: () => void; unresolved?: { message: string; detail?: string }; dim?: boolean; @@ -33,7 +36,7 @@ export interface TablePreviewRowProps { export const TablePreviewRow: React.FC = ({ name, meta, leading, trailing, filterChips, - preview, expanded, onTogglePreview, unresolved, dim = false, + preview, expanded, loadingHeight, onTogglePreview, unresolved, dim = false, }) => { const { t } = useTranslation(); const showPreviewButton = !!onTogglePreview && !unresolved; @@ -51,10 +54,10 @@ export const TablePreviewRow: React.FC = ({ {leading} {unresolved && } - {name} + {name} {meta && {meta}} - {trailing} + {trailing} {showPreviewButton && ( ) : hasDelegated ? ( @@ -591,18 +678,25 @@ export const DataLoaderForm: React.FC<{ {delegatedLogin!.label || t('db.delegatedLogin')} ) : ( - /* Manual credentials only + connect */ - <> - {renderParamGrid(authParams)} - - + /* Manual credentials only */ + renderParamGrid(authParams) )} + + {/* Primary submit action is separate from all + parameter sections. Delegated-only sources + complete through their sign-in button. */} + {(!hasDelegated || authParams.length > 0) && ( + + + + )} ); })()} diff --git a/src/views/DataFormulator.tsx b/src/views/DataFormulator.tsx index 1ddd4eb8..9ed77935 100644 --- a/src/views/DataFormulator.tsx +++ b/src/views/DataFormulator.tsx @@ -312,7 +312,16 @@ export const DataFormulatorFC = ({ }) => { if (!activeWorkspace) { dispatch(dfActions.setActiveWorkspace({ id: generateSessionId(), displayName: 'Untitled Session' })); } - setUploadDialogInitialTab(tab); + // Compact mode: when opening the generic menu but a data-loading + // conversation is already in progress, land directly on the chat so + // the prior history (and any in-progress extractions / load plan) is + // visible instead of the empty menu hero. Explicit tab requests + // (connector, upload, paste, …) are respected as-is; the menu's + // connectors / direct-load options stay one back-arrow click away. + const resolvedTab = (tab === 'menu' && dataLoadingChatMessages.length > 0) + ? 'extract' + : tab; + setUploadDialogInitialTab(resolvedTab); setUploadDialogOpen(true); }; @@ -324,11 +333,11 @@ export const DataFormulatorFC = ({ }) => { // survived if their name was baked into the prompt text). const startDataLoadingChat = (text: string, images: string[] = [], attachments: string[] = []) => { if (text.trim().length > 0 || images.length > 0 || attachments.length > 0) { - // Fresh query replaces any prior conversation. - if (dataLoadingChatMessages.length > 0) { - dispatch(dfActions.clearChatMessages()); - } - dispatch(dfActions.setDataLoadingChatPending({ text, images, attachments })); + // Preserve any prior conversation (Option A). `queueDataLoadingTask` + // drops a "new request" divider when a thread already exists, then + // enqueues the submission; the user resets explicitly via the + // header reset button when they want a blank slate. + dispatch(dfActions.queueDataLoadingTask({ text, images, attachments })); } openUploadDialog('extract'); }; diff --git a/src/views/DataFrameTable.tsx b/src/views/DataFrameTable.tsx index afb03bbd..5606ac04 100644 --- a/src/views/DataFrameTable.tsx +++ b/src/views/DataFrameTable.tsx @@ -14,6 +14,7 @@ import React from 'react'; import { Box, Tooltip, Typography, useTheme } from '@mui/material'; +import { useTranslation } from 'react-i18next'; const CODE_FONT = '"SF Mono", "Cascadia Code", "Fira Code", Menlo, Consolas, "Liberation Mono", monospace'; @@ -38,6 +39,9 @@ export interface DataFrameTableProps { showIndex?: boolean; /** Optional column descriptions keyed by column name, shown as header tooltips. */ columnDescriptions?: Record; + /** How to indicate that the preview omits additional rows. Defaults to the + * historical ellipsis row; `caption` renders an explicit count below. */ + truncationIndicator?: 'row' | 'caption' | 'none'; /** * When true, columns size to content (CSS `tableLayout: auto`, * `width: max-content`) instead of stretching to fill the container. @@ -60,12 +64,18 @@ export const DataFrameTable: React.FC = ({ headerFontSize = 10, showIndex = false, columnDescriptions, + truncationIndicator = 'row', autoWidth = false, }) => { const theme = useTheme(); + const { t } = useTranslation(); const visibleRows = maxRows != null ? rows.slice(0, maxRows) : rows; - const hasMore = totalRows == null - || totalRows > visibleRows.length + // The preview displays at most `maxRows` data rows, followed by one `…` + // row only when we know additional rows exist. Unknown total alone is not + // evidence of truncation: a three-row result should render three rows, not + // a misleading ellipsis. Callers that reserve a fixed preview height keep + // short tables layout-stable via whitespace instead of fake rows. + const hasMore = (totalRows != null && totalRows > visibleRows.length) || (maxRows != null && rows.length > maxRows); // Abbreviate columns: first half + … + last half @@ -171,7 +181,7 @@ export const DataFrameTable: React.FC = ({ })} ))} - {hasMore && ( + {hasMore && truncationIndicator === 'row' && ( {showIndex && ( = ({ )} + {hasMore && truncationIndicator === 'caption' && ( + + {totalRows != null + ? t('dataLoading.previewShowingRows', { + shown: visibleRows.length, + total: totalRows.toLocaleString(), + }) + : t('dataLoading.previewShowingFirstRows', { + shown: visibleRows.length, + })} + + )} ); }; diff --git a/src/views/DataLoadingChat.tsx b/src/views/DataLoadingChat.tsx index d4a0db98..8486196d 100644 --- a/src/views/DataLoadingChat.tsx +++ b/src/views/DataLoadingChat.tsx @@ -2,13 +2,13 @@ // Licensed under the MIT License. import * as React from 'react'; -import { useEffect, useRef, useState, useCallback } from 'react'; +import { useEffect, useLayoutEffect, useRef, useState, useCallback } from 'react'; import Markdown from 'react-markdown'; import { Box, Button, Chip, CircularProgress, IconButton, Paper, Stack, Tooltip, Typography, - alpha, useTheme, Collapse, + alpha, useTheme, Collapse, Divider, } from '@mui/material'; import AttachFileIcon from '@mui/icons-material/AttachFile'; import InsertDriveFileOutlinedIcon from '@mui/icons-material/InsertDriveFileOutlined'; @@ -304,32 +304,58 @@ const CodeBlockView: React.FC<{ block: CodeExecution }> = ({ block }) => { {block.code} + {block.stdout && ( + + + {block.stdout} + + + )} + {block.error && ( + + + {block.error} + + + )} + {block.resultTable && ( + + + + )} - {block.stdout && ( - - - {block.stdout} - - - )} - {block.error && ( - - - {block.error} - - - )} - {block.resultTable && } ); }; +// --------------------------------------------------------------------------- +// New-request divider +// --------------------------------------------------------------------------- + +// Rendered between the previous conversation and a freshly-started task +// (agent delegate, a new query from the menu, or a sample-task click). +// Preserving history keeps prior extractions recoverable; this separator +// makes the boundary between tasks obvious. Excluded from the agent history +// payload (see `sendMessage`). +const TaskDivider: React.FC = () => { + const { t } = useTranslation(); + return ( + + + + {t('dataLoading.newRequestDivider', 'New request')} + + + + ); +}; + // --------------------------------------------------------------------------- // Single chat message bubble // --------------------------------------------------------------------------- @@ -812,12 +838,65 @@ export const DataLoadingChat: React.FC = ({ onTableLoaded const lastResetRef = useRef(chatResetCounter); const messagesEndRef = useRef(null); const inputRef = useRef(null); + // The scrollable messages viewport and its inner content. Load-plan rows + // reserve a stable spinner area while fetching, then resize once to the + // result's natural height (compact for short tables; five preview rows plus + // a count caption when truncated). Track whether the view is "pinned" so + // that resize follows the bottom without yanking users who scrolled up. + const scrollContainerRef = useRef(null); + const messagesContentRef = useRef(null); + const pinnedToBottomRef = useRef(true); - // Auto-scroll to bottom + const scrollToBottom = () => { + const el = scrollContainerRef.current; + if (!el) return; + // Keep follow-mode synchronous. Smooth scrolling emits intermediate + // positions that can look user-initiated and incorrectly clear the + // pinned state while other dynamic content is still settling. + el.scrollTop = el.scrollHeight; + }; + const updatePinned = () => { + const el = scrollContainerRef.current; + if (!el) return; + // Treat "within 80px of the bottom" as pinned so a slightly-short + // scroll still counts and content growth keeps following. + pinnedToBottomRef.current = el.scrollHeight - el.scrollTop - el.clientHeight < 80; + }; + + // Auto-scroll to bottom on new messages / streaming text — but only when + // the user is pinned to the bottom. useEffect(() => { - messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' }); + if (pinnedToBottomRef.current) scrollToBottom(); }, [chatMessages, streamingContent]); + // On mount (this component remounts each time the chat surface opens), + // jump straight to the latest message with no animation so landing on an + // existing conversation starts at the bottom. + useLayoutEffect(() => { + pinnedToBottomRef.current = true; + scrollToBottom(); + // A second pass after paint catches content that measures its height + // asynchronously (markdown, table previews). + const id = requestAnimationFrame(() => { + if (pinnedToBottomRef.current) scrollToBottom(); + }); + return () => cancelAnimationFrame(id); + }, []); + + // Follow content that changes size AFTER paint: load-plan previews settling + // from their fixed loading slot to natural result height, uploaded images, + // and inline extraction tables. The synchronous scroll avoids a smooth- + // scroll race, and the pinned guard preserves deliberate upward scrolling. + useEffect(() => { + const content = messagesContentRef.current; + if (!content || typeof ResizeObserver === 'undefined') return; + const ro = new ResizeObserver(() => { + if (pinnedToBottomRef.current) scrollToBottom(); + }); + ro.observe(content); + return () => ro.disconnect(); + }, []); + // Auto-focus input useEffect(() => { inputRef.current?.focus(); }, []); @@ -891,7 +970,7 @@ export const DataLoadingChat: React.FC = ({ onTableLoaded setStreamingContent(''); setStreamingToolSteps([]); - const allMessages = [...chatMessages, userMsg].map(m => { + const allMessages = [...chatMessages, userMsg].filter(m => !m.divider).map(m => { // Re-hydrate `[Uploaded: name]` mentions from file attachments // so the backend still sees them as text references, while // the chat UI shows clean text + chips. @@ -948,7 +1027,10 @@ export const DataLoadingChat: React.FC = ({ onTableLoaded sortBy: c.sort_by, sortOrder: c.sort_order, resolutionError: c.resolution_error, - selected: !c.resolution_error, + // Honor the agent's recommendation. Missing + // `selected` means an older backend, so retain + // the historical select-all fallback. + selected: !c.resolution_error && c.selected !== false, })), reasoning: action.reasoning, }; @@ -1111,18 +1193,18 @@ export const DataLoadingChat: React.FC = ({ onTableLoaded // Reuse the shared sample-task list so this in-session panel stays in // sync with the upload-dialog entry point (`UnifiedDataUploadDialog`). // Auto-run is wired through the redux pending slot so the click — - // even on a chat with prior history — atomically clears the thread - // and queues the new submission. + // even on a chat with prior history — preserves the thread, appends a + // "new request" divider, and queues the new submission. const focusSuggestions = React.useMemo(() => buildDataLoadingSuggestions({ t, setInput: setPrompt, setImages: setUserImages, setAttachments: setUserAttachments, requestAutoSend: (payload) => { - if (chatMessages.length > 0) { - dispatch(dfActions.clearChatMessages()); - } - dispatch(dfActions.setDataLoadingChatPending(payload)); + // Preserve prior history (Option A): a sample-task click on a chat + // with existing messages appends a "new request" divider and queues + // the submission rather than wiping the thread. + dispatch(dfActions.queueDataLoadingTask(payload)); }, // eslint-disable-next-line react-hooks/exhaustive-deps }), [t, dispatch]); @@ -1139,19 +1221,26 @@ export const DataLoadingChat: React.FC = ({ onTableLoaded return ( {/* ── Messages area ─────────────────────────────────── */} - - + + {isEmpty ? ( @@ -1187,7 +1276,14 @@ export const DataLoadingChat: React.FC = ({ onTableLoaded ) : ( <> {chatMessages.map((msg) => ( - + msg.divider + ? + : ))} {streamingContent !== '' && } {chatInProgress && !streamingContent && } diff --git a/src/views/SimpleChartRecBox.tsx b/src/views/SimpleChartRecBox.tsx index 3fe84f18..c7461dc6 100644 --- a/src/views/SimpleChartRecBox.tsx +++ b/src/views/SimpleChartRecBox.tsx @@ -1223,58 +1223,35 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ // the same UI position above the input box. if (result.type === "delegate") { const message = String(result.message || '').trim(); - const rawOptions = Array.isArray(result.options) ? result.options : []; - const options: string[] = rawOptions - .map((o: any) => (typeof o === 'string' ? o.trim() : '')) - .filter((s: string) => s.length > 0) - .slice(0, 2); + // The agent now emits a single `delegate_prompt` (auto-sent — + // the user no longer picks from choices). Fall back to the + // legacy `options[]` shape for older backends / cached specs. + const delegatePrompt = String(result.delegate_prompt || '').trim(); + const legacyOption = Array.isArray(result.options) + ? String(result.options.find((o: any) => typeof o === 'string' && o.trim()) || '').trim() + : ''; const target = (result.target === 'report_gen' ? 'report_gen' : 'data_loading') as 'data_loading' | 'report_gen'; - if (target === 'report_gen') { - // Auto-delegate to the report flow — no user approval gate. - // When the user asks for a report, jumping straight into - // report generation is the expected behavior, so we pick the - // agent's first seed prompt (falling back to its message) and - // hand off directly. The report_gen handoff useEffect picks - // this up and re-runs the analyst with the seeded prompt. The - // placeholder draft has no role in the report view, so we - // drop it like a normal completion would. - if (currentDraftId) { - thinkingSteps = []; - pendingThought = ''; - dispatch(dfActions.updateDraftRunningPlan({ draftId: currentDraftId, plan: '' })); - dispatch(dfActions.removeDraftNode(currentDraftId)); - currentDraftId = null; - } - const seedPrompt = options[0] || message; - if (seedPrompt) { - dispatch(dfActions.requestAgentHandoff({ target: 'report_gen', prompt: seedPrompt })); - } - } else if (currentDraftId) { - // data_loading: keep the one-click approval panel — that's a - // different agent / context the user should confirm. - const priorSteps = thinkingSteps.filter(s => s.trim()).join('\n'); + // Auto-delegate for both targets — no user approval gate. When + // the agent decides a peer agent should take over (report gen + // for a narrative, data loading when the workspace lacks the + // needed data), we hand off directly using the agent's + // delegate prompt (falling back to a legacy option, then its + // message). The matching handoff consumer (SimpleChartRecBox + // for report_gen, DataFormulator for data_loading) picks it up + // and starts the target agent with the seeded prompt. The + // placeholder draft has no role once we hand off, so we drop it + // like a normal completion would. + if (currentDraftId) { thinkingSteps = []; pendingThought = ''; dispatch(dfActions.updateDraftRunningPlan({ draftId: currentDraftId, plan: '' })); - - const pauseEntry: InteractionEntry = { - from: 'data-agent', to: 'user', - role: 'delegate', - plan: priorSteps || result.thought || undefined, - content: message, - delegateTarget: target, - delegateOptions: options, - timestamp: Date.now(), - }; - dispatch(dfActions.appendDraftInteraction({ draftId: currentDraftId, entry: pauseEntry })); - currentDraftInteraction.push(pauseEntry); - dispatch(dfActions.updateDeriveStatus({ nodeId: currentDraftId, status: 'clarifying' })); - dispatch(dfActions.updateDraftClarification({ draftId: currentDraftId, pendingClarification: { - trajectory: result.trajectory || [], - completedStepCount: result.completed_step_count || 0, - lastCreatedTableId, - }})); + dispatch(dfActions.removeDraftNode(currentDraftId)); + currentDraftId = null; + } + const seedPrompt = delegatePrompt || legacyOption || message; + if (seedPrompt) { + dispatch(dfActions.requestAgentHandoff({ target, prompt: seedPrompt })); } setIsChatFormulating(false); agentAbortRef.current = null; diff --git a/src/views/UnifiedDataUploadDialog.tsx b/src/views/UnifiedDataUploadDialog.tsx index 32061b33..c85de8bc 100644 --- a/src/views/UnifiedDataUploadDialog.tsx +++ b/src/views/UnifiedDataUploadDialog.tsx @@ -1832,19 +1832,15 @@ export const UnifiedDataUploadDialog: React.FC = ( const hasText = prompt.trim().length > 0; const hasImages = images.length > 0; const hasAttachments = attachments.length > 0; - // Always surface the chat. If the user - // is starting a fresh query, clear any - // prior conversation and enqueue the new - // submission as a redux `pending` slot - // — `DataLoadingChat` consumes it on - // render and auto-sends. Doing both - // dispatches in the same tick keeps the - // handoff atomic; there's no prop race. + // Always surface the chat. Preserve any prior + // conversation (Option A): `queueDataLoadingTask` + // drops a "new request" divider when a thread + // already exists, then enqueues the submission + // as a redux `pending` slot — `DataLoadingChat` + // consumes it on render and auto-sends. The + // header reset button starts a blank slate. if (hasText || hasImages || hasAttachments) { - if (dataLoadingChatMessages.length > 0) { - dispatch(dfActions.clearChatMessages()); - } - dispatch(dfActions.setDataLoadingChatPending({ + dispatch(dfActions.queueDataLoadingTask({ text: prompt, images, attachments, })); } diff --git a/tests/backend/agents/test_client_utils.py b/tests/backend/agents/test_client_utils.py index 6fb81567..fffc3b58 100644 --- a/tests/backend/agents/test_client_utils.py +++ b/tests/backend/agents/test_client_utils.py @@ -277,8 +277,8 @@ def _core_action_tools(): "name": "delegate", "parameters": {"type": "object", "properties": {"target": {"type": "string"}, - "options": {"type": "array"}}, - "required": ["target", "options"]}}}, + "delegate_prompt": {"type": "string"}}, + "required": ["target", "delegate_prompt"]}}}, ] diff --git a/tests/backend/agents/test_data_loading_discovery_tools.py b/tests/backend/agents/test_data_loading_discovery_tools.py index 7c9be80f..a6c66f7b 100644 --- a/tests/backend/agents/test_data_loading_discovery_tools.py +++ b/tests/backend/agents/test_data_loading_discovery_tools.py @@ -253,12 +253,14 @@ def test_returns_load_plan_action(self, tmp_path: Path) -> None: "table_key": "public.orders", "display_name": "orders", "source_table": "public.orders", + "selected": True, }, { "source_id": "pg_prod", "table_key": "public.customers", "display_name": "customers", "source_table": "public.customers", + "selected": False, }, ], "reasoning": "Orders + customer dimension", @@ -269,6 +271,7 @@ def test_returns_load_plan_action(self, tmp_path: Path) -> None: assert action["type"] == "load_plan" assert len(action["candidates"]) == 2 assert action["reasoning"] == "Orders + customer dimension" + assert [candidate["selected"] for candidate in action["candidates"]] == [True, False] def test_empty_candidates_returns_empty_action(self) -> None: agent = DataLoadingAgent(client=None, workspace=_FakeWorkspace()) @@ -276,6 +279,20 @@ def test_empty_candidates_returns_empty_action(self) -> None: assert result["actions"][0]["type"] == "load_plan" assert result["actions"][0]["candidates"] == [] + def test_selects_first_resolvable_when_agent_selects_none(self, tmp_path: Path) -> None: + save_catalog(tmp_path, "pg_prod", [ + {"name": "orders", "table_key": "public.orders", "metadata": {}}, + {"name": "returns", "table_key": "public.returns", "metadata": {}}, + ]) + agent = DataLoadingAgent(client=None, workspace=_FakeWorkspace(tmp_path)) + result = agent._tool_propose_load_plan({"candidates": [ + {"source_id": "pg_prod", "table_key": "public.orders", "display_name": "orders", "selected": False}, + {"source_id": "pg_prod", "table_key": "public.returns", "display_name": "returns", "selected": False}, + ]}) + + candidates = result["actions"][0]["candidates"] + assert [candidate["selected"] for candidate in candidates] == [True, False] + def test_resolves_superset_dataset_id_from_catalog(self) -> None: agent = DataLoadingAgent(client=None, workspace=_FakeWorkspace("/tmp/home")) catalog = [{ @@ -307,6 +324,8 @@ def test_resolves_superset_dataset_id_from_catalog(self) -> None: assert candidate["filters"] == [ {"column": "brand", "operator": "EQ", "value": "Pantum"}, ] + # Legacy callers that omit `selected` retain select-all behavior. + assert candidate["selected"] is True assert "row_limit" not in candidate diff --git a/tests/frontend/unit/components/filterFormat.test.ts b/tests/frontend/unit/components/filterFormat.test.ts new file mode 100644 index 00000000..7d1fddbb --- /dev/null +++ b/tests/frontend/unit/components/filterFormat.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from 'vitest'; +import { formatFilterChipLabel } from '../../../../src/components/filterFormat'; + +describe('formatFilterChipLabel', () => { + it('uses label-value language for equality and membership', () => { + expect(formatFilterChipLabel('grade', 'EQ', 'regular')).toBe('Grade: regular'); + expect(formatFilterChipLabel('sales_region', 'IN', ['West', 'East'])) + .toBe('Sales region: West, East'); + }); + + it('uses a compact range instead of query syntax', () => { + expect(formatFilterChipLabel('date', 'BETWEEN', ['2005-08-01', '2025-08-01'])) + .toBe('Date: 2005-08-01 – 2025-08-01'); + }); + + it('spells out operators whose meaning matters', () => { + expect(formatFilterChipLabel('grade', 'NEQ', 'regular')).toBe('Grade is not regular'); + expect(formatFilterChipLabel('product_name', 'ILIKE', 'market')).toBe('Product name contains market'); + expect(formatFilterChipLabel('owner', 'IS_NULL')).toBe('Owner is empty'); + }); + + it('keeps familiar comparison symbols', () => { + expect(formatFilterChipLabel('price', 'GTE', 10)).toBe('Price ≥ 10'); + }); +}); From ae4b22873a29c9b8a7474f06e3abe67478ec9553 Mon Sep 17 00:00:00 2001 From: Chenglong Wang Date: Fri, 10 Jul 2026 00:16:58 -0700 Subject: [PATCH 39/40] experience improvement --- py-src/data_formulator/data_connector.py | 3 + .../data_loader/external_data_loader.py | 71 ++- .../data_loader/kusto_data_loader.py | 31 +- .../data_loader/mysql_data_loader.py | 12 + src/components/ComponentType.tsx | 13 +- src/i18n/locales/en/upload.json | 1 + src/i18n/locales/zh/upload.json | 1 + src/views/DBTableManager.tsx | 415 +++++++++++++----- src/views/UnifiedDataUploadDialog.tsx | 49 +-- .../routes/test_data_loaders_discovery.py | 12 + 10 files changed, 465 insertions(+), 143 deletions(-) diff --git a/py-src/data_formulator/data_connector.py b/py-src/data_formulator/data_connector.py index 7ace63b0..d4cec2d1 100644 --- a/py-src/data_formulator/data_connector.py +++ b/py-src/data_formulator/data_connector.py @@ -469,6 +469,7 @@ def get_frontend_config(self, include_pinned_in_form: bool = False) -> dict[str, "effective_hierarchy": _hierarchy_dicts(effective), "auth_instructions": self._loader_class.auth_instructions(), "auth_mode": self._loader_class.auth_mode(), + "auth_paths": self._loader_class.auth_paths(), "delegated_login": self._resolve_delegated_login(), } @@ -893,6 +894,7 @@ def list_data_loaders(): "params": params, "hierarchy": _hierarchy_dicts(loader_class.catalog_hierarchy()), "auth_mode": loader_class.auth_mode(), + "auth_paths": loader_class.auth_paths(), "auth_instructions": loader_class.auth_instructions(), "delegated_login": loader_class.delegated_login_config(), "source": "plugin" if plugin_path else "builtin", @@ -1127,6 +1129,7 @@ def list_connectors(): "hierarchy": cfg["hierarchy"], "effective_hierarchy": cfg["effective_hierarchy"], "auth_mode": cfg["auth_mode"], + "auth_paths": cfg["auth_paths"], "delegated_login": cfg.get("delegated_login"), }) diff --git a/py-src/data_formulator/data_loader/external_data_loader.py b/py-src/data_formulator/data_loader/external_data_loader.py index 07ce3d4d..c36db91f 100644 --- a/py-src/data_formulator/data_loader/external_data_loader.py +++ b/py-src/data_formulator/data_loader/external_data_loader.py @@ -551,6 +551,31 @@ def validate_params( When *skip_auth_tier* is True, parameters with ``tier="auth"`` are not checked (useful for SSO/token flows where auth comes externally). """ + # Apply declared defaults before validation. Historically defaults were + # only placeholders in the frontend, which made required fields such as + # MySQL's default user fail unless the user retyped them. + effective = { + pdef["name"]: pdef["default"] + for pdef in cls.list_params() + if "default" in pdef + } + effective.update(params) + for name, value in effective.items(): + params.setdefault(name, value) + + paths = cls.auth_paths() + selected_path = str(effective.get("_auth_path") or "").strip() + if paths: + if not selected_path: + selected_path = cls.infer_auth_path(effective) + effective["_auth_path"] = selected_path + params.setdefault("_auth_path", selected_path) + path = next((item for item in paths if item.get("id") == selected_path), None) + if path is None: + raise ConnectorParamError(["_auth_path"], cls.__name__) + else: + path = None + missing: list[str] = [] for pdef in cls.list_params(): name = pdef.get("name", "") @@ -558,11 +583,53 @@ def validate_params( continue if skip_auth_tier and pdef.get("tier") == "auth": continue - val = params.get(name) + val = effective.get(name) if val is None or (isinstance(val, str) and not val.strip()): missing.append(name) + if path is not None: + for name in path.get("required_fields", []): + val = effective.get(name) + if val is None or (isinstance(val, str) and not val.strip()): + missing.append(name) if missing: - raise ConnectorParamError(missing, cls.__name__) + raise ConnectorParamError(list(dict.fromkeys(missing)), cls.__name__) + + @classmethod + def auth_paths(cls) -> list[dict[str, Any]]: + """Declare mutually exclusive authentication paths for the form. + + The compatibility adapter exposes existing auth-tier fields as one + credentials path. Loaders with alternatives should override this. + """ + auth_fields = [ + pdef["name"] + for pdef in cls.list_params() + if pdef.get("tier") == "auth" + ] + if not auth_fields: + return [] + return [{ + "id": "credentials", + "label": "Credentials", + "description": "Enter credentials for this data source.", + "fields": auth_fields, + "required_fields": [ + pdef["name"] + for pdef in cls.list_params() + if pdef.get("tier") == "auth" and pdef.get("required") + ], + "kind": "credentials", + "default": True, + }] + + @classmethod + def infer_auth_path(cls, params: dict[str, Any]) -> str: + """Choose a path for legacy callers that do not send ``_auth_path``.""" + paths = cls.auth_paths() + return next( + (str(path["id"]) for path in paths if path.get("default")), + str(paths[0]["id"]) if paths else "", + ) @classmethod def discover_param_options( diff --git a/py-src/data_formulator/data_loader/kusto_data_loader.py b/py-src/data_formulator/data_loader/kusto_data_loader.py index ff9480e1..eced05d9 100644 --- a/py-src/data_formulator/data_loader/kusto_data_loader.py +++ b/py-src/data_formulator/data_loader/kusto_data_loader.py @@ -50,6 +50,34 @@ def list_params() -> list[dict[str, Any]]: ] return params_list + @classmethod + def auth_paths(cls) -> list[dict[str, Any]]: + return [ + { + "id": "ambient", + "label": "Azure default identity", + "description": "Use Azure CLI, managed identity, VS Code, or environment credentials.", + "fields": [], + "required_fields": [], + "kind": "ambient", + "default": True, + }, + { + "id": "service_principal", + "label": "Service principal", + "description": "Use an Entra application client ID, secret, and tenant ID.", + "fields": ["client_id", "client_secret", "tenant_id"], + "required_fields": ["client_id", "client_secret", "tenant_id"], + "kind": "credentials", + }, + ] + + @classmethod + def infer_auth_path(cls, params: dict[str, Any]) -> str: + if all(params.get(name) for name in ("client_id", "client_secret", "tenant_id")): + return "service_principal" + return "ambient" + @staticmethod def auth_instructions() -> str: return """**Option 1 — Azure Default Identity (recommended):** Leave the auth fields empty. DF connects using the host's ambient Azure credentials — your Azure CLI login (`az login`) when running locally, or a Managed Identity when deployed to Azure. That identity must be granted access to the cluster. @@ -58,6 +86,7 @@ def auth_instructions() -> str: def __init__(self, params: dict[str, Any]): self.params = params + self.auth_path = params.get("_auth_path") or self.infer_auth_path(params) self.kusto_cluster = params.get("kusto_cluster", None) self.kusto_database = params.get("kusto_database", None) @@ -96,7 +125,7 @@ def _build_kcsb(self) -> KustoConnectionStringBuilder: self.kusto_cluster, self.access_token) # 2. Service principal. - if self.client_id and self.client_secret and self.tenant_id: + if self.auth_path == "service_principal": logger.info("Using service principal authentication for Kusto client.") return KustoConnectionStringBuilder.with_aad_application_key_authentication( self.kusto_cluster, self.client_id, self.client_secret, self.tenant_id) diff --git a/py-src/data_formulator/data_loader/mysql_data_loader.py b/py-src/data_formulator/data_loader/mysql_data_loader.py index ad0a857a..16b5c354 100644 --- a/py-src/data_formulator/data_loader/mysql_data_loader.py +++ b/py-src/data_formulator/data_loader/mysql_data_loader.py @@ -35,6 +35,18 @@ def list_params() -> list[dict[str, Any]]: ] return params_list + @classmethod + def auth_paths(cls) -> list[dict[str, Any]]: + return [{ + "id": "password", + "label": "Username and password", + "description": "Connect with a MySQL user. Password may be blank.", + "fields": ["user", "password"], + "required_fields": ["user"], + "kind": "credentials", + "default": True, + }] + @staticmethod def auth_instructions() -> str: return """**Example:** user: `root` · host: `localhost` · port: `3306` · database: `mydb` diff --git a/src/components/ComponentType.tsx b/src/components/ComponentType.tsx index a3a861c3..bea62189 100644 --- a/src/components/ComponentType.tsx +++ b/src/components/ComponentType.tsx @@ -484,6 +484,16 @@ export interface EncodingDropResult { channel: Channel } +export interface ConnectorAuthPath { + id: string; + label: string; + description?: string; + fields: string[]; + required_fields?: string[]; + kind: 'credentials' | 'ambient' | 'delegated_login' | 'token_exchange'; + default?: boolean; +} + /** A registered connector instance from GET /api/connectors */ export interface ConnectorInstance { id: string; @@ -498,11 +508,12 @@ export interface ConnectorInstance { /** Backend signals that SSO token exchange can auto-connect this source. */ sso_auto_connect?: boolean; deletable?: boolean; - params_form: Array<{name: string; type: string; required: boolean; default?: string; description?: string; sensitive?: boolean; tier?: 'connection' | 'auth' | 'filter'}>; + params_form: Array<{name: string; type: string; required: boolean; default?: string | number | boolean; description?: string; sensitive?: boolean; tier?: 'connection' | 'auth' | 'filter'}>; pinned_params: Record; hierarchy: Array<{key: string; label: string}>; effective_hierarchy: Array<{key: string; label: string}>; auth_mode?: string; + auth_paths?: ConnectorAuthPath[]; auth_instructions?: string; delegated_login?: { login_url: string; label?: string } | null; } diff --git a/src/i18n/locales/en/upload.json b/src/i18n/locales/en/upload.json index 6a6012fb..8fb8667a 100644 --- a/src/i18n/locales/en/upload.json +++ b/src/i18n/locales/en/upload.json @@ -102,6 +102,7 @@ "loadingData": "Loading data...", "loadingDataset": "Loading {{name}}...", "connect": "Connect", + "createConnectionTo": "Create a connection to {{name}}", "connectionNameLabel": "connection name", "dataSourceTypes": "Data Sources", "folderPathPlaceholder": "/path/to/your/data/folder", diff --git a/src/i18n/locales/zh/upload.json b/src/i18n/locales/zh/upload.json index 1daf5519..4e22847e 100644 --- a/src/i18n/locales/zh/upload.json +++ b/src/i18n/locales/zh/upload.json @@ -102,6 +102,7 @@ "loadingData": "加载数据中...", "loadingDataset": "正在加载 {{name}}...", "connect": "连接", + "createConnectionTo": "创建 {{name}} 连接", "connectionNameLabel": "连接名称", "dataSourceTypes": "数据源", "folderPathPlaceholder": "/你的数据文件夹路径", diff --git a/src/views/DBTableManager.tsx b/src/views/DBTableManager.tsx index 6f4c08a5..27f66986 100644 --- a/src/views/DBTableManager.tsx +++ b/src/views/DBTableManager.tsx @@ -12,6 +12,8 @@ import { IconButton, Tooltip, Autocomplete, + Radio, + RadioGroup, } from '@mui/material'; import OpenInNewIcon from '@mui/icons-material/OpenInNew'; @@ -24,6 +26,7 @@ import { useDispatch, useSelector } from 'react-redux'; import { dfActions } from '../app/dfSlice'; import { DataFormulatorState } from '../app/dfSlice'; import { AppDispatch } from '../app/store'; +import { ConnectorAuthPath } from '../components/ComponentType'; const KUSTO_HELP_CLUSTER = 'https://help.kusto.windows.net'; @@ -38,13 +41,45 @@ function extractConnectError(body: any, fallback: string): string { return body.message ?? fallback; } +type DraftTextFieldProps = Omit, 'value' | 'onChange'> & { + value: string; + onDraftChange: (value: string) => void; + onCommit: (value: string) => void; +}; + +// Keep keystroke state inside the active field. Connector-wide state is +// committed on blur, avoiding a Redux update and full form render per key. +const DraftTextField = React.memo(function DraftTextField({ + value, + onDraftChange, + onCommit, + ...props +}: DraftTextFieldProps) { + const [draft, setDraft] = useState(value); + + useEffect(() => setDraft(value), [value]); + + return ( + { + const nextValue = event.target.value; + setDraft(nextValue); + onDraftChange(nextValue); + }} + onBlur={() => onCommit(draft)} + /> + ); +}); + // --------------------------------------------------------------------------- export const DataLoaderForm: React.FC<{ dataLoaderType: string, /** Loader registry key (e.g. "mysql") for i18n lookups. Falls back to dataLoaderType. */ loaderType?: string, - paramDefs: {name: string, default?: string, type: string, required: boolean, description?: string, sensitive?: boolean, tier?: 'connection' | 'auth' | 'filter'}[], + paramDefs: {name: string, default?: string | number | boolean, type: string, required: boolean, description?: string, sensitive?: boolean, tier?: 'connection' | 'auth' | 'filter'}[], authInstructions: string, connectorId?: string, autoConnect?: boolean, @@ -52,6 +87,14 @@ export const DataLoaderForm: React.FC<{ ssoAutoConnect?: boolean, delegatedLogin?: { login_url: string; label?: string } | null, authMode?: string, + authPaths?: ConnectorAuthPath[], + connectionName?: { + label: string, + value: string, + placeholder: string, + onChange: (value: string) => void, + }, + formTitle?: React.ReactNode, onImport: () => void, onFinish: (status: "success" | "error" | "warning", message: string, importedTables?: string[]) => void, onConnected?: () => void, @@ -64,11 +107,11 @@ export const DataLoaderForm: React.FC<{ * user knows credentials are stored on the server (and sees the field * is intentionally empty for security, not a missing config). */ hasStoredCredentials?: boolean, -}> = ({dataLoaderType, loaderType, paramDefs, authInstructions, connectorId, autoConnect, ssoAutoConnect, delegatedLogin, authMode, onImport, onFinish, onConnected, onDelete, onBeforeConnect, hasStoredCredentials}) => { +}> = ({dataLoaderType, loaderType, paramDefs, authInstructions, connectorId, autoConnect, ssoAutoConnect, delegatedLogin, authMode, authPaths = [], connectionName, formTitle, onImport, onFinish, onConnected, onDelete, onBeforeConnect, hasStoredCredentials}) => { const { t } = useTranslation(); const dispatch = useDispatch(); const loaderTypeKey = loaderType || dataLoaderType; - const getParamPlaceholder = (paramDef: {name: string; default?: string; description?: string}) => { + const getParamPlaceholder = (paramDef: {name: string; default?: string | number | boolean; description?: string}) => { // Sensitive fields whose stored credentials we have on the server // get a masked dot placeholder — signals "a value is set, leave // blank to keep, type to replace." @@ -93,6 +136,29 @@ export const DataLoaderForm: React.FC<{ useEffect(() => { connectorIdRef.current = connectorId; }, [connectorId]); const params = useSelector((state: DataFormulatorState) => state.dataLoaderConnectParams[dataLoaderType] ?? {}); + // Materialize declared defaults and the default authentication path as + // actual form values rather than placeholders. Existing user-entered or + // pinned values always win. + useEffect(() => { + for (const paramDef of paramDefs) { + if (params[paramDef.name] === undefined && paramDef.default !== undefined) { + dispatch(dfActions.updateDataLoaderConnectParam({ + dataLoaderType, + paramName: paramDef.name, + paramValue: String(paramDef.default), + })); + } + } + if (authPaths.length > 0 && !params._auth_path) { + const defaultPath = authPaths.find(path => path.default) || authPaths[0]; + dispatch(dfActions.updateDataLoaderConnectParam({ + dataLoaderType, + paramName: '_auth_path', + paramValue: defaultPath.id, + })); + } + }, [authPaths, dataLoaderType, dispatch, paramDefs, params]); + let [isConnecting, setIsConnecting] = useState(false); const [persistCredentials, setPersistCredentials] = useState(true); @@ -121,9 +187,29 @@ export const DataLoaderForm: React.FC<{ () => ({ ...params, ...sensitiveParams }), [params, sensitiveParams] ); + const draftParamsRef = useRef>({}); + useEffect(() => { draftParamsRef.current = {}; }, [dataLoaderType]); + const getCurrentParams = useCallback( + () => ({ ...mergedParams, ...draftParamsRef.current }), + [mergedParams], + ); + const updateParamDraft = useCallback((name: string, value: string) => { + draftParamsRef.current[name] = value; + }, []); + const commitParamDraft = useCallback((name: string, value: string) => { + if (sensitiveParamNames.has(name)) { + setSensitiveParams(previous => ({ ...previous, [name]: value })); + } else { + dispatch(dfActions.updateDataLoaderConnectParam({ + dataLoaderType, + paramName: name, + paramValue: value, + })); + } + }, [dataLoaderType, dispatch, sensitiveParamNames]); const loadKustoDatabases = useCallback(async (paramOverrides?: Record) => { - const discoveryParams = { ...mergedParams, ...paramOverrides }; + const discoveryParams = { ...getCurrentParams(), ...paramOverrides }; if (!String(discoveryParams.kusto_cluster || '').trim() || isLoadingDatabases) return; setDatabaseMenuOpen(true); setIsLoadingDatabases(true); @@ -150,7 +236,7 @@ export const DataLoaderForm: React.FC<{ } finally { setIsLoadingDatabases(false); } - }, [isLoadingDatabases, loaderTypeKey, mergedParams, t]); + }, [getCurrentParams, isLoadingDatabases, loaderTypeKey, t]); // Connection timeout in milliseconds (30 seconds) const CONNECTION_TIMEOUT_MS = 30_000; @@ -182,7 +268,7 @@ export const DataLoaderForm: React.FC<{ const progressTimer = setInterval(pollProgress, 700); try { // Strip table_filter from params sent to connect (it's a catalog-side filter) - const { table_filter: _tf, ...connectParams } = mergedParams as Record; + const { table_filter: _tf, ...connectParams } = getCurrentParams() as Record; // If onBeforeConnect is provided (e.g. AddConnectionPanel), create the connector first if (onBeforeConnect) { connectorIdRef.current = await onBeforeConnect(connectParams); @@ -211,7 +297,7 @@ export const DataLoaderForm: React.FC<{ setConnectProgress(''); setIsConnecting(false); } - }, [mergedParams, persistCredentials, onFinish, onConnected, onBeforeConnect, t]); + }, [getCurrentParams, persistCredentials, onFinish, onConnected, onBeforeConnect, t]); // Delegated (popup-based) login flow for token-based connectors const pollTimerRef = useRef | null>(null); @@ -219,10 +305,11 @@ export const DataLoaderForm: React.FC<{ const handleDelegatedLogin = useCallback(async () => { if (!delegatedLogin?.login_url) return; setIsConnecting(true); + const currentParams = getCurrentParams(); try { // If onBeforeConnect is provided (e.g. AddConnectionPanel), create the connector first if (onBeforeConnect) { - const { table_filter: _tf, ...connectParams } = mergedParams as Record; + const { table_filter: _tf, ...connectParams } = currentParams as Record; connectorIdRef.current = await onBeforeConnect(connectParams); } if (!connectorIdRef.current) return; @@ -236,8 +323,8 @@ export const DataLoaderForm: React.FC<{ url.searchParams.set('df_origin', window.location.origin); // Pass auth-tier form params (e.g. client_id, tenant_id) to the login endpoint for (const p of paramDefs) { - if (p.tier === 'auth' && !p.sensitive && p.type !== 'password' && mergedParams[p.name]) { - url.searchParams.set(p.name, mergedParams[p.name]); + if (p.tier === 'auth' && !p.sensitive && p.type !== 'password' && currentParams[p.name]) { + url.searchParams.set(p.name, currentParams[p.name]); } } @@ -288,7 +375,7 @@ export const DataLoaderForm: React.FC<{ access_token, refresh_token, user, - params: mergedParams, // include any filled-in params (e.g. url) + params: getCurrentParams(), // include any filled-in params (e.g. url) persist: persistCredentials, }), }); @@ -312,7 +399,7 @@ export const DataLoaderForm: React.FC<{ setIsConnecting(false); } }, 1000); - }, [delegatedLogin, mergedParams, persistCredentials, onFinish, onConnected, onBeforeConnect, t]); + }, [delegatedLogin, getCurrentParams, persistCredentials, onFinish, onConnected, onBeforeConnect, t]); // Auto-connect on mount from vault credentials or SSO token passthrough. @@ -374,6 +461,11 @@ export const DataLoaderForm: React.FC<{ the data-source sidebar — this dialog is for create / edit / re-auth only. */} <> + {formTitle && ( + + {formTitle} + + )} {!onBeforeConnect && ( {dataLoaderType} @@ -381,32 +473,84 @@ export const DataLoaderForm: React.FC<{ )} {(() => { const hasTiers = paramDefs.some(p => p.tier); - // Section wrapper: subtle background, rounded, with label - const sectionSx = { mt: 1, px: 1.5, pt: 0.75, pb: 1.5, borderRadius: 1, backgroundColor: 'rgba(0,0,0,0.025)' }; - // Shared input style: standard variant (underline), label always shrunk so placeholder is visible + const renderTimelineStep = ( + step: number, + title: React.ReactNode, + content: React.ReactNode, + isLast = false, + ) => ( + + + + ({ + fontFamily: theme.typography.fontFamily, + fontSize: 10, + lineHeight: 1, + fontWeight: theme.typography.fontWeightMedium, + fontVariantNumeric: 'tabular-nums', + })} + > + {step} + + + {!isLast && ( + + )} + + + {title && ( + + {title} + + )} + {content} + + + ); + // Keep Material UI's native colors, spacing, focus, + // label, and placeholder treatments. Only compact the + // form copy to the surrounding 12px type scale. const inputSx = { - '& .MuiInput-underline:before': { borderBottomColor: 'rgba(0,0,0,0.15)' }, - '& .MuiInputBase-root': { fontSize: 12, mt: 1.5 }, - '& .MuiInputBase-input': { fontSize: 12, py: 0.5, px: 0 }, - '& .MuiInputBase-input::placeholder': { fontSize: 11, opacity: 0.45 }, - '& .MuiInputLabel-root': { fontSize: 11, color: 'text.secondary', fontWeight: 500 }, - '& .MuiInputLabel-root.Mui-focused': { color: 'primary.main' }, + '& .MuiInputBase-root': { fontSize: 12 }, + '& .MuiInputBase-input': { fontSize: 12 }, + '& .MuiInputBase-input::placeholder': { fontSize: 12 }, }; const labelShrinkSlotProps = { inputLabel: { shrink: true } }; - // Pick 2 or 3 columns to minimise orphan fields on the last row - const balancedCols = (n: number) => { - if (n <= 2) return 2; - if (n % 3 === 0) return 3; // 3,6,9 → perfect 3-col rows - if (n % 2 === 0) return 2; // 4,8 → perfect 2-col rows - return 3; // 5,7 → 3 cols (3+2, 3+3+1) is acceptable + const paramGridSx = { + display: 'grid', + gridTemplateColumns: 'repeat(2, minmax(0, 280px))', + columnGap: 2, + rowGap: 2.25, + maxWidth: 576, }; if (!hasTiers) { // Legacy: no tier field, render flat grid - const cols = balancedCols(paramDefs.length); return ( - + {paramDefs.map((paramDef) => ( - { - if (sensitiveParamNames.has(paramDef.name)) { - setSensitiveParams(prev => ({ ...prev, [paramDef.name]: event.target.value })); - } else { - dispatch(dfActions.updateDataLoaderConnectParam({ dataLoaderType, paramName: paramDef.name, paramValue: event.target.value })); - } - }} + onDraftChange={(value) => updateParamDraft(paramDef.name, value)} + onCommit={(value) => commitParamDraft(paramDef.name, value)} /> ))} @@ -430,7 +569,6 @@ export const DataLoaderForm: React.FC<{ } const renderParamGrid = (tierParams: typeof paramDefs) => { - const cols = balancedCols(tierParams.length); // Kusto cluster field: manual URL input plus a // public sample-cluster hint. The hint is never // prefilled; selecting it explicitly starts database @@ -440,7 +578,7 @@ export const DataLoaderForm: React.FC<{ const isKustoDatabase = (name: string) => loaderTypeKey === 'kusto' && name === 'kusto_database'; return ( - + {tierParams.map((paramDef) => ( isKustoCluster(paramDef.name) ? ( ) : ( - { - if (sensitiveParamNames.has(paramDef.name)) { - setSensitiveParams(prev => ({ ...prev, [paramDef.name]: event.target.value })); - } else { - dispatch(dfActions.updateDataLoaderConnectParam({ dataLoaderType, paramName: paramDef.name, paramValue: event.target.value })); - } - }} + onDraftChange={(value) => updateParamDraft(paramDef.name, value)} + onCommit={(value) => commitParamDraft(paramDef.name, value)} /> ) ))} @@ -593,40 +726,89 @@ export const DataLoaderForm: React.FC<{ const connectionParams = paramDefs.filter(p => p.tier === 'connection'); const filterParams = paramDefs.filter(p => p.tier === 'filter'); const authParams = paramDefs.filter(p => p.tier === 'auth'); + const selectedAuthPath = authPaths.find(path => path.id === params._auth_path) + || authPaths.find(path => path.default) + || authPaths[0]; + const selectedAuthFieldNames = new Set(selectedAuthPath?.fields || authParams.map(p => p.name)); + const selectedAuthParams = authParams.filter(p => selectedAuthFieldNames.has(p.name)); const hasDelegated = !!delegatedLogin?.login_url; const connectLabel = onBeforeConnect ? t('db.createConnector', { defaultValue: 'Create Connector' }) : t('db.connect', { suffix: (params.table_filter || '').trim() ? t('db.withFilter') : '' }); + let stepNumber = 0; + const nameStep = connectionName ? ++stepNumber : 0; + const connectionStep = connectionParams.length > 0 ? ++stepNumber : 0; + const authStep = ++stepNumber; + const scopeStep = filterParams.length > 0 ? ++stepNumber : 0; + const actionStep = ++stepNumber; return ( - <> + + {connectionName && renderTimelineStep( + nameStep, + null, + + + , + )} + {/* Tier 1: Connection */} {connectionParams.length > 0 && ( - - - {t('db.tierConnection')} - - {renderParamGrid(connectionParams)} - + renderTimelineStep( + connectionStep, + t('db.tierConnection'), + renderParamGrid(connectionParams), + ) )} - {/* Tier 2: Scope */} - {filterParams.length > 0 && ( - - - {t('db.tierFilter')} - - {renderParamGrid(filterParams)} - - )} + {/* Tier 2: choose an authentication path, then + reveal only that path's credential fields. */} + {renderTimelineStep( + authStep, + t('db.tierAuth'), + <> + {authPaths.length > 1 && ( + dispatch(dfActions.updateDataLoaderConnectParam({ + dataLoaderType, + paramName: '_auth_path', + paramValue: event.target.value, + }))} + sx={{ gap: 1, mb: selectedAuthParams.length > 0 ? 1 : 0 }} + > + {authPaths.map(path => ( + } + label={( + {path.label} + )} + sx={{ mr: 2 }} + /> + ))} + + )} - {/* Tier 3: Sign in */} - - - {t('db.tierAuth')} - + {authPaths.length > 1 && selectedAuthPath?.description && ( + 0 ? 0.5 : 0 }}> + {selectedAuthPath.description} + + )} - {hasDelegated && authParams.length > 0 ? ( + {hasDelegated && selectedAuthParams.length > 0 ? ( /* Left/right split: delegated | or | credentials */ {/* Left: delegated login */} @@ -644,9 +826,9 @@ export const DataLoaderForm: React.FC<{ {/* Right: credential fields + connect */} - - {authParams.map((paramDef) => ( - + {selectedAuthParams.map((paramDef) => ( + { - if (sensitiveParamNames.has(paramDef.name)) { - setSensitiveParams(prev => ({ ...prev, [paramDef.name]: event.target.value })); - } else { - dispatch(dfActions.updateDataLoaderConnectParam({ dataLoaderType, paramName: paramDef.name, paramValue: event.target.value })); - } - }} + onDraftChange={(value) => updateParamDraft(paramDef.name, value)} + onCommit={(value) => commitParamDraft(paramDef.name, value)} /> ))} @@ -679,48 +856,64 @@ export const DataLoaderForm: React.FC<{ ) : ( /* Manual credentials only */ - renderParamGrid(authParams) + renderParamGrid(selectedAuthParams) )} - + , + )} + + {/* Tier 3: general scope options remain + available after every authentication path. */} + {filterParams.length > 0 && ( + renderTimelineStep( + scopeStep, + t('db.tierFilter'), + renderParamGrid(filterParams), + ) + )} {/* Primary submit action is separate from all parameter sections. Delegated-only sources complete through their sign-in button. */} - {(!hasDelegated || authParams.length > 0) && ( - - - + {renderTimelineStep( + actionStep, + null, + + {(!hasDelegated || selectedAuthParams.length > 0) && ( + + )} + {paramDefs.length > 0 && ( + setPersistCredentials(event.target.checked)} + sx={{ p: 0.5 }} + /> + )} + label={( + + {t('db.rememberCredentials')} + + )} + /> + )} + , + true, )} - + ); })()} - {paramDefs.length > 0 && ( - setPersistCredentials(e.target.checked)} - sx={{ p: 0.5 }} - /> - } - label={ - - {t('db.rememberCredentials')} - - } - /> - )} {localizedAuthInstructions && ( ({ - mt: 2, px: 1.5, py: 1, + mt: 3, px: 1.5, py: 1, backgroundColor: 'rgba(0,0,0,0.02)', borderRadius: 1, border: '1px solid rgba(0,0,0,0.06)', diff --git a/src/views/UnifiedDataUploadDialog.tsx b/src/views/UnifiedDataUploadDialog.tsx index c85de8bc..1f436e72 100644 --- a/src/views/UnifiedDataUploadDialog.tsx +++ b/src/views/UnifiedDataUploadDialog.tsx @@ -39,7 +39,7 @@ import { useDispatch, useSelector } from 'react-redux'; import { DataFormulatorState, dfActions } from '../app/dfSlice'; import { AppDispatch } from '../app/store'; import { loadTable } from '../app/tableThunks'; -import { DataSourceConfig, DictTable, ConnectorInstance } from '../components/ComponentType'; +import { DataSourceConfig, DictTable, ConnectorAuthPath, ConnectorInstance } from '../components/ComponentType'; import { createTableFromFromObjectArray, createTableFromText, loadTextDataWrapper, loadBinaryDataWrapper, readFileText } from '../data/utils'; import { DataLoadingChat } from './DataLoadingChat'; import { AnimatedAgentToyIcon } from './AgentToyIcon'; @@ -851,9 +851,10 @@ export const DataLoadMenu: React.FC = ({ interface LoaderType { type: string; name: string; - params: Array<{name: string; type: string; required: boolean; default?: string; description?: string; sensitive?: boolean; tier?: 'connection' | 'auth' | 'filter'}>; + params: Array<{name: string; type: string; required: boolean; default?: string | number | boolean; description?: string; sensitive?: boolean; tier?: 'connection' | 'auth' | 'filter'}>; hierarchy: Array<{key: string; label: string}>; auth_mode?: string; + auth_paths?: ConnectorAuthPath[]; auth_instructions?: string; delegated_login?: { login_url: string; label?: string } | null; source?: 'plugin' | 'builtin'; @@ -879,7 +880,7 @@ const AddConnectionPanel: React.FC<{ const [disabledLoaders, setDisabledLoaders] = useState>({}); const [pluginsInfo, setPluginsInfo] = useState(null); const [selectedType, setSelectedType] = useState(''); - const [displayName, setDisplayName] = useState(''); + const displayNameRef = useRef(''); const dispatch = useDispatch(); const identityKey = useSelector((state: DataFormulatorState) => `${state.identity.type}:${state.identity.id}`); // Track the created connector ID so DataLoaderForm can use it @@ -899,7 +900,7 @@ const AddConnectionPanel: React.FC<{ setPluginsInfo(data.plugins || null); if (data.loaders?.length > 0) { setSelectedType(data.loaders[0].type); - setDisplayName(data.loaders[0].name); + displayNameRef.current = data.loaders[0].name; } }) .catch(() => { /* loader types unavailable — form will be empty */ }); @@ -909,7 +910,7 @@ const AddConnectionPanel: React.FC<{ const handleSelectLoader = (loader: LoaderType) => { setSelectedType(loader.type); - setDisplayName(loader.name); + displayNameRef.current = loader.name; createdIdRef.current = null; }; @@ -923,7 +924,7 @@ const AddConnectionPanel: React.FC<{ headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ loader_type: selectedType, - display_name: displayName.trim() || selectedLoader?.name || selectedType, + display_name: displayNameRef.current.trim() || selectedLoader?.name || selectedType, icon: selectedType, params, persist: true, @@ -931,7 +932,7 @@ const AddConnectionPanel: React.FC<{ }); createdIdRef.current = data.id; return data.id; - }, [selectedType, displayName, selectedLoader]); + }, [selectedType, selectedLoader]); // After DataLoaderForm successfully connects, fetch full connector info and notify parent const handleConnected = useCallback(async () => { @@ -952,16 +953,6 @@ const AddConnectionPanel: React.FC<{ } }, [onCreated, dispatch]); - // Shared input style - const inputSx = { - '& .MuiInput-underline:before': { borderBottomColor: 'rgba(0,0,0,0.15)' }, - '& .MuiInputBase-root': { fontSize: 12, mt: 1.5 }, - '& .MuiInputBase-input': { fontSize: 12, py: 0.5, px: 0 }, - '& .MuiInputBase-input::placeholder': { fontSize: 11, opacity: 0.45 }, - '& .MuiInputLabel-root': { fontSize: 11, color: 'text.secondary', fontWeight: 500 }, - '& .MuiInputLabel-root.Mui-focused': { color: 'primary.main' }, - }; - // Left sidebar button style const sidebarButtonSx = (typeKey: string) => ({ fontSize: 12, @@ -1089,24 +1080,25 @@ const AddConnectionPanel: React.FC<{ /> ) : selectedLoader ? ( - {/* Connection name + DataLoaderForm */} + {/* Connector setup timeline */} - setDisplayName(e.target.value)} - style={{ width: 280, marginBottom: 8 }} - /> { displayNameRef.current = value; }, + }} onImport={() => {}} onFinish={(status, message) => { dispatch(dfActions.addMessages({ @@ -2317,6 +2309,7 @@ export const UnifiedDataUploadDialog: React.FC = ( ssoAutoConnect={false} delegatedLogin={conn.delegated_login} authMode={conn.auth_mode} + authPaths={conn.auth_paths} hasStoredCredentials={conn.has_stored_credentials} onImport={() => {}} onFinish={(status, message) => { diff --git a/tests/backend/routes/test_data_loaders_discovery.py b/tests/backend/routes/test_data_loaders_discovery.py index 52ab084e..5f771630 100644 --- a/tests/backend/routes/test_data_loaders_discovery.py +++ b/tests/backend/routes/test_data_loaders_discovery.py @@ -109,6 +109,7 @@ def test_plugin_appears_in_discovery_endpoint(client_with_plugin): assert "table_filter" in param_names # Auth instructions surface verbatim. assert "Test plugin" in plugin["auth_instructions"] + assert plugin["auth_paths"] == [] def test_builtin_loader_marked_as_builtin(client_with_plugin): @@ -125,6 +126,17 @@ def test_builtin_loader_marked_as_builtin(client_with_plugin): assert builtin["source_path"] is None +def test_mysql_auth_path_surfaces_in_discovery(client_with_plugin): + resp = client_with_plugin.get("/api/data-loaders") + loaders = _loaders_by_type(resp.get_json()) + if "mysql" not in loaders: + pytest.skip("MySQL optional dependency unavailable") + + paths = loaders["mysql"]["auth_paths"] + assert [path["id"] for path in paths] == ["password"] + assert paths[0]["fields"] == ["user", "password"] + + def test_display_name_default_titlecases_registry_key(tmp_path, monkeypatch): """A plugin without DISPLAY_NAME gets title-cased registry-key as name.""" plugins_dir = tmp_path / "plugins" From 5b908ee2f3571f28ac88b1d6b5c383e4475d9343 Mon Sep 17 00:00:00 2001 From: Chenglong Wang Date: Fri, 10 Jul 2026 00:20:05 -0700 Subject: [PATCH 40/40] data connector experience update --- src/views/DataSourceSidebar.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/views/DataSourceSidebar.tsx b/src/views/DataSourceSidebar.tsx index 34b5f5a0..7d2e342d 100644 --- a/src/views/DataSourceSidebar.tsx +++ b/src/views/DataSourceSidebar.tsx @@ -42,6 +42,7 @@ import { VirtualizedCatalogTree } from '../components/VirtualizedCatalogTree'; import StorageIcon from '@mui/icons-material/Storage'; import AddIcon from '@mui/icons-material/Add'; +import AddCircleIcon from '@mui/icons-material/AddCircle'; import FolderOpenIcon from '@mui/icons-material/FolderOpen'; import FolderOutlinedIcon from '@mui/icons-material/FolderOutlined'; import UploadFileIcon from '@mui/icons-material/UploadFile'; @@ -314,7 +315,7 @@ export const DataSourceSidebar: React.FC<{ borderRadius: 1, '&:hover': { bgcolor: 'action.hover' }, }}> - +