diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bfb999..677538a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,24 @@ All notable documentation and user-facing behavior changes are tracked in this f ### Added +- Added **Dual-Stream Audio Capture & Meeting Recording**: + - Captures microphone and Windows desktop/system audio loopback simultaneously using Web Audio API AudioContext mixer (`src/utils/audioMixer.js`). + - Supports 3 capture modes: Meeting Mode (Mic + System Audio), Microphone Only, and System Audio Only. + - Live audio waveform visualizer canvas and recording timer with pause/resume support (`AudioRecorderBar.jsx`). + - Keyboard shortcut `Alt + V` to toggle audio/meeting recorder in the editor toolbar. +- Added **Speech-to-Text (STT) Transcription Service & Model Settings**: + - Offline on-device transcription via Local ONNX Whisper (`whisper-tiny.en`, `whisper-base.en`, `whisper-small`). + - High-speed cloud transcription via Groq (`whisper-large-v3`) and OpenAI (`whisper-1`). + - Dedicated **Speech-to-Text** subtab in AI Settings (`AISettings.jsx`) for selecting engines, local models, capture defaults, and languages. + - Transcripts saved as structured companion `.json` files alongside audio in `media/audio/*.json` with timestamps, speaker attributions, key points, and action items. +- Added **Landing Page Quick Capture Bar**: + - Integrated header toolbar on the landing dashboard (`LandingQuickCaptureBar.jsx`) for one-click Screen Snip, Desktop Recording, and Audio/Meeting Recording. + - Saves captured media directly to workspace media storage with zero note modifications. + - Notification toast provides direct 1-click `[Open Gallery]` action. +- Added **Workspace Media Gallery Unused Media Filter & Audio Player**: + - Physical disk scanner (`media:list-disk-assets` IPC) discovers files in `media/`, `assets/`, and `images/`. + - New `⚠️ Unused / Orphans` filter tab and header stats indicator isolating unreferenced media files (`referenceCount === 0`). + - Interactive audio player preview (`AudioPlayerPreviewItem`) and orphan action buttons (`Copy Markdown Embed`). - Added **Multiple Developer-Friendly Fonts Support** (`View → Font`). - Choose between 5 curated IDE/developer-friendly typefaces: Inter (Default), JetBrains Mono, Fira Code, Cascadia Code, and Source Code Pro. - Changes cascade in real time across all app labels, navigation items, buttons, dialogs, and controls. diff --git a/README.md b/README.md index 1822c0e..ffd7bc8 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,9 @@ Notely is built with Electron + React and is designed for project notes, meeting - **Regex search** with validation and pattern matching for advanced queries. - **Code-aware search** to find patterns inside code blocks only. - Insert common Markdown snippets from the toolbar. +- Record audio notes and full meetings with dual-stream audio capture (Microphone + Windows System Audio loopback) with automatic Speech-to-Text (STT) transcription powered by offline local ONNX Whisper or ultra-fast cloud providers (Groq whisper-large-v3, OpenAI whisper-1). +- Quick capture directly from the landing dashboard with one-click screen snip, desktop recording, and meeting audio capture that saves directly to workspace media without altering notes until you embed them. +- Explore and audit all diagrams, media, audio, and documents in the **Workspace Diagrams & Media Gallery**, featuring physical disk scanning, unreferenced orphan detection, and an **Unused Media** filter with audio playback and transcript preview. - Edit Markdown tables inline with a focused grid editor (row/column add/remove, alignment controls, and compact action chips). - Browse, annotate, optimize, and manage linked media. - Open note files in VS Code or the system default app. diff --git a/THIRD_PARTY_NOTICES.txt b/THIRD_PARTY_NOTICES.txt index 53daccc..c34f3f5 100644 --- a/THIRD_PARTY_NOTICES.txt +++ b/THIRD_PARTY_NOTICES.txt @@ -1,7 +1,7 @@ THIRD-PARTY SOFTWARE NOTICES AND INFORMATION Notely incorporates components from the projects listed below. -This file was generated automatically on 2026-09-24 by +This file was generated automatically on 2026-09-26 by scripts/generate-notices.cjs. Do not edit by hand. Total third-party packages: 189 diff --git a/docs/ai/index.md b/docs/ai/index.md index 11a42b7..02f3b8c 100644 --- a/docs/ai/index.md +++ b/docs/ai/index.md @@ -29,7 +29,12 @@ Notely features a modular, local-first AI platform designed around private data - Runs entirely offline using a local ONNX runtime for `BGE-small-en-v1.5` 384-dimensional dense vectors, with optional fallback to cloud embedding APIs. - Background worker process handles queue processing and debounced note indexing to prevent UI thread latency. -### 4. In-App Tools Catalog & MCP Diagnostics +### 4. Speech-to-Text (STT) Transcription Pipeline +- Transcribe voice memos and meeting discussions into Markdown and companion JSON transcript files. +- Runs offline via on-device WebAssembly/ONNX Whisper models (`whisper-tiny.en`, `whisper-base.en`, `whisper-small`), with optional sub-second cloud transcription via Groq or OpenAI Whisper. +- Synchronized transcript view with key points and action items embedded into notes and the Media Gallery. + +### 5. In-App Tools Catalog & MCP Diagnostics - **MCP Tools Page** (`Ctrl/Cmd + Shift + M`): Interactive catalog to view tool definitions, test inputs, and verify outputs. - **AI Health / Diagnostics**: Real-time telemetry dashboard monitoring MCP connection status, client sessions, request volume, and error diagnostics. -- **AI Settings** (`Ctrl/Cmd + Shift + ,`): Configure API providers, embedding options, and Knowledge Graph extraction confidence thresholds. +- **AI Settings** (`Ctrl/Cmd + Shift + ,`): Configure API providers, embedding options, STT engines, and Knowledge Graph extraction confidence thresholds. diff --git a/docs/ai/setup.md b/docs/ai/setup.md index d5ad90e..f5bec33 100644 --- a/docs/ai/setup.md +++ b/docs/ai/setup.md @@ -38,7 +38,16 @@ Relationship extraction and entity graph generation: --- -## 4. SQLite Database Locality +## 4. Speech-to-Text (STT) Engine Setup + +Configure meeting and audio transcription inside **AI → AI Settings → Speech-to-Text**: +- **Local ONNX Whisper (Offline)**: Download `whisper-tiny.en` (~40MB), `whisper-base.en` (~140MB), or `whisper-small` (~460MB) for private, zero-latency on-device transcription via WebAssembly/ONNX Runtime. +- **Cloud Whisper (Groq / OpenAI)**: Sub-second cloud transcription using your saved Groq or OpenAI API key. +- **Audio Capture Preference**: Set default recording mode to `Meeting: Mic + System Audio`, `Microphone Only`, or `System Audio Only`. + +--- + +## 5. SQLite Database Locality All AI databases are workspace-scoped and stored inside the hidden `{workspace}/.notes-app/` folder to keep your data local and portable: 1. `ai-embeddings.db`: Stores chunk text, line mappings, content hashes, and indexing queues. diff --git a/docs/architecture.md b/docs/architecture.md index 66b2c77..f993e4b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -135,8 +135,8 @@ The following diagram shows the full request path from the React UI through each flowchart TD subgraph Renderer["Renderer Process (React / Vite)"] direction LR - ACP["AIChatPanel"] & AIS["AISettings"] & EBP["EmbeddingsPage"] & KGV["KnowledgeGraph"] - UAI["useAIAssistant hook"] + AIS["AISettings"] & MCP["MCPToolsPage"] & MED["WorkspaceDiagramsMediaPage"] & KGV["KnowledgeGraph"] + ARB["AudioRecorderBar"] & LQC["LandingQuickCaptureBar"] end subgraph Preload["Preload Bridge (preload.cjs)"] diff --git a/docs/data-sync-security.md b/docs/data-sync-security.md index 29faf50..2fd6b88 100644 --- a/docs/data-sync-security.md +++ b/docs/data-sync-security.md @@ -45,13 +45,12 @@ Tips: If you do not share notes between devices, you can ignore this section. -## 4. AI Features for Daily Use +## 4. AI & MCP Integration for Daily Use -1. Open **AI -> AI Settings**. -2. Add the sign-in details for the AI service you want to use. -3. Use AI chat, smarter search, and related-note features as needed. - -If the service you chose cannot do something, Notely shows a warning in AI Settings. +1. Open **AI -> AI Settings** (`Ctrl/Cmd + Shift + ,`). +2. Configure your preferred Speech-to-Text engine (Local ONNX Whisper or Cloud Groq/OpenAI) and API keys. +3. Connect external AI assistants (Google Antigravity, Claude Desktop, Cursor) directly to Notely via the built-in MCP server on port `3700`. +4. Use hybrid semantic search and knowledge graph discovery locally and securely. ## 5. When to Use Workspace Graph diff --git a/docs/feature-availability.md b/docs/feature-availability.md index e688f24..59e34a4 100644 --- a/docs/feature-availability.md +++ b/docs/feature-availability.md @@ -19,14 +19,17 @@ The matrix below details which features run entirely offline, which require loca { feature: 'Help Center', available: true, setup: 'No', internet: false }, { feature: 'Tasks Dashboard', available: true, setup: 'No', internet: false }, { feature: 'Version History (Git)', available: true, setup: 'No', internet: false }, - { feature: 'Media Library', available: true, setup: 'No', internet: false }, + { feature: 'Media Library & Disk Scanner', available: true, setup: 'No', internet: false }, { feature: 'Embedded Terminal', available: true, setup: 'No', internet: false }, - { feature: 'Screen Capture (Windows)', available: true, setup: 'No', internet: false }, + { feature: 'Screen Capture & Snipping', available: true, setup: 'No', internet: false }, + { feature: 'Screen Video Recording & Overlay', available: true, setup: 'No', internet: false }, + { feature: 'Dual-Stream Audio & Meeting Recording', available: true, setup: 'No', internet: false }, + { feature: 'Local Speech-to-Text (ONNX Whisper)', available: true, setup: 'Download Model (~40-460MB)', internet: false }, + { feature: 'Cloud Speech-to-Text (Groq / OpenAI)', available: false, setup: 'Configure API Key in AI Settings', internet: true }, + { feature: 'Model Context Protocol (MCP) Server', available: true, setup: 'No (Runs on port 3700)', internet: false }, { feature: 'Mermaid Diagrams', available: true, setup: 'No', internet: false }, { feature: 'Excalidraw Diagrams', available: true, setup: 'No', internet: false }, - { feature: 'Workspace Graph', available: true, setup: 'No', internet: false }, - { feature: 'Sync with other devices', available: false, setup: 'Pair Trusted Devices', internet: 'Local network' }, - { feature: 'AI Chat & Rewriting', available: false, setup: 'Setup AI Provider', internet: true }, - { feature: 'Meaning-based Search', available: false, setup: 'Setup AI Provider', internet: true }, - { feature: 'Graph Clustering', available: false, setup: 'Setup AI Provider', internet: true } + { feature: 'Workspace Graph & Neural Extraction', available: true, setup: 'No (Local GLiNER2 ONNX)', internet: false }, + { feature: 'Semantic Hybrid Search (BGE)', available: true, setup: 'No (Local BGE ONNX)', internet: false }, + { feature: 'Sync with other devices (P2P)', available: false, setup: 'Pair Trusted Devices', internet: 'Local network' } ]" /> diff --git a/docs/feature-reference.md b/docs/feature-reference.md index 0235bb9..594af9f 100644 --- a/docs/feature-reference.md +++ b/docs/feature-reference.md @@ -79,9 +79,12 @@ During transfer: Open via **Workspace -> Diagrams & Media Gallery** (`Ctrl/Cmd + Alt + M`) or Command Palette: - Catalogs all used diagrams (inline Mermaid, Draw.io, Excalidraw), images, videos, audio, and PDF documents in the workspace. +- **Physical Disk Scanner & Orphan Detection**: Scans workspace folders (`media/`, `assets/`, `images/`) to identify files present on disk that have zero note references. +- **Unused Media Filter**: Quickly isolate unreferenced assets with the `⚠️ Unused / Orphans` filter tab. +- **Audio & Media Preview**: Embedded audio players, video thumbnails, and transcript drawer with key points and action items. - Reuses Knowledge Graph layout aesthetics with collapsible category filters, search, and stat indicators. - Inspect details for any item with high-res/diagram live previews, file paths, and exact note references with line numbers and snippets. -- Direct "Open Note" navigation into the editor. +- Direct "Open Note" navigation and "Copy Markdown Link" actions. ## 2. Editor and Writing Experience @@ -328,6 +331,26 @@ Notely supports area-based screen capture and full screen/window video recording - **Auto-Save & Linking**: Saves `.webm` files under `media/recordings/` and inserts `![Screen Recording](media/recordings/*.webm)` into the note. - **Note Preview & Modal Player**: Renders video links as interactive thumbnail cards with `▶ Play Video` badges and frame decoding. Clicking opens a full-screen playback modal with Download and Copy Path options. +### Audio & Meeting Recording with Speech-to-Text + +Record voice notes or entire conference calls (Zoom, Google Meet, Teams, YouTube) directly into Notely with synchronized audio and transcription. + +- **Dual-Stream Audio Capture**: + - **Meeting Mode**: Simultaneously captures your local microphone and Windows desktop/system audio loopback, combining them into a single stereo track with Web Audio API mixer. + - **Microphone Only**: Dedicated voice memo recording. + - **System Audio Only**: Captures speakers/audio loopback without mic. +- **Speech-to-Text (STT) Transcription**: + - **Local ONNX Whisper (Offline)**: Runs locally in-app via WebAssembly/ONNX runtime (`whisper-tiny.en`, `whisper-base.en`, `whisper-small`). No internet connection or cloud API keys required. + - **Cloud Whisper**: High-speed cloud transcription using Groq (`whisper-large-v3`) or OpenAI (`whisper-1`) with API keys configured in AI Settings. +- **Transcript Storage**: + - Audio files are stored under `media/audio/*.webm`. + - Transcripts are saved alongside audio in `media/audio/*.json` containing segmented timestamps, speaker attributions, key points, and action items. +- **Workflow Differences**: +- **Media Gallery Integration & Transcripts Filter**: + - The **Diagrams, Media & PDFs** gallery (`Workspace -> Diagrams, Media & PDFs`) includes a dedicated **Transcripts** filter category alongside Diagrams, Images, Videos, Audio, PDFs, and Documents. + - Selecting a transcript in the gallery opens a rich inspector showing the AI executive summary, timestamped speaker segments, and a one-click button to copy the full transcript text. + - **On-Demand Transcript Generation**: Any existing audio or video recording in the Media Gallery has a **Generate AI Transcript** button (`✨`). Clicking it decodes the media track, resamples it to 16kHz mono Float32, runs Whisper (Local ONNX or Cloud Groq/OpenAI based on your AI Settings), and saves a companion transcript JSON file in `media/audio/`. + ### Media preview tools Use zoom and media-aware preview controls to inspect assets. @@ -373,40 +396,27 @@ This helps recover from aggressive crops or accidental edits. - Excalidraw previews have their own right-click actions. - If the diagram started from an image, you can switch back to the original image later. -## 9. AI Assistance - -### AI settings - -Set up AI services and sign-in details in **AI -> AI Settings**. - -You can set up: - -- a writing assistant service for chat and rewriting -- a separate service for smarter search and graph features - -### AI chat and commands - -Use AI for writing support, note understanding, and quick content actions. - -### Semantic features - -When AI search data is turned on, Notely can find related notes by meaning, not just exact words. - -### What each AI service can do - -Not every AI service supports every feature. +## 9. AI & Machine Intelligence -Notely shows capability warnings in settings when a selected provider cannot run a feature. +### Privacy-First Architecture -You can also choose which AI model to use in AI settings. +Notely features a local-first, privacy-conscious AI architecture. Rather than relying on cloud-dependent chat sidebars, AI functionality is centered around: +1. **Model Context Protocol (MCP) Server**: Local standard interface on port `3700` (`http://127.0.0.1:3700/mcp` and `/sse`) for external AI assistants (Google Antigravity, Claude Desktop, Cursor). +2. **Local Speech-to-Text (STT)**: On-device Whisper transcription via WebAssembly/ONNX, plus optional cloud transcription. +3. **Local Vector Embeddings & Hybrid Search**: SQLite FTS5 combined with local `BGE-small-en-v1.5` dense embeddings. +4. **Offline Knowledge Graph**: Neural zero-shot entity and relation extraction powered by local `gliner2-multi-v1-onnx`. -### AI operation feedback +### AI Settings & Configuration -Longer AI actions show progress and completion messages so you know something is happening. +Configure AI services and endpoints in **AI -> AI Settings** (`Ctrl/Cmd + Shift + ,`): +- **LLM Provider Setup**: Configure Google Gemini, Groq, or OpenAI / OpenAI-compatible endpoints with custom Base URLs for cloud STT and external workflows. +- **Speech-to-Text (STT) Settings**: Choose between Local ONNX Whisper (`whisper-tiny.en`, `whisper-base.en`, `whisper-small`) and Cloud Whisper (Groq `whisper-large-v3`, OpenAI `whisper-1`). +- **Feature Toggles**: Toggle pattern learning, embedding generation, relationship discovery, and adjust the neural entity extraction confidence threshold slider. -### How recent the AI search data is +### MCP Diagnostics & Interactive Testing -Notely shows whether its AI search data is fresh or getting old. +- **MCP Tools Catalog** (`Ctrl/Cmd + Shift + M`): Inspect all 7 enterprise tools (`search`, `read_note`, `edit_note`, `manage_tasks`, `manage_diagrams`, `workspace_overview`, `git_control`), run live tool test calls, and inspect JSON payloads. +- **MCP Telemetry & Diagnostics**: View live connection status, active client sessions, request rates, execution latency, and error logs. ## 10. Peer-to-Peer Sync diff --git a/docs/keyboard-shortcuts.md b/docs/keyboard-shortcuts.md index 75d412b..a6a73f2 100644 --- a/docs/keyboard-shortcuts.md +++ b/docs/keyboard-shortcuts.md @@ -55,6 +55,8 @@ Use shortcuts to navigate and edit notes quickly across Notely. | Toggle Outline Panel | `Ctrl/Cmd + Alt + L` | Editor | | Open Reference Note | `Ctrl/Cmd + Shift + K` | Editor | | Insert Reference Link | `Ctrl/Cmd + Shift + L` | Editor | +| Capture Screen Snip | `Ctrl/Cmd + Shift + S` | Editor | +| Toggle Audio / Meeting Recorder | `Alt + V` | Editor | ## Search & Find Panel diff --git a/docs/settings-reference.md b/docs/settings-reference.md index 20578c0..cad329b 100644 --- a/docs/settings-reference.md +++ b/docs/settings-reference.md @@ -154,6 +154,19 @@ Open **AI -> AI Settings**. Use lower temperature for predictable output. Use higher temperature for brainstorming or variation. +### Speech-to-Text (STT) setup + +Open the **Speech-to-Text** tab in AI Settings: + +- **Transcription Engine**: + - `Local ONNX Whisper (Offline, In-Browser / WebAssembly)`: default offline engine; runs private on-device transcription. + - `Groq Cloud Whisper`: utilizes Groq's high-throughput `whisper-large-v3` for sub-second transcriptions using your configured Groq API key. + - `OpenAI Cloud Whisper`: utilizes OpenAI's `whisper-1` model using your configured OpenAI API key. +- **Local Whisper Model**: Select from `whisper-tiny.en` (~40MB, fastest), `whisper-base.en` (~140MB, balanced), or `whisper-small` (~460MB, multilingual). +- **Default Audio Capture Mode**: Choose default mode when launching recorder (`Meeting: Mic + System Audio`, `Microphone Only`, `System Audio Only`). +- **Primary Spoken Language**: Specify expected audio language (or `Auto-Detect`). +- **Auto-transcribe toggle**: Automatically triggers Speech-to-Text generation as soon as an audio or meeting recording is stopped. + ### Data and privacy controls - Shows where AI-related app data is stored on your device diff --git a/docs/user-guide.md b/docs/user-guide.md index 1f00e63..d9f5286 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -177,3 +177,28 @@ Customize app look and feel to fit your workflow: - **Help -> Help Center** (`F1`) for in-app help. - **Help -> Keyboard Shortcuts** (`Ctrl/Cmd + /`) for key bindings. - **Help -> About Notely** for app version details. + +## 14. Record Meetings, Capture Audio & Manage Media Gallery + +### Record a Meeting with Dual-Stream Audio (Mic + System Loopback) + +1. Open any note or press `Alt + V` (or click the 🎙️ **Record Audio** icon on the editor toolbar). +2. Choose **Meeting Mode** to capture both your voice and speakers/participants (Zoom, Google Meet, Teams, YouTube). +3. Click **Start Recording**. A live waveform visualizer and recording timer start. +4. When the discussion finishes, click **Stop & Transcribe**. +5. Notely transcribes the discussion with Whisper (local offline ONNX or cloud), saves the `.webm` audio and `.json` transcript to `media/audio/`, and automatically inserts the audio player and transcript block at your cursor. + +### Quick Capture from the Landing Dashboard + +1. On the landing dashboard, use the **Quick Capture** toolbar: + - 📸 **Snip Screen**: Capture any desktop region to an image file. + - 🎥 **Record Screen**: Select a screen or application window to record. + - 🎙️ **Record Audio**: Capture quick voice memos or meeting audio. +2. Captured media is stored directly in workspace media storage without modifying any notes. +3. A notification toast appears with a 1-click **[Open Gallery]** button to inspect the asset immediately. + +### Find and Embed Unused / Orphaned Media + +1. Open **Workspace -> Diagrams & Media Gallery** (`Ctrl/Cmd + Alt + M`). +2. Click the **⚠️ Unused / Orphans** filter tab in the left sidebar or the header stats pill. +3. Inspect unreferenced files, listen to audio recordings, or click **Copy Markdown Embed** to link them into your notes. diff --git a/docs/workspace/audio-recording.md b/docs/workspace/audio-recording.md new file mode 100644 index 0000000..7a1f5fa --- /dev/null +++ b/docs/workspace/audio-recording.md @@ -0,0 +1,56 @@ +--- +title: Audio Recording & Speech-to-Text +description: Record voice notes, conference meetings with dual-stream audio loopback, and generate AI transcripts with local Whisper. +keywords: audio recording, voice notes, meeting recorder, speech to text, whisper, transcription, transcript drawer +category: Workspace +--- + +# Audio Recording & Speech-to-Text + +Notely includes a dual-stream audio recording engine and speech-to-text pipeline for voice notes and conference call transcriptions. + +--- + +## 1. Recording Modes + +Trigger audio recording via `Alt + V`, the **🎙️ Record Audio** toolbar button in the Markdown editor, or the **Quick Capture Bar** on the landing dashboard. + +1. **Meeting Mode (Mic + System Loopback)**: + - Captures microphone input and Windows system audio (Zoom, Google Meet, Microsoft Teams, YouTube) concurrently. + - Merges both streams into a synchronized stereo track via Web Audio API. +2. **Microphone Only**: + - Voice memo and dictation capture from your primary recording device. +3. **System Audio Only**: + - Records desktop/speaker audio output without microphone commentary. + +--- + +## 2. Speech-to-Text (STT) Engines + +Configure STT preferences in **AI -> AI Settings -> Speech-to-Text**: + +- **Local ONNX Whisper (Offline)**: + - Runs on-device via WebAssembly/ONNX Runtime. + - Models: `whisper-tiny.en` (~40MB), `whisper-base.en` (~140MB), `whisper-small` (~460MB). + - Private, zero external network traffic. +- **Cloud Whisper**: + - High-throughput cloud transcription via **Groq** (`whisper-large-v3`) or **OpenAI** (`whisper-1`). + - Sub-second turnaround using your configured API keys. + +--- + +## 3. Storage & Markdown Integration + +- **Audio Files**: Stored in `{workspace}/media/audio/*.webm`. +- **Transcripts**: Stored alongside audio files in `{workspace}/media/audio/*.json`. +- **Note Insertion**: Inserting a recording places both the audio player and transcript block at your cursor: + ```markdown + ![Audio Recording](media/audio/recording-2026-09-26.webm) + ``` + +--- + +## 4. Diagrams & Media Gallery Integration + +- Filter by the **Transcripts** and **Audio** categories in `Workspace -> Diagrams & Media Gallery` (`Ctrl/Cmd + Alt + M`). +- **On-Demand AI Transcript Generation (`✨`)**: Run speech-to-text on any existing recording without re-recording. diff --git a/docs/workspace/media.md b/docs/workspace/media.md index 2ac0d49..2406a9d 100644 --- a/docs/workspace/media.md +++ b/docs/workspace/media.md @@ -38,8 +38,9 @@ Keep your assets tidy using the Media Health Dashboard: ## 4. Diagrams & Media Gallery -Notely provides a dedicated full-screen page under **Workspace > Diagrams & Media Gallery** (shortcut: `Ctrl+Alt+M` or `Cmd+Alt+M`) inspired by the Knowledge Graph layout: -- **Only Used Items**: Displays every diagram (inline Mermaid, Draw.io, Excalidraw), image, audio, video, and PDF actively referenced in notes. -- **Collapsible Category Filter**: Filter by category (Diagrams, Images, PDFs, Videos, Audio, Documents) and usage scope (Single-note vs Multi-note). +Notely provides a dedicated full-screen page under **Workspace > Diagrams & Media Gallery** (shortcut: `Ctrl+Alt+M` or `Cmd+Alt+M`): +- **Physical Disk Scanner & Orphan Detection**: Scans workspace folders (`media/`, `assets/`, `images/`) to identify files present on disk that have zero note references, letting you clean up orphaned files with the `⚠️ Unused / Orphans` filter tab. +- **Collapsible Category Filter**: Filter by category (Diagrams, Images, PDFs, Videos, Audio, Documents, Transcripts) and usage scope. - **Referenced Notes & Line Jumping**: Click any item to inspect details and see all notes referencing it, complete with line numbers and markdown snippets. Click **Open Note** to jump directly to the note and line. -- **Live Previews**: Interactive rendered Mermaid previews and high-resolution media previews. +- **Audio & Media Preview**: Embedded audio player with waveform preview, video modal player, and expandable transcript drawer with action items and key points. +- **On-Demand AI Transcript Generation (`✨`)**: Run speech-to-text directly on any existing video or audio recording in the gallery. diff --git a/electron/lib/core/appMenu.cjs b/electron/lib/core/appMenu.cjs index 9a6010d..db14c06 100644 --- a/electron/lib/core/appMenu.cjs +++ b/electron/lib/core/appMenu.cjs @@ -851,7 +851,7 @@ function buildAppMenuTemplate(win, context = {}, deps = {}) { click: () => sendMenuAction(win, "open-workspace-index") }, { - label: "Diagrams & Media Gallery", + label: "Media Gallery", accelerator: "CmdOrCtrl+Alt+M", click: () => sendMenuAction(win, "open-workspace-diagrams-media") }, @@ -860,10 +860,6 @@ function buildAppMenuTemplate(win, context = {}, deps = {}) { accelerator: "CmdOrCtrl+Shift+A", click: () => sendMenuAction(win, "open-workspace-activity") }, - { - label: "Assets Library", - click: () => sendMenuAction(win, "open-assets") - }, { label: "Trash / Removed Items", click: () => sendMenuAction(win, "open-trash") diff --git a/electron/lib/core/windowLifecycle.cjs b/electron/lib/core/windowLifecycle.cjs index 826f7f4..8ccef89 100644 --- a/electron/lib/core/windowLifecycle.cjs +++ b/electron/lib/core/windowLifecycle.cjs @@ -345,12 +345,12 @@ function createWindowLifecycle(deps) { const isDev = Boolean(rendererUrl); const scriptSrc = isDev - ? "script-src 'self' 'unsafe-inline' 'unsafe-eval'" - : "script-src 'self' 'unsafe-inline'"; + ? "script-src 'self' 'unsafe-inline' 'unsafe-eval' 'wasm-unsafe-eval' blob: https://cdn.jsdelivr.net" + : "script-src 'self' 'unsafe-inline' 'wasm-unsafe-eval' blob: https://cdn.jsdelivr.net"; const connectSrc = isDev - ? "connect-src 'self' https://api.languagetool.org https://esm.sh https://unpkg.com ws: http://127.0.0.1:* http://localhost:*" - : "connect-src 'self' https://api.languagetool.org https://esm.sh https://unpkg.com"; + ? "connect-src 'self' data: blob: https://huggingface.co https://*.huggingface.co https://*.hf.co https://cdn.jsdelivr.net https://api.languagetool.org https://esm.sh https://unpkg.com ws: http://127.0.0.1:* http://localhost:*" + : "connect-src 'self' data: blob: https://huggingface.co https://*.huggingface.co https://*.hf.co https://cdn.jsdelivr.net https://api.languagetool.org https://esm.sh https://unpkg.com"; return [ "default-src 'self'", @@ -360,7 +360,7 @@ function createWindowLifecycle(deps) { "media-src 'self' data: blob: file:", "font-src 'self' https://fonts.gstatic.com https://esm.sh https://unpkg.com data:", connectSrc, - "worker-src 'self' blob:", + "worker-src 'self' blob: data: https://cdn.jsdelivr.net", "child-src 'self' blob: data:", "object-src 'none'", "base-uri 'self'", @@ -462,7 +462,8 @@ function createWindowLifecycle(deps) { contextIsolation: true, nodeIntegration: false, sandbox: true, - webviewTag: false + webviewTag: false, + backgroundThrottling: false, } }; @@ -616,6 +617,7 @@ function createWindowLifecycle(deps) { nodeIntegration: false, sandbox: true, webviewTag: false, + backgroundThrottling: false, }, }); diff --git a/electron/lib/documents/documentIpc.cjs b/electron/lib/documents/documentIpc.cjs index 0922d4b..c205b86 100644 --- a/electron/lib/documents/documentIpc.cjs +++ b/electron/lib/documents/documentIpc.cjs @@ -104,7 +104,10 @@ function registerDocumentIpcHandlers(ipcMain, deps) { const requestedFolderPath = String( (typeof payload === "string" ? payload : payload?.folderPath) || "" ).trim(); - const targetDir = path.resolve(requestedFolderPath || projectRoot); + let targetDir = path.resolve(requestedFolderPath || projectRoot); + if (fs.existsSync(targetDir) && fs.statSync(targetDir).isFile()) { + targetDir = path.dirname(targetDir); + } if (!filePathWithin(projectRoot, targetDir)) { throw new Error("Invalid folder path."); @@ -246,7 +249,34 @@ function registerDocumentIpcHandlers(ipcMain, deps) { const activeProject = getActiveProject(); const notesRoot = getNotesRoot(); const projectRoot = path.resolve(activeProject?.rootPath || notesRoot); - const resolved = path.resolve(filePath); + + let targetPath = String(filePath || "").trim(); + if (!targetPath.toLowerCase().endsWith(".md")) { + targetPath = `${targetPath}.md`; + } + + let resolved = path.resolve(targetPath); + if (!fs.existsSync(resolved)) { + const inProject = path.resolve(projectRoot, targetPath); + if (fs.existsSync(inProject)) { + resolved = inProject; + } else if (notesRoot) { + const inNotes = path.resolve(notesRoot, targetPath); + if (fs.existsSync(inNotes)) { + resolved = inNotes; + } + } + } + + if (!fs.existsSync(resolved)) { + const fileName = path.basename(targetPath).toLowerCase(); + const allFiles = listWorkspaceFileEntries(projectRoot); + const found = allFiles.find((f) => path.basename(f.filePath).toLowerCase() === fileName); + if (found && fs.existsSync(found.filePath)) { + resolved = found.filePath; + } + } + const isAllowed = (filePathWithin(projectRoot, resolved) || (notesRoot && filePathWithin(notesRoot, resolved))); if (!isAllowed || path.extname(resolved).toLowerCase() !== ".md") { throw new Error("Invalid document path."); @@ -263,7 +293,34 @@ function registerDocumentIpcHandlers(ipcMain, deps) { const activeProject = getActiveProject(); const notesRoot = getNotesRoot(); const projectRoot = path.resolve(activeProject?.rootPath || notesRoot); - const resolved = path.resolve(String(filePath || "")); + + let targetPath = String(filePath || "").trim(); + if (!targetPath.toLowerCase().endsWith(".md")) { + targetPath = `${targetPath}.md`; + } + + let resolved = path.resolve(targetPath); + if (!fs.existsSync(resolved)) { + const inProject = path.resolve(projectRoot, targetPath); + if (fs.existsSync(inProject)) { + resolved = inProject; + } else if (notesRoot) { + const inNotes = path.resolve(notesRoot, targetPath); + if (fs.existsSync(inNotes)) { + resolved = inNotes; + } + } + } + + if (!fs.existsSync(resolved)) { + const fileName = path.basename(targetPath).toLowerCase(); + const allFiles = listWorkspaceFileEntries(projectRoot); + const found = allFiles.find((f) => path.basename(f.filePath).toLowerCase() === fileName); + if (found && fs.existsSync(found.filePath)) { + resolved = found.filePath; + } + } + const isAllowed = (filePathWithin(projectRoot, resolved) || (notesRoot && filePathWithin(notesRoot, resolved))); if (!isAllowed || path.extname(resolved).toLowerCase() !== ".md") { throw new Error("Invalid document path."); diff --git a/electron/lib/media/imageMedia.cjs b/electron/lib/media/imageMedia.cjs index 335e5bc..820e22b 100644 --- a/electron/lib/media/imageMedia.cjs +++ b/electron/lib/media/imageMedia.cjs @@ -226,6 +226,19 @@ function buildPdfExportHtml({ title, markdownContent, baseHref, sourceDir, downs let annotation = null; if (srcIndex >= 0) { const rawSrc = String(tokens[idx].attrs[srcIndex][1] || "").trim(); + const isExplicitAudio = /\.(mp3|wav|m4a|aac|flac|wma)(\?|#|$)/i.test(rawSrc); + const isAudioDir = /[/\\]audio[/\\]/i.test(rawSrc); + const isAudioNamed = /(recording|voice|meeting|mic|audio).*?\.(webm|ogg)$/i.test(rawSrc); + const isAudioAlt = /(audio|voice)/i.test(tokens[idx].content || tokens[idx].attrGet("alt") || ""); + const isAudio = isExplicitAudio || (isAudioDir && /\.(webm|ogg|wav|mp3|m4a|aac|flac)(\?|#|$)/i.test(rawSrc)) || (isAudioNamed && !rawSrc.includes("screen") && !rawSrc.includes("rec_")) || (isAudioAlt && !rawSrc.includes("screen") && /\.(webm|ogg)(\?|#|$)/i.test(rawSrc)); + + if (isAudio) { + const altText = tokens[idx].content || tokens[idx].attrGet("alt") || "Audio Recording"; + const fileName = rawSrc.split(/[?#]/)[0].split(/[/\\]/).pop() || "recording"; + const ext = fileName.split(".").pop()?.toUpperCase() || "AUDIO"; + return `
🎙️
${escapeCodeHtml(altText)}${escapeCodeHtml(fileName)} (${escapeCodeHtml(ext)})
`; + } + const isVideo = /\.(webm|mp4|ogg|mov|mkv|avi|m4v)(\?|#|$)/i.test(rawSrc); if (isVideo) { const altText = tokens[idx].content || tokens[idx].attrGet("alt") || "Video Recording"; @@ -470,10 +483,10 @@ function resolveImageAssetPath(basePath, assetPath) { break; } } - const isWorkspaceImageLink = /^[/\\]+(images|media)[/\\]/i.test(decodedAsset); + const isWorkspaceImageLink = /^[/\\]+(images|media|audio|transcripts)[/\\]/i.test(decodedAsset); const normalizedAsset = decodedAsset .replace(/^\.\//, "") - .replace(/^[/\\]+(images|media)[/\\]/i, "$1/"); + .replace(/^[/\\]+(images|media|audio|transcripts)[/\\]/i, "$1/"); const legacyDiagramMatch = normalizedAsset.match(/^(?:\.notes-app[\\/])?excali-diagrams[\\/]([^\\/]+)[\\/]([^\\/]+)[\\/]diagram\.png$/i); const sluglessDiagramMatch = normalizedAsset.match(/^(?:\.notes-app[\\/])?excali-diagrams[\\/]([^\\/]+)[\\/]diagram\.png$/i); @@ -488,16 +501,20 @@ function resolveImageAssetPath(basePath, assetPath) { if (isWorkspaceImageLink) { candidates.push(path.resolve(getNotesRoot(), normalizedAsset)); - } else if (/^(images|media)[\\/]/i.test(normalizedAsset)) { + } else if (/^(images|media|audio|transcripts)[\\/]/i.test(normalizedAsset)) { candidates.push(path.resolve(baseDir, normalizedAsset)); candidates.push(path.resolve(getNotesRoot(), normalizedAsset)); } else { candidates.push(path.resolve(baseDir, normalizedAsset)); candidates.push(path.resolve(baseDir, "images", normalizedAsset)); candidates.push(path.resolve(baseDir, "media", normalizedAsset)); + candidates.push(path.resolve(baseDir, "audio", normalizedAsset)); + candidates.push(path.resolve(baseDir, "transcripts", normalizedAsset)); candidates.push(path.resolve(getNotesRoot(), normalizedAsset)); candidates.push(path.resolve(getNotesRoot(), "images", normalizedAsset)); candidates.push(path.resolve(getNotesRoot(), "media", normalizedAsset)); + candidates.push(path.resolve(getNotesRoot(), "audio", normalizedAsset)); + candidates.push(path.resolve(getNotesRoot(), "transcripts", normalizedAsset)); if (sluglessDiagramMatch && !/^\.notes-app[\\/]/i.test(normalizedAsset)) { const [, diagramId] = sluglessDiagramMatch; candidates.push(path.resolve(baseDir, `.notes-app/excali-diagrams/${diagramId}/diagram.png`)); @@ -780,6 +797,125 @@ registerTrustedHandler("video:save", (_event, payload) => { return `media/recordings/${safeFileName}`; }); +registerTrustedHandler("audio:save", (_event, payload) => { + const { fileName, base64Data, transcript } = payload || {}; + const safeFileName = path.basename(fileName || "transcript.json").replace(/[<>:"/\\|?*]+/g, "-"); + const audioDir = path.join(getNotesRoot(), "media", "audio"); + ensureDir(audioDir); + + // Handle standalone companion transcript saving + if (transcript && !base64Data) { + const transcriptFileName = safeFileName.endsWith(".json") ? safeFileName : safeFileName.replace(/\.[^.]+$/, "") + ".json"; + const transcriptPath = path.join(audioDir, transcriptFileName); + const content = typeof transcript === "string" ? transcript : JSON.stringify(transcript, null, 2); + fs.writeFileSync(transcriptPath, content, "utf8"); + return { + audioPath: null, + transcriptPath: `media/audio/${transcriptFileName}`, + fileName: transcriptFileName, + }; + } + + if (!base64Data || typeof base64Data !== "string" || !base64Data.includes(",")) { + throw new Error("Invalid audio payload."); + } + + const targetPath = path.join(audioDir, safeFileName); + if (!filePathWithin(getNotesRoot(), targetPath)) { + throw new Error("Invalid audio path."); + } + + const buffer = Buffer.from(base64Data.split(",")[1], "base64"); + if (!buffer.length) { + throw new Error("Audio data is empty."); + } + fs.writeFileSync(targetPath, buffer); + + let transcriptRelPath = null; + if (transcript) { + const transcriptFileName = safeFileName.replace(/\.[^.]+$/, "") + ".json"; + const transcriptPath = path.join(audioDir, transcriptFileName); + const content = typeof transcript === "string" ? transcript : JSON.stringify(transcript, null, 2); + fs.writeFileSync(transcriptPath, content, "utf8"); + transcriptRelPath = `media/audio/${transcriptFileName}`; + } + + return { + audioPath: `media/audio/${safeFileName}`, + transcriptPath: transcriptRelPath, + fileName: safeFileName, + }; +}); + +registerTrustedHandler("media:list-disk-assets", () => { + const notesRoot = getNotesRoot(); + if (!notesRoot || !fs.existsSync(notesRoot)) { + return []; + } + + const scanDirs = [ + path.join(notesRoot, "media"), + path.join(notesRoot, "assets"), + path.join(notesRoot, "images"), + ]; + + const results = []; + const visitedPaths = new Set(); + + for (const rootDir of scanDirs) { + if (!fs.existsSync(rootDir)) continue; + + const queue = [rootDir]; + while (queue.length > 0) { + const currentDir = queue.shift(); + let entries; + try { + entries = fs.readdirSync(currentDir, { withFileTypes: true }); + } catch { + continue; + } + + for (const entry of entries) { + if (entry.name.startsWith(".")) continue; + if (entry.name === THUMBNAIL_DIR_NAME || entry.name === ORIGINAL_IMAGE_DIR_NAME) continue; + + const fullPath = path.join(currentDir, entry.name); + if (entry.isDirectory()) { + queue.push(fullPath); + } else if (entry.isFile()) { + const relFromRoot = path.relative(notesRoot, fullPath).replace(/\\/g, "/"); + if (visitedPaths.has(relFromRoot.toLowerCase())) continue; + visitedPaths.add(relFromRoot.toLowerCase()); + + const ext = path.extname(entry.name).slice(1).toLowerCase(); + // Skip internal metadata/temporary files + if (["tmp", "crswap", "bak"].includes(ext)) continue; + + let size = 0; + let mtime = null; + try { + const stat = fs.statSync(fullPath); + size = stat.size; + mtime = stat.mtime.toISOString(); + } catch { + // Ignore stat read failure for transient files + } + + results.push({ + path: relFromRoot, + name: entry.name, + ext, + size, + mtime, + }); + } + } + } + } + + return results; +}); + registerTrustedHandler("images:download", async (event, payload) => { const { getExportManager } = require("../export/ExportManager.cjs"); const exportManager = getExportManager(); @@ -836,28 +972,32 @@ registerTrustedHandler("images:list", (_event, payload) => { }; const baseDir = path.dirname(path.resolve(basePath)); - const localImagesDir = path.join(baseDir, "images"); - const rootImagesDir = path.join(getNotesRoot(), "images"); - const mediaImagesDir = path.join(getNotesRoot(), "media", "images"); - const mediaDocsDir = path.join(getNotesRoot(), "media", "docs"); - - const localNames = readImagesIn(localImagesDir); - const seen = new Set(localNames.map((name) => name.toLowerCase())); - - const rootNames = readImagesIn(rootImagesDir).filter((name) => !seen.has(name.toLowerCase())); - rootNames.forEach(name => seen.add(name.toLowerCase())); - - const mediaImagesNames = readImagesIn(mediaImagesDir).filter((name) => !seen.has(name.toLowerCase())); - mediaImagesNames.forEach(name => seen.add(name.toLowerCase())); + const paths = []; + const seen = new Set(); + + const addEntries = (dir, prefix) => { + const names = readImagesIn(dir); + for (const name of names) { + const assetPath = `${prefix}/${name}`.replace(/\/+/g, "/").replace(/^\.\//, "./"); + const key = assetPath.toLowerCase(); + if (!seen.has(key)) { + seen.add(key); + paths.push(assetPath); + } + } + }; - const mediaDocsNames = readImagesIn(mediaDocsDir).filter((name) => !seen.has(name.toLowerCase())); + // Local note-adjacent directories + addEntries(path.join(baseDir, "images"), "./images"); + addEntries(path.join(baseDir, "audio"), "./audio"); + addEntries(path.join(baseDir, "transcripts"), "./transcripts"); + addEntries(path.join(baseDir, "media"), "./media"); - const paths = [ - ...localNames.map((name) => `./images/${name}`), - ...rootNames.map((name) => `/images/${name}`), - ...mediaImagesNames.map((name) => `/media/images/${name}`), - ...mediaDocsNames.map((name) => `/media/docs/${name}`), - ]; + // Workspace-root directories + addEntries(path.join(getNotesRoot(), "images"), "/images"); + addEntries(path.join(getNotesRoot(), "audio"), "/audio"); + addEntries(path.join(getNotesRoot(), "transcripts"), "/transcripts"); + addEntries(path.join(getNotesRoot(), "media"), "/media"); if (!includeAnnotations && !includeOriginalStatus) return paths; @@ -1141,6 +1281,8 @@ registerTrustedHandler("images:read", (_event, payload) => { ".wav": "audio/wav", ".ogg": "audio/ogg", ".m4a": "audio/mp4", + ".aac": "audio/aac", + ".flac": "audio/flac", ".pdf": "application/pdf", ".doc": "application/msword", ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", @@ -1160,7 +1302,10 @@ registerTrustedHandler("images:read", (_event, payload) => { ".7z": "application/x-7z-compressed", ".rar": "application/vnd.rar" }; - const mimeType = mimeMap[ext] || "application/octet-stream"; + let mimeType = mimeMap[ext] || "application/octet-stream"; + if (ext === ".webm" && (/[/\\]audio[/\\]/i.test(fileToRead) || /(recording|voice|mic|audio)/i.test(path.basename(fileToRead)))) { + mimeType = "audio/webm"; + } const buffer = fs.readFileSync(fileToRead); return `data:${mimeType};base64,${buffer.toString("base64")}`; }); diff --git a/electron/lib/media/pdf.cjs b/electron/lib/media/pdf.cjs index 21434c3..00226c2 100644 --- a/electron/lib/media/pdf.cjs +++ b/electron/lib/media/pdf.cjs @@ -1,5 +1,3 @@ -const path = require("node:path"); - function sanitizePdfMarkdown(markdown) { const source = String(markdown || ""); diff --git a/electron/preload.cjs b/electron/preload.cjs index 9446b97..af3cb7c 100644 --- a/electron/preload.cjs +++ b/electron/preload.cjs @@ -145,6 +145,8 @@ contextBridge.exposeInMainWorld("notesApi", { captureCurrentDisplay: () => ipcRenderer.invoke("screen:capture-current-display"), getDesktopSources: () => ipcRenderer.invoke("screen:get-sources"), saveVideo: (payload) => ipcRenderer.invoke("video:save", payload), + saveAudio: (payload) => ipcRenderer.invoke("audio:save", payload), + listDiskMediaAssets: () => ipcRenderer.invoke("media:list-disk-assets"), minimizeMainWindow: () => ipcRenderer.invoke("window:minimize-main"), restoreMainWindow: () => ipcRenderer.invoke("window:restore-main"), openRecordingOverlay: () => ipcRenderer.invoke("window:open-recording-overlay"), diff --git a/src/App.jsx b/src/App.jsx index dfa2c1b..507dcf5 100644 --- a/src/App.jsx +++ b/src/App.jsx @@ -302,7 +302,7 @@ export default function App() { useEffect(() => { const handleToast = (e) => { if (e.detail && e.detail.message) { - notify(e.detail.message, e.detail.type || "info"); + notify(e.detail.message, e.detail.type || "info", e.detail.action || null); } }; window.addEventListener("app:toast", handleToast); @@ -318,7 +318,6 @@ export default function App() { return () => unsubscribe(); }, [notify]); const { - landingAssetsOpen, setLandingAssetsOpen, commandPaletteOpen, setCommandPaletteOpen, globalSearchOpen, setGlobalSearchOpen, globalSearchQuery, setGlobalSearchQuery, @@ -416,7 +415,6 @@ export default function App() { saving, writeMetadataToFile, setWriteMetadataToFile, - error, setError, activeProject, newNoteTitle, @@ -1101,7 +1099,7 @@ export default function App() { async function handleOpenReferencedDocumentFromUI(filePath, optionsOrLineNumber) { await handleOpenReferencedDocument(filePath, optionsOrLineNumber); - setLandingAssetsOpen(false); + setDiagramsMediaOpen(false); } const refreshGitWorkspaceMeta = useCallback(async function refreshGitWorkspaceMeta() { @@ -1645,7 +1643,7 @@ export default function App() { } if (action === "open-assets") { - setLandingAssetsOpen(true); + setDiagramsMediaOpen(true); return; } @@ -2463,8 +2461,7 @@ export default function App() { disabled: recentWorkspacePaths.length === 0, aliases: "recent workspaces recently opened folders", }, - { id: "open-assets", label: "Open Assets Library", group: "Workspace", aliases: "media images assets" }, - { id: "open-workspace-diagrams-media", label: "Open Diagrams & Media Gallery", group: "Workspace", aliases: "diagrams media pdfs gallery assets mermaid drawio excalidraw" }, + { id: "open-workspace-diagrams-media", label: "Open Media Gallery", group: "Workspace", aliases: "media gallery assets diagrams pdfs images video audio mermaid drawio excalidraw" }, { id: "open-workspace-activity", label: "Open Workspace Activity", group: "Sync", aliases: "activity timeline sync events" }, { id: "open-p2p-status", label: "Open P2P Status", group: "Sync", aliases: "peer status p2p" }, { id: "open-knowledge-graph", label: "Open Knowledge Graph", group: "AI", aliases: "workspace graph mind map network relations nodes" }, @@ -2889,12 +2886,7 @@ export default function App() { return; } - if (resolvedCommandId === "open-assets") { - setLandingAssetsOpen(true); - return; - } - - if (resolvedCommandId === "open-workspace-diagrams-media") { + if (resolvedCommandId === "open-workspace-diagrams-media" || resolvedCommandId === "open-assets") { setDiagramsMediaOpen(true); return; } @@ -3089,7 +3081,7 @@ export default function App() { } if (action === "assets") { - setLandingAssetsOpen(true); + setDiagramsMediaOpen(true); return; } @@ -3250,7 +3242,6 @@ export default function App() { ); })} - {error &&
{error}
} {!showTerminal && !(focusModeEnabled && current) ? (
@@ -3955,14 +3946,6 @@ export default function App() { feedbackOpen={feedbackOpen} setFeedbackOpen={setFeedbackOpen} themePreference={themePreference} - landingAssetsOpen={landingAssetsOpen} - setLandingAssetsOpen={setLandingAssetsOpen} - landingFolderPath={landingFolderPath} - current={current} - activeProject={activeProject} - notesFolderPath={notesFolderPath} - notify={notify} - handleOpenReferencedDocumentFromUI={handleOpenReferencedDocumentFromUI} showUpdateModal={showUpdateModal} setShowUpdateModal={setShowUpdateModal} updateStatus={updateStatus} diff --git a/src/components/AISettings.jsx b/src/components/AISettings.jsx index 560db75..1e77141 100644 --- a/src/components/AISettings.jsx +++ b/src/components/AISettings.jsx @@ -1,10 +1,22 @@ import React, { useEffect, useState } from 'react'; -import { Save, Trash2, Zap, AlertCircle, Eye, EyeOff, Download, Database } from 'lucide-react'; +import { Save, Trash2, Zap, AlertCircle, Eye, EyeOff, Download, Database, Mic, Cpu, Network } from 'lucide-react'; import AppInput from './AppInput'; import AppSelect from './AppSelect'; import "../styles/AISettings.css"; import OverlayDialog from './OverlayDialog'; import KnowledgeGraphSettings from './KnowledgeGraphSettings'; +import { + getSTTPreferences, + setSTTPreferences, + checkLocalWhisperModelStatus, + preDownloadLocalWhisperModel, + deleteLocalWhisperModel, +} from '../services/sttService'; +import { + showSuccessToast, + showErrorToast, + showInfoToast, +} from '../utils/notificationUtils'; import { aiClearData, aiGetApiKey, @@ -61,6 +73,47 @@ export const AISettingsContent = ({ _onClose }) => { const [activeSubTab, setActiveSubTab] = useState("providers"); const [modelStatus, setModelStatus] = useState({ downloaded: false, isDownloading: false, progress: 0 }); + const [sttSettings, setSttSettings] = useState({ + engine: "local-onnx", + localModel: "onnx-community/whisper-tiny.en", + language: "auto", + defaultSourceMode: "meeting", + autoTranscribe: true, + }); + const [whisperModelStatus, setWhisperModelStatus] = useState({ + downloaded: false, + isDownloading: false, + progress: 0, + statusText: '', + }); + + useEffect(() => { + getSTTPreferences().then((res) => { + if (res) setSttSettings(res); + }).catch(() => {}); + }, []); + + // Check whether selected local Whisper model is downloaded/cached + useEffect(() => { + let active = true; + const checkStatus = async () => { + if (sttSettings.engine !== "local-onnx") return; + try { + const res = await checkLocalWhisperModelStatus(sttSettings.localModel); + if (active) { + setWhisperModelStatus(prev => ({ + ...prev, + downloaded: Boolean(res.downloaded), + cachedFilesCount: res.cachedFilesCount, + })); + } + } catch (err) { + console.warn('Error checking Whisper model status:', err); + } + }; + checkStatus(); + return () => { active = false; }; + }, [sttSettings.localModel, sttSettings.engine]); useEffect(() => { const loadModelStatus = async () => { @@ -353,9 +406,6 @@ export const AISettingsContent = ({ _onClose }) => { Enable AI Subsystem - - Toggle the global switch to enable or disable all background AI services, embeddings, and graph extraction. -
-
+
@@ -494,55 +508,32 @@ export const AISettingsContent = ({ _onClose }) => {

Providers Setup

-
- -
- { - const val = e.target.value; - setSelectedProvider(val); - setPreferences((prev) => ({ ...prev, aiProvider: val })); - }} - disabled={loading} - style={{ flex: 1 }} - > - {providers.filter(p => p.available).map((p) => ( - - ))} - - -
+
+ + { + const val = e.target.value; + setSelectedProvider(val); + const updatedPrefs = { ...preferences, aiProvider: val }; + setPreferences(updatedPrefs); + try { + await aiSetPreferences(updatedPrefs); + window.dispatchEvent(new CustomEvent('app:toast', { + detail: { message: `Active provider set to ${val}.`, type: 'success' } + })); + } catch (err) { + console.error('Failed to set active provider:', err); + } + }} + disabled={loading} + style={{ width: "100%", marginTop: "4px" }} + > + {providers.filter(p => p.available).map((p) => ( + + ))} +
@@ -641,36 +632,15 @@ export const AISettingsContent = ({ _onClose }) => {
{/* Provider-specific details and helper links */} -
+
{selectedProvider === "groq" && ( - <> -
Groq Cloud Provider Info:
-
- Get your API key at: console.groq.com/keys -
-
- ⚠️ Rate Limit Warning: Groq free tier has daily token limits (TPD). If you hit a 429 rate limit error, you will need to wait for quota reset or upgrade to a developer tier. -
- + API Keys available at console.groq.com/keys )} {selectedProvider === "gemini" && ( - <> -
Google Gemini Provider Info:
-
- Get your API key at: aistudio.google.com/app/apikey -
-
- Gemini offers a generous free tier for developers with high limits. -
- + API Keys available at aistudio.google.com/app/apikey )} {selectedProvider === "openai" && ( - <> -
OpenAI Provider Info:
-
- Get your API key at: platform.openai.com/api-keys -
- + API Keys available at platform.openai.com/api-keys )}
@@ -688,6 +658,70 @@ export const AISettingsContent = ({ _onClose }) => { ))}
)} + + {/* Advanced Features & Storage */} +
+

+ Advanced & Local Storage +

+
+ + + +
+ +
+
+ Local AI Memory & Cache + Data stored in .notes-app/ai-memory.db +
+ +
+
)} @@ -903,74 +937,207 @@ export const AISettingsContent = ({ _onClose }) => {
)} - {activeSubTab === "behavior" && ( + + + {activeSubTab === "stt" && ( <> -
-

Features

-
-
-
-
{EDITOR_MODE_OPTIONS.map((item) => ( + + {/* Screen Record Button */} + + + {/* Audio & Meeting Record Button */} + + +
+ +
+ + {/* Source Picker Modal for Screen Recording */} + {sourcePickerOpen && ( + setSourcePickerOpen(false)} + /> + )} + + {/* Audio Recorder Bar */} + {audioRecorderOpen && ( +
+ setAudioRecorderOpen(false)} + onSaveSuccess={handleAudioSaveSuccess} + initialMode="meeting" + /> +
+ )} +
+ ); +} diff --git a/src/components/MarkdownPreview.jsx b/src/components/MarkdownPreview.jsx index 7a12c24..6cab39d 100644 --- a/src/components/MarkdownPreview.jsx +++ b/src/components/MarkdownPreview.jsx @@ -1,5 +1,5 @@ import { useEffect, useMemo, useRef, useState, memo } from "react"; -import { Search, Copy, ExternalLink, Pencil, RefreshCw, Trash2, RotateCcw } from "lucide-react"; +import { Search, Copy, ExternalLink, Pencil, RefreshCw, Trash2, RotateCcw, Download } from "lucide-react"; import { renderMarkdown, parseDiagramBlocks, @@ -36,10 +36,12 @@ function imageCacheKey(assetPath, variant = "thumbnail") { function getImageActionElement(target) { if (!(target instanceof HTMLElement)) return null; if (target.closest?.(".excalidraw-block")) return null; - if (target.tagName === "IMG") return target; + if (target.tagName === "IMG" || target.tagName === "AUDIO" || target.tagName === "VIDEO") return target; const frame = target.closest?.(".markdown-image-frame"); - const framedImage = frame?.querySelector?.("img"); - return framedImage instanceof HTMLImageElement ? framedImage : null; + const media = frame?.querySelector?.("img, audio, video"); + if (media instanceof HTMLElement) return media; + if (frame && (frame.getAttribute("data-asset-path") || frame.getAttribute("data-audio-src") || frame.getAttribute("data-video-src"))) return frame; + return null; } function getExcalidrawActionContext(target) { @@ -499,14 +501,46 @@ export const MarkdownPreview = memo(function MarkdownPreviewContent({ return parseDiagramBlocks(content); }, [content]); + const lastBasePathRef = useRef(basePath); + const lastOriginalsRef = useRef(showOriginalImages); + useEffect(() => { - imageResolveCacheRef.current.clear(); + if (lastBasePathRef.current !== basePath || lastOriginalsRef.current !== showOriginalImages) { + imageResolveCacheRef.current.clear(); + lastBasePathRef.current = basePath; + lastOriginalsRef.current = showOriginalImages; + } let cancelled = false; const previewElement = previewRef.current; if (!previewElement || !basePath) return undefined; + const setMediaSrcSafely = (element, nextSrc) => { + if (!nextSrc) return; + const currentResolved = element.getAttribute("data-resolved-src"); + if (currentResolved === nextSrc && element.src === nextSrc) return; + + const isMedia = element instanceof HTMLMediaElement; + const wasPlaying = isMedia && !element.paused; + const prevTime = isMedia ? element.currentTime : 0; + + element.setAttribute("data-resolved-src", nextSrc); + if (element.src !== nextSrc) { + element.src = nextSrc; + if (isMedia && prevTime > 0) { + try { + element.currentTime = prevTime; + if (wasPlaying) { + element.play().catch(() => {}); + } + } catch { + // Ignore seek error + } + } + } + }; + const resolveMediaElement = async (element) => { - if (!element || (!(element instanceof HTMLImageElement) && !(element instanceof HTMLVideoElement))) return; + if (!element || (!(element instanceof HTMLImageElement) && !(element instanceof HTMLVideoElement) && !(element instanceof HTMLAudioElement))) return; const isImage = element instanceof HTMLImageElement; if (isImage) { @@ -517,7 +551,7 @@ export const MarkdownPreview = memo(function MarkdownPreviewContent({ element.setAttribute("aria-label", element.getAttribute("alt") || "Image"); } - const existingAssetPath = element.getAttribute("data-asset-path") || element.getAttribute("data-video-src") || ""; + const existingAssetPath = element.getAttribute("data-asset-path") || element.getAttribute("data-video-src") || element.getAttribute("data-audio-src") || ""; const src = element.getAttribute("src") || ""; const assetPath = (existingAssetPath && !/^(data:|blob:)/i.test(existingAssetPath)) ? existingAssetPath @@ -535,7 +569,9 @@ export const MarkdownPreview = memo(function MarkdownPreviewContent({ const cacheKey = imageCacheKey(assetPath, variant); if (cache.has(cacheKey)) { const cached = cache.get(cacheKey); - if (!cancelled && cached) element.src = cached; + if (!cancelled && cached) { + setMediaSrcSafely(element, cached); + } if (isImage) { const annotationKey = `annotation:${assetPath}`; if (cache.has(annotationKey)) { @@ -550,8 +586,8 @@ export const MarkdownPreview = memo(function MarkdownPreviewContent({ const resolved = await readImage(basePath, cleanAssetPath, { thumbnail: isImage ? !showOriginalImages : false }); if (!cancelled && resolved) { cache.set(cacheKey, resolved); - element.src = resolved; - if (!isImage) { + setMediaSrcSafely(element, resolved); + if (!isImage && element instanceof HTMLVideoElement) { element.onloadeddata = () => { try { if (element.currentTime === 0) element.currentTime = 0.1; @@ -585,7 +621,7 @@ export const MarkdownPreview = memo(function MarkdownPreviewContent({ }; const resolveAllMedia = () => { - const mediaElements = Array.from(previewElement.querySelectorAll("img, video")); + const mediaElements = Array.from(previewElement.querySelectorAll("img, video, audio")); mediaElements.forEach((element) => { void resolveMediaElement(element); }); @@ -598,11 +634,11 @@ export const MarkdownPreview = memo(function MarkdownPreviewContent({ mutations.forEach((mutation) => { mutation.addedNodes.forEach((node) => { if (!(node instanceof HTMLElement)) return; - if (node.tagName === "IMG" || node.tagName === "VIDEO") { + if (node.tagName === "IMG" || node.tagName === "VIDEO" || node.tagName === "AUDIO") { void resolveMediaElement(node); } if (typeof node.querySelectorAll === "function") { - node.querySelectorAll("img, video").forEach((child) => { + node.querySelectorAll("img, video, audio").forEach((child) => { void resolveMediaElement(child); }); } @@ -1220,12 +1256,25 @@ export const MarkdownPreview = memo(function MarkdownPreviewContent({ return; } + if (imageAction.dataset.imageAction === "copy") { + event.preventDefault(); + event.stopPropagation(); + const frame = imageElement.closest?.(".markdown-image-frame") || imageElement; + const assetPath = imageElement.getAttribute("data-asset-path") || frame.getAttribute?.("data-asset-path") || imageElement.getAttribute("data-audio-src") || frame.getAttribute?.("data-audio-src") || imageElement.getAttribute("data-video-src") || frame.getAttribute?.("data-video-src") || imageElement.getAttribute("src") || ""; + if (assetPath) { + navigator.clipboard.writeText(assetPath); + onNotify?.(`Copied path: ${assetPath}`, "success"); + } + return; + } + if (imageAction.dataset.imageAction === "download") { event.preventDefault(); event.stopPropagation(); - const assetPath = imageElement.getAttribute("data-asset-path") || imageElement.getAttribute("src") || ""; - const altText = imageElement.getAttribute("alt") || "image.png"; - const rawName = (assetPath || altText).split(/[?#]/)[0].split(/[/\\]/).pop() || "image.png"; + const frame = imageElement.closest?.(".markdown-image-frame") || imageElement; + const assetPath = imageElement.getAttribute("data-asset-path") || frame.getAttribute?.("data-asset-path") || imageElement.getAttribute("data-audio-src") || frame.getAttribute?.("data-audio-src") || imageElement.getAttribute("data-video-src") || frame.getAttribute?.("data-video-src") || imageElement.getAttribute("src") || ""; + const altText = imageElement.getAttribute("alt") || frame.getAttribute?.("data-audio-title") || frame.getAttribute?.("data-video-title") || "media"; + const rawName = (assetPath || altText).split(/[?#]/)[0].split(/[/\\]/).pop() || "media"; (async () => { try { @@ -1256,21 +1305,27 @@ export const MarkdownPreview = memo(function MarkdownPreviewContent({ srcPath = downloadSrc; } + const ext = (rawName.split(".").pop() || "").toLowerCase(); + const isAudioMedia = ["mp3", "wav", "webm", "ogg", "m4a", "aac", "flac"].includes(ext) && (imageElement.tagName === "AUDIO" || frame.classList?.contains("markdown-audio-card")); + const isVideoMedia = ["mp4", "webm", "ogg", "mov", "mkv"].includes(ext) && (imageElement.tagName === "VIDEO" || frame.classList?.contains("markdown-video-card")); + const customExportType = isAudioMedia ? (ext || "audio") : isVideoMedia ? (ext || "video") : (ext || "image"); + const mediaTypeLabel = isAudioMedia ? "audio" : isVideoMedia ? "video" : "image"; + const result = await runExport("media", { dataUrl, srcPath, filename: rawName, - customExportType: "image", + customExportType, category: "media", }); if (result?.success) { onNotify?.(`Downloaded ${result.filename} to Downloads folder`, "success"); } else { - onNotify?.(result?.error || "Failed to download image.", "error"); + onNotify?.(result?.error || `Failed to download ${mediaTypeLabel}.`, "error"); } } catch (err) { - onNotify?.(`Image download failed: ${err.message}`, "error"); + onNotify?.(`Download failed: ${err.message}`, "error"); } })(); return; @@ -1441,18 +1496,27 @@ export const MarkdownPreview = memo(function MarkdownPreviewContent({ return; } - const rawAsset = imageElement.getAttribute("data-asset-path") || imageElement.getAttribute("src") || ""; + const frame = imageElement.closest?.(".markdown-image-frame"); + const rawAsset = imageElement.getAttribute("data-asset-path") || + frame?.getAttribute?.("data-asset-path") || + imageElement.getAttribute("data-audio-src") || + frame?.getAttribute?.("data-audio-src") || + imageElement.getAttribute("data-video-src") || + frame?.getAttribute?.("data-video-src") || + imageElement.getAttribute("src") || ""; let assetPath = rawAsset.replace(/^https?:\/\/[^/]+\//i, ""); if (/^(?:file|app|atom):\/\//i.test(assetPath) || /^(?:[a-z]:\/|\/)/i.test(assetPath)) { assetPath = toComparableAssetPath(assetPath, basePath); } const isWorkspaceImage = Boolean(basePath && assetPath && !/^(https?:|data:|blob:)/i.test(assetPath)); + const isAudio = imageElement.tagName === "AUDIO" || Boolean(frame?.classList.contains("markdown-audio-card")); + const isVideo = imageElement.tagName === "VIDEO" || Boolean(frame?.classList.contains("markdown-video-card")); event?.preventDefault?.(); const bounds = imageElement.getBoundingClientRect(); menuSourceRef.current = imageElement; setContextMenu({ - kind: "image", + kind: isAudio ? "audio" : isVideo ? "video" : "image", x: Number.isFinite(x) ? x : event.clientX, y: Number.isFinite(y) ? y : event.clientY, keyboardOpened: !Number.isFinite(event?.clientX), @@ -1461,7 +1525,7 @@ export const MarkdownPreview = memo(function MarkdownPreviewContent({ isWorkspaceImage, src: imageElement.currentSrc || imageElement.src || "", assetPath, - imageLabel: imageElement.getAttribute("alt") || assetPath, + imageLabel: imageElement.getAttribute("alt") || frame?.getAttribute?.("data-audio-title") || frame?.getAttribute?.("data-video-title") || assetPath, }); }; @@ -1866,6 +1930,64 @@ export const MarkdownPreview = memo(function MarkdownPreviewContent({ } }; + const handleDownloadFromContextMenu = async () => { + if (!contextMenu?.assetPath && !contextMenu?.src) { + closeContextMenu(); + return; + } + const assetPath = contextMenu.assetPath || contextMenu.src; + const rawName = assetPath.split(/[?#]/)[0].split(/[/\\]/).pop() || (contextMenu.kind === "audio" ? "audio.webm" : "media"); + closeContextMenu(); + + try { + let downloadSrc = assetPath; + if (basePath && assetPath && !/^(https?:|data:|blob:)/i.test(assetPath)) { + try { + downloadSrc = (await readImage(basePath, assetPath)) || assetPath; + } catch { /* fallback */ } + } + if (!downloadSrc) { + downloadSrc = contextMenu.src || ""; + } + + let dataUrl; + let srcPath; + + if (typeof downloadSrc === "string" && downloadSrc.startsWith("data:")) { + dataUrl = downloadSrc; + } else if (typeof downloadSrc === "string" && (downloadSrc.startsWith("http") || downloadSrc.startsWith("blob:"))) { + const resp = await fetch(downloadSrc); + const blob = await resp.blob(); + dataUrl = await new Promise((resolve) => { + const reader = new FileReader(); + reader.onloadend = () => resolve(reader.result); + reader.readAsDataURL(blob); + }); + } else { + srcPath = downloadSrc; + } + + const ext = (rawName.split(".").pop() || "").toLowerCase(); + const isAudioMedia = ["mp3", "wav", "webm", "ogg", "m4a", "aac", "flac"].includes(ext) && contextMenu.kind === "audio"; + const customExportType = isAudioMedia ? (ext || "audio") : (contextMenu.kind === "video" ? (ext || "video") : (ext || "image")); + const result = await runExport("media", { + dataUrl, + srcPath, + filename: rawName, + customExportType, + category: "media", + }); + + if (result?.success) { + onNotify?.(`Downloaded ${result.filename} to Downloads folder`, "success"); + } else { + onNotify?.(result?.error || "Failed to download media.", "error"); + } + } catch (err) { + onNotify?.(`Download failed: ${err.message}`, "error"); + } + }; + const imageMenuActions = [ { key: "view-image", @@ -1874,6 +1996,13 @@ export const MarkdownPreview = memo(function MarkdownPreviewContent({ onSelect: viewImageFromMenu, disabled: false, }, + { + key: "download-image", + label: "Download image", + icon: , + onSelect: handleDownloadFromContextMenu, + disabled: false, + }, { key: "crop", label: "Edit image", @@ -1918,6 +2047,108 @@ export const MarkdownPreview = memo(function MarkdownPreviewContent({ }, ]; + const audioMenuActions = [ + { + key: "download-audio", + label: "Download audio", + icon: , + onSelect: handleDownloadFromContextMenu, + disabled: false, + }, + { + key: "copy-path", + label: "Copy audio path", + icon: , + onSelect: async () => { + if (contextMenu?.assetPath) { + await navigator.clipboard.writeText(contextMenu.assetPath); + onNotify?.(`Copied audio path: ${contextMenu.assetPath}`, "success"); + } + closeContextMenu(); + }, + disabled: false, + }, + { + key: "open-default", + label: "Open in default app", + icon: , + onSelect: () => { + if (basePath && contextMenu?.assetPath && typeof openMediaInDefaultApp === "function") { + openMediaInDefaultApp(basePath, contextMenu.assetPath).catch((err) => { + onNotify?.(err?.message || "Failed to open file in default app.", "error"); + }); + } + closeContextMenu(); + }, + disabled: false, + }, + { + key: "rename-audio", + label: "Rename audio", + icon: , + onSelect: handleRenameFromMenu, + disabled: replaceState.busy || !contextMenu?.isWorkspaceImage, + }, + { + key: "delete-audio", + label: "Delete audio", + icon: , + onSelect: handleDeleteFromMenu, + disabled: replaceState.busy || !contextMenu?.isWorkspaceImage, + }, + ]; + + const videoMenuActions = [ + { + key: "download-video", + label: "Download video", + icon: , + onSelect: handleDownloadFromContextMenu, + disabled: false, + }, + { + key: "copy-path", + label: "Copy video path", + icon: , + onSelect: async () => { + if (contextMenu?.assetPath) { + await navigator.clipboard.writeText(contextMenu.assetPath); + onNotify?.(`Copied video path: ${contextMenu.assetPath}`, "success"); + } + closeContextMenu(); + }, + disabled: false, + }, + { + key: "open-default", + label: "Open in default app", + icon: , + onSelect: () => { + if (basePath && contextMenu?.assetPath && typeof openMediaInDefaultApp === "function") { + openMediaInDefaultApp(basePath, contextMenu.assetPath).catch((err) => { + onNotify?.(err?.message || "Failed to open file in default app.", "error"); + }); + } + closeContextMenu(); + }, + disabled: false, + }, + { + key: "rename-video", + label: "Rename video", + icon: , + onSelect: handleRenameFromMenu, + disabled: replaceState.busy || !contextMenu?.isWorkspaceImage, + }, + { + key: "delete-video", + label: "Delete video", + icon: , + onSelect: handleDeleteFromMenu, + disabled: replaceState.busy || !contextMenu?.isWorkspaceImage, + }, + ]; + const diagramMenuActions = [ { key: "edit-diagram", @@ -1977,6 +2208,10 @@ export const MarkdownPreview = memo(function MarkdownPreviewContent({ ? textMenuActions : contextMenu?.kind === "diagram" ? diagramMenuActions + : contextMenu?.kind === "audio" + ? audioMenuActions + : contextMenu?.kind === "video" + ? videoMenuActions : imageMenuActions; const handleMenuKeyDown = (event) => { @@ -2289,7 +2524,7 @@ export const MarkdownPreview = memo(function MarkdownPreviewContent({ >
- {contextMenu?.kind === "diagram" ? "Diagram actions" : contextMenu?.kind === "text" ? "Text actions" : "Image actions"} + {contextMenu?.kind === "diagram" ? "Diagram actions" : contextMenu?.kind === "text" ? "Text actions" : contextMenu?.kind === "audio" ? "Audio actions" : contextMenu?.kind === "video" ? "Video actions" : "Image actions"}
{activeMenuActions.map((action, index) => (
)} - {showAssetLinker && ( -
-
- Insert Media From Workspace - -
- -
- - - - -
- -
- -
- - {assetUrl.trim() && !hasValidAssetUrl ? ( -

Use a valid http/https URL.

- ) : null} - - {assetsError &&

{assetsError}

} - {assetsLoading ?

Loading assets...

: null} - - {!assetsLoading && !filteredAssets.length ? ( -

No matching assets found.

- ) : ( -
- {filteredAssets.map((asset) => ( - - ))} -
- )} -
- )} + setShowAssetLinker(false)} + basePath={basePath} + availableAssets={availableAssets} + assetsLoading={assetsLoading} + assetsError={assetsError} + onInsert={(markdown) => { + insertTextAtCursor(value, onChange, `${markdown} `, textareaRef); + onNotify?.("Inserted into note.", "success"); + }} + onNotify={onNotify} + /> {showReferenceLinker && (
@@ -1707,13 +1664,6 @@ export function MarkdownToolbar({ ) : null}
)} -
)} + {audioRecorderOpen && ( +
+ setAudioRecorderOpen(false)} + onSaveSuccess={handleAudioRecordingSuccess} + initialMode="meeting" + /> +
+ )} ); } diff --git a/src/components/MediaAttachmentPicker.jsx b/src/components/MediaAttachmentPicker.jsx new file mode 100644 index 0000000..0e7040a --- /dev/null +++ b/src/components/MediaAttachmentPicker.jsx @@ -0,0 +1,813 @@ +import { useState, useEffect, useMemo, useRef, forwardRef } from "react"; +import { + Upload, + Music, + Film, + FileDigit, + MessageSquareText, + Image as ImageIcon, + FileText, + Search, + X, + LayoutGrid, + List, + Globe, + FolderOpen, + UploadCloud, + Paperclip, + Plus, +} from "lucide-react"; +import { AppSelect } from "./AppSelect"; +import { listImages, listDocuments, readImage } from "../services/electronService"; +import { insertMediaFromFile } from "../services/imageService"; +import { MEDIA_FILE_INPUT_ACCEPT } from "../utils/mediaTypeUtils"; +import { + getAssetMediaType, + decodePathForDisplay, + createMediaMarkdown, + toRelativeDocPath, + normalizeImagePathForMarkdown, + hasMarkdownExtension, +} from "../utils/markdownUtils"; + +function isValidHttpUrl(value) { + const trimmed = String(value || "").trim(); + if (!trimmed) return false; + try { + const parsed = new URL(trimmed); + return parsed.protocol === "http:" || parsed.protocol === "https:"; + } catch { + return false; + } +} + +export const MediaAttachmentPicker = forwardRef(function MediaAttachmentPicker( + { + isOpen, + onClose, + basePath, + availableAssets: propAvailableAssets, + assetsLoading: propAssetsLoading, + assetsError: propAssetsError, + onInsert, + onNotify, + }, + ref +) { + const fileInputRef = useRef(null); + const searchInputRef = useRef(null); + const [activeTab, setActiveTab] = useState("workspace"); // "workspace" | "web" | "upload" + const [viewMode, setViewMode] = useState("grid"); // "grid" | "list" + const [assetSearch, setAssetSearch] = useState(""); + const [assetFilter, setAssetFilter] = useState("all"); + const [linkText, setLinkText] = useState(""); + const [assetUrl, setAssetUrl] = useState(""); + const [internalAssets, setInternalAssets] = useState([]); + const [internalLoading, setInternalLoading] = useState(false); + const [internalError, setInternalError] = useState(""); + const [uploading, setUploading] = useState(false); + const [isDragOver, setIsDragOver] = useState(false); + const [selectedIndex, setSelectedIndex] = useState(-1); + const [selectedAsset, setSelectedAsset] = useState(null); + const [thumbnailMap, setThumbnailMap] = useState({}); + + const rawAvailableAssets = propAvailableAssets !== undefined ? propAvailableAssets : internalAssets; + const availableAssets = useMemo(() => { + return rawAvailableAssets.map((asset) => { + const rawNormalized = String(asset.path || asset.filePath || "").replace(/\\/g, "/").trim(); + const withoutQuery = rawNormalized.split(/[?#]/)[0]; + const parts = withoutQuery.split("/").filter(Boolean); + const rawFileName = parts.length ? parts[parts.length - 1] : rawNormalized; + const decodedFileName = decodePathForDisplay(rawFileName) || rawFileName; + + let folderDir = ""; + if (parts.length > 1) { + const dirParts = parts.slice(0, -1).filter((p) => p !== "."); + folderDir = dirParts.length ? `${dirParts.join("/")}/` : ""; + } + + const title = asset.type === "document" + ? (asset.title || asset.fileName || decodedFileName || "Untitled note").trim() + : (asset.title && asset.title !== rawNormalized && !asset.title.includes("/") + ? asset.title + : decodedFileName); + + let displayPath = asset.displayPath || folderDir || ""; + if (displayPath === title || displayPath === decodedFileName) { + displayPath = folderDir; + } + + return { + ...asset, + title, + displayPath: displayPath.replace(/^\.\//, ""), + }; + }); + }, [rawAvailableAssets]); + + const assetsLoading = propAssetsLoading !== undefined ? propAssetsLoading : internalLoading; + const [localError, setLocalError] = useState(""); + const assetsError = propAssetsError || internalError || localError; + + useEffect(() => { + if (!isOpen) { + setAssetSearch(""); + setLinkText(""); + setAssetUrl(""); + setLocalError(""); + setActiveTab("workspace"); + setSelectedIndex(-1); + setSelectedAsset(null); + setThumbnailMap({}); + return; + } + + searchInputRef.current?.focus(); + + if (propAvailableAssets !== undefined) return; + + if (!basePath) { + setInternalAssets([]); + setInternalError("Save or open a note file before linking workspace assets."); + return; + } + + let isCancelled = false; + setInternalLoading(true); + setInternalError(""); + + Promise.allSettled([ + listImages(basePath), + listDocuments(), + ]).then(([imagesResult, docsResult]) => { + if (isCancelled) return; + const mediaList = imagesResult.status === "fulfilled" && Array.isArray(imagesResult.value) + ? imagesResult.value.map((pathValue) => { + const rawNormalized = String(pathValue || "").replace(/\\/g, "/").trim(); + const withoutQuery = rawNormalized.split(/[?#]/)[0]; + const parts = withoutQuery.split("/").filter(Boolean); + const rawFileName = parts.length ? parts[parts.length - 1] : rawNormalized; + const decodedTitle = decodePathForDisplay(rawFileName) || rawFileName; + + let folderDir = ""; + if (parts.length > 1) { + const dirParts = parts.slice(0, -1).filter((p) => p !== "."); + folderDir = dirParts.length ? `${dirParts.join("/")}/` : ""; + } + + return { + type: "media", + path: pathValue, + fileName: rawFileName, + mediaType: getAssetMediaType(pathValue), + title: decodedTitle, + displayPath: folderDir, + }; + }) + : []; + + const currentPathLower = String(basePath || "").trim().toLowerCase(); + const docsList = docsResult.status === "fulfilled" && Array.isArray(docsResult.value) + ? docsResult.value + .filter((doc) => String(doc?.filePath || "").trim().toLowerCase() !== currentPathLower) + .map((doc) => { + const fileNameLower = String(doc.fileName || "").toLowerCase(); + const titleLower = String(doc.title || "").toLowerCase(); + const pathLower = String(doc.filePath || "").toLowerCase(); + const isTranscriptDoc = + fileNameLower.includes("transcript") || + titleLower.includes("transcript") || + pathLower.includes("transcript"); + + const title = (doc.title || doc.fileName || "Untitled note").trim(); + let displayPath = doc.displayPath || ""; + if (!displayPath || displayPath === title || displayPath === doc.fileName) { + const normPath = String(doc.filePath || "").replace(/\\/g, "/"); + const parts = normPath.split("/").filter(Boolean); + displayPath = parts.length > 1 ? `${parts.slice(0, -1).join("/")}/` : ""; + } + + return { + type: "document", + path: doc.filePath, + fileName: doc.fileName, + title, + displayPath, + mediaType: isTranscriptDoc ? "transcript" : "document", + }; + }) + : []; + + setInternalAssets([...mediaList, ...docsList]); + setInternalLoading(false); + }).catch((err) => { + if (isCancelled) return; + setInternalAssets([]); + setInternalError(err?.message || "Unable to load workspace assets."); + setInternalLoading(false); + }); + + return () => { + isCancelled = true; + }; + }, [isOpen, basePath, propAvailableAssets]); + + // Resolve disk thumbnails for images & videos via IPC + useEffect(() => { + let isCancelled = false; + if (!isOpen || !availableAssets.length) return; + + const visualMedia = availableAssets.filter( + (a) => a.type === "media" && (a.mediaType === "image" || a.mediaType === "video") + ); + + if (!visualMedia.length) return; + + Promise.allSettled( + visualMedia.map(async (asset) => { + try { + if ( + asset.path.startsWith("http://") || + asset.path.startsWith("https://") || + asset.path.startsWith("data:") + ) { + return { path: asset.path, src: asset.path }; + } + const src = await readImage(basePath || "", asset.path, { thumbnail: true }); + return { path: asset.path, src }; + } catch { + return { path: asset.path, src: null }; + } + }) + ).then((results) => { + if (isCancelled) return; + const newMap = {}; + for (const r of results) { + if (r.status === "fulfilled" && r.value?.src) { + newMap[r.value.path] = r.value.src; + } + } + setThumbnailMap((prev) => ({ ...prev, ...newMap })); + }); + + return () => { + isCancelled = true; + }; + }, [isOpen, availableAssets, basePath]); + + const hasValidAssetUrl = isValidHttpUrl(assetUrl.trim()); + + const filteredAssets = useMemo(() => { + return availableAssets.filter((asset) => { + if (assetFilter !== "all" && assetFilter !== "All Media") { + const f = assetFilter.toLowerCase(); + if (f === "document" || f === "documents" || f === "notes") { + if (asset.type !== "document" && asset.mediaType !== "document") return false; + } else if (f === "transcript" || f === "transcripts") { + const isTranscript = + asset.mediaType === "transcript" || + (asset.title && asset.title.toLowerCase().includes("transcript")) || + (asset.fileName && asset.fileName.toLowerCase().includes("transcript")) || + (asset.path && asset.path.toLowerCase().includes("transcript")); + if (!isTranscript) return false; + } else { + const norm = f.replace(/s$/, ""); + if (asset.mediaType !== norm && asset.mediaType !== f) { + return false; + } + } + } + + const search = assetSearch.trim().toLowerCase(); + if (!search) return true; + const label = [asset.title, asset.path, asset.fileName, asset.displayPath] + .filter(Boolean) + .join(" ") + .toLowerCase(); + return label.includes(search); + }); + }, [availableAssets, assetFilter, assetSearch]); + + // Reset selected asset & index when filter or tab changes + useEffect(() => { + setSelectedIndex(-1); + setSelectedAsset(null); + }, [assetFilter, assetSearch, activeTab]); + + const handleProcessFile = async (file) => { + if (!file) return; + + setUploading(true); + try { + const { mediaPath, altText } = await insertMediaFromFile(file); + const markdown = createMediaMarkdown(altText, mediaPath); + onInsert?.(markdown, altText); + onNotify?.("Media uploaded and inserted.", "success"); + onClose?.(); + } catch (error) { + console.error("Upload failed:", error); + setLocalError(error?.message || "Failed to upload file."); + onNotify?.(error?.message || "Failed to upload file.", "error"); + } finally { + setUploading(false); + } + }; + + const handleFileInputChange = async (event) => { + const file = event.target.files?.[0]; + await handleProcessFile(file); + if (event.target) event.target.value = ""; + }; + + const handleDrop = async (event) => { + event.preventDefault(); + setIsDragOver(false); + const file = event.dataTransfer.files?.[0]; + if (file) { + await handleProcessFile(file); + } + }; + + const handleDragOver = (event) => { + event.preventDefault(); + setIsDragOver(true); + }; + + const handleDragLeave = (event) => { + event.preventDefault(); + setIsDragOver(false); + }; + + if (!isOpen) return null; + + const handleLinkAssetFromUrl = () => { + const rawUrl = assetUrl.trim(); + if (!isValidHttpUrl(rawUrl)) { + setLocalError("Use a valid http/https URL."); + return; + } + const label = linkText.trim() || "Web link"; + const markdown = `[${label}](${rawUrl})`; + onInsert?.(markdown, label); + onClose?.(); + }; + + const handleSelectAsset = (asset, idx) => { + setSelectedAsset(asset); + if (typeof idx === "number") { + setSelectedIndex(idx); + } + }; + + const handleInsertAsset = (asset) => { + const targetAsset = asset || selectedAsset; + if (!targetAsset) return; + + if (targetAsset.type === "document") { + const text = linkText.trim() || targetAsset.title || targetAsset.fileName || "Linked note"; + let relativePath = toRelativeDocPath(basePath, targetAsset.path); + if (relativePath && !hasMarkdownExtension(relativePath) && hasMarkdownExtension(targetAsset.fileName)) { + relativePath = `${relativePath}.md`; + } + const normalizedPath = normalizeImagePathForMarkdown(relativePath); + if (!normalizedPath || normalizedPath === "./" || normalizedPath === ".") { + setLocalError("Choose a different note. Linking current note is not supported."); + return; + } + onInsert?.(`[${text}](${normalizedPath})`, text); + onClose?.(); + return; + } + + const label = linkText.trim() || targetAsset.title || targetAsset.fileName || "Media"; + const markdown = createMediaMarkdown(label, targetAsset.path); + onInsert?.(markdown, label); + onClose?.(); + }; + + const handleKeyDown = (event) => { + if (event.key === "Escape") { + onClose?.(); + return; + } + + if (activeTab !== "workspace" || !filteredAssets.length) return; + + if (event.key === "ArrowDown") { + event.preventDefault(); + const next = selectedIndex < 0 ? 0 : (selectedIndex + 1) % filteredAssets.length; + setSelectedIndex(next); + setSelectedAsset(filteredAssets[next]); + } else if (event.key === "ArrowUp") { + event.preventDefault(); + const prev = selectedIndex <= 0 ? filteredAssets.length - 1 : selectedIndex - 1; + setSelectedIndex(prev); + setSelectedAsset(filteredAssets[prev]); + } else if (event.key === "Enter") { + const target = selectedAsset || (selectedIndex >= 0 ? filteredAssets[selectedIndex] : null); + if (target) { + event.preventDefault(); + handleInsertAsset(target); + } + } + }; + + const getMediaIcon = (mediaType, type) => { + if (type === "document") return ; + switch (mediaType) { + case "image": return ; + case "audio": return ; + case "video": return ; + case "transcript": return ; + case "pdf": return ; + default: return ; + } + }; + + const renderCardVisual = (asset) => { + const resolvedSrc = thumbnailMap[asset.path]; + const fallbackSrc = asset.path.startsWith("http") + ? asset.path + : `file://${asset.path.replace(/\\/g, "/")}`; + const imageSrc = resolvedSrc || fallbackSrc; + + if (asset.mediaType === "image") { + return ( + <> + {asset.title} { + e.target.style.display = "none"; + const fallback = e.target.parentElement?.querySelector(".media-picker-card-preview-fallback"); + if (fallback) fallback.style.display = "flex"; + }} + /> +
+ +
+ + ); + } + + if (asset.mediaType === "video") { + if (resolvedSrc) { + return ( +