From 827b2be115bef3089478e69f4fb72b7b375b7259 Mon Sep 17 00:00:00 2001 From: Bikash Panda Date: Sat, 26 Sep 2026 22:20:04 +0530 Subject: [PATCH 1/8] Added Audio recording feature --- CHANGELOG.md | 18 + README.md | 3 + THIRD_PARTY_NOTICES.txt | 2 +- docs/feature-reference.md | 25 +- docs/keyboard-shortcuts.md | 2 + docs/settings-reference.md | 13 + docs/user-guide.md | 25 + electron/lib/core/windowLifecycle.cjs | 14 +- electron/lib/media/imageMedia.cjs | 139 ++++- electron/lib/media/pdf.cjs | 2 - electron/preload.cjs | 2 + src/App.jsx | 2 +- src/components/AISettings.jsx | 533 +++++++++++------ src/components/AudioRecorderBar.jsx | 421 ++++++++++++++ src/components/DocumentDetail.jsx | 2 - src/components/ExportPdfModal.jsx | 1 - src/components/LandingQuickCaptureBar.jsx | 315 +++++++++++ src/components/MarkdownPreview.jsx | 241 +++++++- src/components/MarkdownToolbar.jsx | 86 ++- src/components/RecordingOverlayApp.jsx | 109 ++-- src/components/WebViewPreview.jsx | 55 +- src/components/WorkspaceDiagramsMediaPage.jsx | 534 +++++++++++++++++- src/components/layout/AppSubpageViews.jsx | 1 + src/components/layout/LandingView.jsx | 4 + src/components/layout/TitleBar.jsx | 14 +- src/services/electron/mediaService.js | 36 ++ src/services/sttService.js | 496 ++++++++++++++++ src/services/workspaceMediaService.js | 60 +- src/styles/AISettings.css | 65 ++- src/styles/components.css | 196 +++++++ src/styles/editor.css | 15 + src/tests/components/ExportPdfModal.test.jsx | 3 +- .../services/workspaceMediaService.test.js | 92 +++ src/tests/utils/audioTools.test.js | 61 ++ src/tests/utils/notificationUtils.test.js | 70 +++ src/utils/audioExtractor.js | 189 +++++++ src/utils/audioMixer.js | 81 +++ src/utils/mediaTypeUtils.js | 6 + src/utils/mediaUtils.js | 16 +- src/utils/notificationUtils.js | 39 ++ src/utils/renderUtils.js | 54 +- src/workers/sttWorker.js | 105 ++++ tests/eight_stage_pipeline.test.js | 2 +- tests/graph.test.js | 17 +- tests/mcp/mcpServer.spec.js | 7 +- tests/semantic_quality_precision.test.js | 2 +- .../test2/architecture.excalidraw | 1 - 47 files changed, 3863 insertions(+), 313 deletions(-) create mode 100644 src/components/AudioRecorderBar.jsx create mode 100644 src/components/LandingQuickCaptureBar.jsx create mode 100644 src/services/sttService.js create mode 100644 src/tests/utils/audioTools.test.js create mode 100644 src/tests/utils/notificationUtils.test.js create mode 100644 src/utils/audioExtractor.js create mode 100644 src/utils/audioMixer.js create mode 100644 src/utils/notificationUtils.js create mode 100644 src/workers/sttWorker.js delete mode 100644 tests/tmp_parity_workspace/test2/architecture.excalidraw diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bfb9993..677538ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,24 @@ All notable documentation and user-facing behavior changes are tracked in this f ### Added +- Added **Dual-Stream Audio Capture & Meeting Recording**: + - Captures microphone and Windows desktop/system audio loopback simultaneously using Web Audio API AudioContext mixer (`src/utils/audioMixer.js`). + - Supports 3 capture modes: Meeting Mode (Mic + System Audio), Microphone Only, and System Audio Only. + - Live audio waveform visualizer canvas and recording timer with pause/resume support (`AudioRecorderBar.jsx`). + - Keyboard shortcut `Alt + V` to toggle audio/meeting recorder in the editor toolbar. +- Added **Speech-to-Text (STT) Transcription Service & Model Settings**: + - Offline on-device transcription via Local ONNX Whisper (`whisper-tiny.en`, `whisper-base.en`, `whisper-small`). + - High-speed cloud transcription via Groq (`whisper-large-v3`) and OpenAI (`whisper-1`). + - Dedicated **Speech-to-Text** subtab in AI Settings (`AISettings.jsx`) for selecting engines, local models, capture defaults, and languages. + - Transcripts saved as structured companion `.json` files alongside audio in `media/audio/*.json` with timestamps, speaker attributions, key points, and action items. +- Added **Landing Page Quick Capture Bar**: + - Integrated header toolbar on the landing dashboard (`LandingQuickCaptureBar.jsx`) for one-click Screen Snip, Desktop Recording, and Audio/Meeting Recording. + - Saves captured media directly to workspace media storage with zero note modifications. + - Notification toast provides direct 1-click `[Open Gallery]` action. +- Added **Workspace Media Gallery Unused Media Filter & Audio Player**: + - Physical disk scanner (`media:list-disk-assets` IPC) discovers files in `media/`, `assets/`, and `images/`. + - New `⚠️ Unused / Orphans` filter tab and header stats indicator isolating unreferenced media files (`referenceCount === 0`). + - Interactive audio player preview (`AudioPlayerPreviewItem`) and orphan action buttons (`Copy Markdown Embed`). - Added **Multiple Developer-Friendly Fonts Support** (`View → Font`). - Choose between 5 curated IDE/developer-friendly typefaces: Inter (Default), JetBrains Mono, Fira Code, Cascadia Code, and Source Code Pro. - Changes cascade in real time across all app labels, navigation items, buttons, dialogs, and controls. diff --git a/README.md b/README.md index 1822c0e4..ffd7bc8c 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,9 @@ Notely is built with Electron + React and is designed for project notes, meeting - **Regex search** with validation and pattern matching for advanced queries. - **Code-aware search** to find patterns inside code blocks only. - Insert common Markdown snippets from the toolbar. +- Record audio notes and full meetings with dual-stream audio capture (Microphone + Windows System Audio loopback) with automatic Speech-to-Text (STT) transcription powered by offline local ONNX Whisper or ultra-fast cloud providers (Groq whisper-large-v3, OpenAI whisper-1). +- Quick capture directly from the landing dashboard with one-click screen snip, desktop recording, and meeting audio capture that saves directly to workspace media without altering notes until you embed them. +- Explore and audit all diagrams, media, audio, and documents in the **Workspace Diagrams & Media Gallery**, featuring physical disk scanning, unreferenced orphan detection, and an **Unused Media** filter with audio playback and transcript preview. - Edit Markdown tables inline with a focused grid editor (row/column add/remove, alignment controls, and compact action chips). - Browse, annotate, optimize, and manage linked media. - Open note files in VS Code or the system default app. diff --git a/THIRD_PARTY_NOTICES.txt b/THIRD_PARTY_NOTICES.txt index 53daccca..c34f3f5f 100644 --- a/THIRD_PARTY_NOTICES.txt +++ b/THIRD_PARTY_NOTICES.txt @@ -1,7 +1,7 @@ THIRD-PARTY SOFTWARE NOTICES AND INFORMATION Notely incorporates components from the projects listed below. -This file was generated automatically on 2026-09-24 by +This file was generated automatically on 2026-09-26 by scripts/generate-notices.cjs. Do not edit by hand. Total third-party packages: 189 diff --git a/docs/feature-reference.md b/docs/feature-reference.md index 0235bb9e..e2a53306 100644 --- a/docs/feature-reference.md +++ b/docs/feature-reference.md @@ -79,9 +79,12 @@ During transfer: Open via **Workspace -> Diagrams & Media Gallery** (`Ctrl/Cmd + Alt + M`) or Command Palette: - Catalogs all used diagrams (inline Mermaid, Draw.io, Excalidraw), images, videos, audio, and PDF documents in the workspace. +- **Physical Disk Scanner & Orphan Detection**: Scans workspace folders (`media/`, `assets/`, `images/`) to identify files present on disk that have zero note references. +- **Unused Media Filter**: Quickly isolate unreferenced assets with the `⚠️ Unused / Orphans` filter tab. +- **Audio & Media Preview**: Embedded audio players, video thumbnails, and transcript drawer with key points and action items. - Reuses Knowledge Graph layout aesthetics with collapsible category filters, search, and stat indicators. - Inspect details for any item with high-res/diagram live previews, file paths, and exact note references with line numbers and snippets. -- Direct "Open Note" navigation into the editor. +- Direct "Open Note" navigation and "Copy Markdown Link" actions. ## 2. Editor and Writing Experience @@ -328,6 +331,26 @@ Notely supports area-based screen capture and full screen/window video recording - **Auto-Save & Linking**: Saves `.webm` files under `media/recordings/` and inserts `![Screen Recording](media/recordings/*.webm)` into the note. - **Note Preview & Modal Player**: Renders video links as interactive thumbnail cards with `▶ Play Video` badges and frame decoding. Clicking opens a full-screen playback modal with Download and Copy Path options. +### Audio & Meeting Recording with Speech-to-Text + +Record voice notes or entire conference calls (Zoom, Google Meet, Teams, YouTube) directly into Notely with synchronized audio and transcription. + +- **Dual-Stream Audio Capture**: + - **Meeting Mode**: Simultaneously captures your local microphone and Windows desktop/system audio loopback, combining them into a single stereo track with Web Audio API mixer. + - **Microphone Only**: Dedicated voice memo recording. + - **System Audio Only**: Captures speakers/audio loopback without mic. +- **Speech-to-Text (STT) Transcription**: + - **Local ONNX Whisper (Offline)**: Runs locally in-app via WebAssembly/ONNX runtime (`whisper-tiny.en`, `whisper-base.en`, `whisper-small`). No internet connection or cloud API keys required. + - **Cloud Whisper**: High-speed cloud transcription using Groq (`whisper-large-v3`) or OpenAI (`whisper-1`) with API keys configured in AI Settings. +- **Transcript Storage**: + - Audio files are stored under `media/audio/*.webm`. + - Transcripts are saved alongside audio in `media/audio/*.json` containing segmented timestamps, speaker attributions, key points, and action items. +- **Workflow Differences**: +- **Media Gallery Integration & Transcripts Filter**: + - The **Diagrams, Media & PDFs** gallery (`Workspace -> Diagrams, Media & PDFs`) includes a dedicated **Transcripts** filter category alongside Diagrams, Images, Videos, Audio, PDFs, and Documents. + - Selecting a transcript in the gallery opens a rich inspector showing the AI executive summary, timestamped speaker segments, and a one-click button to copy the full transcript text. + - **On-Demand Transcript Generation**: Any existing audio or video recording in the Media Gallery has a **Generate AI Transcript** button (`✨`). Clicking it decodes the media track, resamples it to 16kHz mono Float32, runs Whisper (Local ONNX or Cloud Groq/OpenAI based on your AI Settings), and saves a companion transcript JSON file in `media/audio/`. + ### Media preview tools Use zoom and media-aware preview controls to inspect assets. diff --git a/docs/keyboard-shortcuts.md b/docs/keyboard-shortcuts.md index 75d412b1..a6a73f25 100644 --- a/docs/keyboard-shortcuts.md +++ b/docs/keyboard-shortcuts.md @@ -55,6 +55,8 @@ Use shortcuts to navigate and edit notes quickly across Notely. | Toggle Outline Panel | `Ctrl/Cmd + Alt + L` | Editor | | Open Reference Note | `Ctrl/Cmd + Shift + K` | Editor | | Insert Reference Link | `Ctrl/Cmd + Shift + L` | Editor | +| Capture Screen Snip | `Ctrl/Cmd + Shift + S` | Editor | +| Toggle Audio / Meeting Recorder | `Alt + V` | Editor | ## Search & Find Panel diff --git a/docs/settings-reference.md b/docs/settings-reference.md index 20578c00..cad329b9 100644 --- a/docs/settings-reference.md +++ b/docs/settings-reference.md @@ -154,6 +154,19 @@ Open **AI -> AI Settings**. Use lower temperature for predictable output. Use higher temperature for brainstorming or variation. +### Speech-to-Text (STT) setup + +Open the **Speech-to-Text** tab in AI Settings: + +- **Transcription Engine**: + - `Local ONNX Whisper (Offline, In-Browser / WebAssembly)`: default offline engine; runs private on-device transcription. + - `Groq Cloud Whisper`: utilizes Groq's high-throughput `whisper-large-v3` for sub-second transcriptions using your configured Groq API key. + - `OpenAI Cloud Whisper`: utilizes OpenAI's `whisper-1` model using your configured OpenAI API key. +- **Local Whisper Model**: Select from `whisper-tiny.en` (~40MB, fastest), `whisper-base.en` (~140MB, balanced), or `whisper-small` (~460MB, multilingual). +- **Default Audio Capture Mode**: Choose default mode when launching recorder (`Meeting: Mic + System Audio`, `Microphone Only`, `System Audio Only`). +- **Primary Spoken Language**: Specify expected audio language (or `Auto-Detect`). +- **Auto-transcribe toggle**: Automatically triggers Speech-to-Text generation as soon as an audio or meeting recording is stopped. + ### Data and privacy controls - Shows where AI-related app data is stored on your device diff --git a/docs/user-guide.md b/docs/user-guide.md index 1f00e632..d9f5286d 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -177,3 +177,28 @@ Customize app look and feel to fit your workflow: - **Help -> Help Center** (`F1`) for in-app help. - **Help -> Keyboard Shortcuts** (`Ctrl/Cmd + /`) for key bindings. - **Help -> About Notely** for app version details. + +## 14. Record Meetings, Capture Audio & Manage Media Gallery + +### Record a Meeting with Dual-Stream Audio (Mic + System Loopback) + +1. Open any note or press `Alt + V` (or click the 🎙️ **Record Audio** icon on the editor toolbar). +2. Choose **Meeting Mode** to capture both your voice and speakers/participants (Zoom, Google Meet, Teams, YouTube). +3. Click **Start Recording**. A live waveform visualizer and recording timer start. +4. When the discussion finishes, click **Stop & Transcribe**. +5. Notely transcribes the discussion with Whisper (local offline ONNX or cloud), saves the `.webm` audio and `.json` transcript to `media/audio/`, and automatically inserts the audio player and transcript block at your cursor. + +### Quick Capture from the Landing Dashboard + +1. On the landing dashboard, use the **Quick Capture** toolbar: + - 📸 **Snip Screen**: Capture any desktop region to an image file. + - 🎥 **Record Screen**: Select a screen or application window to record. + - 🎙️ **Record Audio**: Capture quick voice memos or meeting audio. +2. Captured media is stored directly in workspace media storage without modifying any notes. +3. A notification toast appears with a 1-click **[Open Gallery]** button to inspect the asset immediately. + +### Find and Embed Unused / Orphaned Media + +1. Open **Workspace -> Diagrams & Media Gallery** (`Ctrl/Cmd + Alt + M`). +2. Click the **⚠️ Unused / Orphans** filter tab in the left sidebar or the header stats pill. +3. Inspect unreferenced files, listen to audio recordings, or click **Copy Markdown Embed** to link them into your notes. diff --git a/electron/lib/core/windowLifecycle.cjs b/electron/lib/core/windowLifecycle.cjs index 826f7f4c..8ccef898 100644 --- a/electron/lib/core/windowLifecycle.cjs +++ b/electron/lib/core/windowLifecycle.cjs @@ -345,12 +345,12 @@ function createWindowLifecycle(deps) { const isDev = Boolean(rendererUrl); const scriptSrc = isDev - ? "script-src 'self' 'unsafe-inline' 'unsafe-eval'" - : "script-src 'self' 'unsafe-inline'"; + ? "script-src 'self' 'unsafe-inline' 'unsafe-eval' 'wasm-unsafe-eval' blob: https://cdn.jsdelivr.net" + : "script-src 'self' 'unsafe-inline' 'wasm-unsafe-eval' blob: https://cdn.jsdelivr.net"; const connectSrc = isDev - ? "connect-src 'self' https://api.languagetool.org https://esm.sh https://unpkg.com ws: http://127.0.0.1:* http://localhost:*" - : "connect-src 'self' https://api.languagetool.org https://esm.sh https://unpkg.com"; + ? "connect-src 'self' data: blob: https://huggingface.co https://*.huggingface.co https://*.hf.co https://cdn.jsdelivr.net https://api.languagetool.org https://esm.sh https://unpkg.com ws: http://127.0.0.1:* http://localhost:*" + : "connect-src 'self' data: blob: https://huggingface.co https://*.huggingface.co https://*.hf.co https://cdn.jsdelivr.net https://api.languagetool.org https://esm.sh https://unpkg.com"; return [ "default-src 'self'", @@ -360,7 +360,7 @@ function createWindowLifecycle(deps) { "media-src 'self' data: blob: file:", "font-src 'self' https://fonts.gstatic.com https://esm.sh https://unpkg.com data:", connectSrc, - "worker-src 'self' blob:", + "worker-src 'self' blob: data: https://cdn.jsdelivr.net", "child-src 'self' blob: data:", "object-src 'none'", "base-uri 'self'", @@ -462,7 +462,8 @@ function createWindowLifecycle(deps) { contextIsolation: true, nodeIntegration: false, sandbox: true, - webviewTag: false + webviewTag: false, + backgroundThrottling: false, } }; @@ -616,6 +617,7 @@ function createWindowLifecycle(deps) { nodeIntegration: false, sandbox: true, webviewTag: false, + backgroundThrottling: false, }, }); diff --git a/electron/lib/media/imageMedia.cjs b/electron/lib/media/imageMedia.cjs index 335e5bc3..f52fd2c3 100644 --- a/electron/lib/media/imageMedia.cjs +++ b/electron/lib/media/imageMedia.cjs @@ -226,6 +226,19 @@ function buildPdfExportHtml({ title, markdownContent, baseHref, sourceDir, downs let annotation = null; if (srcIndex >= 0) { const rawSrc = String(tokens[idx].attrs[srcIndex][1] || "").trim(); + const isExplicitAudio = /\.(mp3|wav|m4a|aac|flac|wma)(\?|#|$)/i.test(rawSrc); + const isAudioDir = /[/\\]audio[/\\]/i.test(rawSrc); + const isAudioNamed = /(recording|voice|meeting|mic|audio).*?\.(webm|ogg)$/i.test(rawSrc); + const isAudioAlt = /(audio|voice)/i.test(tokens[idx].content || tokens[idx].attrGet("alt") || ""); + const isAudio = isExplicitAudio || (isAudioDir && /\.(webm|ogg|wav|mp3|m4a|aac|flac)(\?|#|$)/i.test(rawSrc)) || (isAudioNamed && !rawSrc.includes("screen") && !rawSrc.includes("rec_")) || (isAudioAlt && !rawSrc.includes("screen") && /\.(webm|ogg)(\?|#|$)/i.test(rawSrc)); + + if (isAudio) { + const altText = tokens[idx].content || tokens[idx].attrGet("alt") || "Audio Recording"; + const fileName = rawSrc.split(/[?#]/)[0].split(/[/\\]/).pop() || "recording"; + const ext = fileName.split(".").pop()?.toUpperCase() || "AUDIO"; + return `
🎙️
${escapeCodeHtml(altText)}${escapeCodeHtml(fileName)} (${escapeCodeHtml(ext)})
`; + } + const isVideo = /\.(webm|mp4|ogg|mov|mkv|avi|m4v)(\?|#|$)/i.test(rawSrc); if (isVideo) { const altText = tokens[idx].content || tokens[idx].attrGet("alt") || "Video Recording"; @@ -780,6 +793,125 @@ registerTrustedHandler("video:save", (_event, payload) => { return `media/recordings/${safeFileName}`; }); +registerTrustedHandler("audio:save", (_event, payload) => { + const { fileName, base64Data, transcript } = payload || {}; + const safeFileName = path.basename(fileName || "transcript.json").replace(/[<>:"/\\|?*]+/g, "-"); + const audioDir = path.join(getNotesRoot(), "media", "audio"); + ensureDir(audioDir); + + // Handle standalone companion transcript saving + if (transcript && !base64Data) { + const transcriptFileName = safeFileName.endsWith(".json") ? safeFileName : safeFileName.replace(/\.[^.]+$/, "") + ".json"; + const transcriptPath = path.join(audioDir, transcriptFileName); + const content = typeof transcript === "string" ? transcript : JSON.stringify(transcript, null, 2); + fs.writeFileSync(transcriptPath, content, "utf8"); + return { + audioPath: null, + transcriptPath: `media/audio/${transcriptFileName}`, + fileName: transcriptFileName, + }; + } + + if (!base64Data || typeof base64Data !== "string" || !base64Data.includes(",")) { + throw new Error("Invalid audio payload."); + } + + const targetPath = path.join(audioDir, safeFileName); + if (!filePathWithin(getNotesRoot(), targetPath)) { + throw new Error("Invalid audio path."); + } + + const buffer = Buffer.from(base64Data.split(",")[1], "base64"); + if (!buffer.length) { + throw new Error("Audio data is empty."); + } + fs.writeFileSync(targetPath, buffer); + + let transcriptRelPath = null; + if (transcript) { + const transcriptFileName = safeFileName.replace(/\.[^.]+$/, "") + ".json"; + const transcriptPath = path.join(audioDir, transcriptFileName); + const content = typeof transcript === "string" ? transcript : JSON.stringify(transcript, null, 2); + fs.writeFileSync(transcriptPath, content, "utf8"); + transcriptRelPath = `media/audio/${transcriptFileName}`; + } + + return { + audioPath: `media/audio/${safeFileName}`, + transcriptPath: transcriptRelPath, + fileName: safeFileName, + }; +}); + +registerTrustedHandler("media:list-disk-assets", () => { + const notesRoot = getNotesRoot(); + if (!notesRoot || !fs.existsSync(notesRoot)) { + return []; + } + + const scanDirs = [ + path.join(notesRoot, "media"), + path.join(notesRoot, "assets"), + path.join(notesRoot, "images"), + ]; + + const results = []; + const visitedPaths = new Set(); + + for (const rootDir of scanDirs) { + if (!fs.existsSync(rootDir)) continue; + + const queue = [rootDir]; + while (queue.length > 0) { + const currentDir = queue.shift(); + let entries; + try { + entries = fs.readdirSync(currentDir, { withFileTypes: true }); + } catch { + continue; + } + + for (const entry of entries) { + if (entry.name.startsWith(".")) continue; + if (entry.name === THUMBNAIL_DIR_NAME || entry.name === ORIGINAL_IMAGE_DIR_NAME) continue; + + const fullPath = path.join(currentDir, entry.name); + if (entry.isDirectory()) { + queue.push(fullPath); + } else if (entry.isFile()) { + const relFromRoot = path.relative(notesRoot, fullPath).replace(/\\/g, "/"); + if (visitedPaths.has(relFromRoot.toLowerCase())) continue; + visitedPaths.add(relFromRoot.toLowerCase()); + + const ext = path.extname(entry.name).slice(1).toLowerCase(); + // Skip internal metadata/temporary files + if (["tmp", "crswap", "bak"].includes(ext)) continue; + + let size = 0; + let mtime = null; + try { + const stat = fs.statSync(fullPath); + size = stat.size; + mtime = stat.mtime.toISOString(); + } catch { + // Ignore stat read failure for transient files + } + + results.push({ + path: relFromRoot, + name: entry.name, + ext, + size, + mtime, + }); + } + } + } + } + + return results; +}); + registerTrustedHandler("images:download", async (event, payload) => { const { getExportManager } = require("../export/ExportManager.cjs"); const exportManager = getExportManager(); @@ -1141,6 +1273,8 @@ registerTrustedHandler("images:read", (_event, payload) => { ".wav": "audio/wav", ".ogg": "audio/ogg", ".m4a": "audio/mp4", + ".aac": "audio/aac", + ".flac": "audio/flac", ".pdf": "application/pdf", ".doc": "application/msword", ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", @@ -1160,7 +1294,10 @@ registerTrustedHandler("images:read", (_event, payload) => { ".7z": "application/x-7z-compressed", ".rar": "application/vnd.rar" }; - const mimeType = mimeMap[ext] || "application/octet-stream"; + let mimeType = mimeMap[ext] || "application/octet-stream"; + if (ext === ".webm" && (/[/\\]audio[/\\]/i.test(fileToRead) || /(recording|voice|mic|audio)/i.test(path.basename(fileToRead)))) { + mimeType = "audio/webm"; + } const buffer = fs.readFileSync(fileToRead); return `data:${mimeType};base64,${buffer.toString("base64")}`; }); diff --git a/electron/lib/media/pdf.cjs b/electron/lib/media/pdf.cjs index 21434c31..00226c21 100644 --- a/electron/lib/media/pdf.cjs +++ b/electron/lib/media/pdf.cjs @@ -1,5 +1,3 @@ -const path = require("node:path"); - function sanitizePdfMarkdown(markdown) { const source = String(markdown || ""); diff --git a/electron/preload.cjs b/electron/preload.cjs index 9446b97e..af3cb7ca 100644 --- a/electron/preload.cjs +++ b/electron/preload.cjs @@ -145,6 +145,8 @@ contextBridge.exposeInMainWorld("notesApi", { captureCurrentDisplay: () => ipcRenderer.invoke("screen:capture-current-display"), getDesktopSources: () => ipcRenderer.invoke("screen:get-sources"), saveVideo: (payload) => ipcRenderer.invoke("video:save", payload), + saveAudio: (payload) => ipcRenderer.invoke("audio:save", payload), + listDiskMediaAssets: () => ipcRenderer.invoke("media:list-disk-assets"), minimizeMainWindow: () => ipcRenderer.invoke("window:minimize-main"), restoreMainWindow: () => ipcRenderer.invoke("window:restore-main"), openRecordingOverlay: () => ipcRenderer.invoke("window:open-recording-overlay"), diff --git a/src/App.jsx b/src/App.jsx index dfa2c1bf..8e541098 100644 --- a/src/App.jsx +++ b/src/App.jsx @@ -302,7 +302,7 @@ export default function App() { useEffect(() => { const handleToast = (e) => { if (e.detail && e.detail.message) { - notify(e.detail.message, e.detail.type || "info"); + notify(e.detail.message, e.detail.type || "info", e.detail.action || null); } }; window.addEventListener("app:toast", handleToast); diff --git a/src/components/AISettings.jsx b/src/components/AISettings.jsx index 560db758..1e77141b 100644 --- a/src/components/AISettings.jsx +++ b/src/components/AISettings.jsx @@ -1,10 +1,22 @@ import React, { useEffect, useState } from 'react'; -import { Save, Trash2, Zap, AlertCircle, Eye, EyeOff, Download, Database } from 'lucide-react'; +import { Save, Trash2, Zap, AlertCircle, Eye, EyeOff, Download, Database, Mic, Cpu, Network } from 'lucide-react'; import AppInput from './AppInput'; import AppSelect from './AppSelect'; import "../styles/AISettings.css"; import OverlayDialog from './OverlayDialog'; import KnowledgeGraphSettings from './KnowledgeGraphSettings'; +import { + getSTTPreferences, + setSTTPreferences, + checkLocalWhisperModelStatus, + preDownloadLocalWhisperModel, + deleteLocalWhisperModel, +} from '../services/sttService'; +import { + showSuccessToast, + showErrorToast, + showInfoToast, +} from '../utils/notificationUtils'; import { aiClearData, aiGetApiKey, @@ -61,6 +73,47 @@ export const AISettingsContent = ({ _onClose }) => { const [activeSubTab, setActiveSubTab] = useState("providers"); const [modelStatus, setModelStatus] = useState({ downloaded: false, isDownloading: false, progress: 0 }); + const [sttSettings, setSttSettings] = useState({ + engine: "local-onnx", + localModel: "onnx-community/whisper-tiny.en", + language: "auto", + defaultSourceMode: "meeting", + autoTranscribe: true, + }); + const [whisperModelStatus, setWhisperModelStatus] = useState({ + downloaded: false, + isDownloading: false, + progress: 0, + statusText: '', + }); + + useEffect(() => { + getSTTPreferences().then((res) => { + if (res) setSttSettings(res); + }).catch(() => {}); + }, []); + + // Check whether selected local Whisper model is downloaded/cached + useEffect(() => { + let active = true; + const checkStatus = async () => { + if (sttSettings.engine !== "local-onnx") return; + try { + const res = await checkLocalWhisperModelStatus(sttSettings.localModel); + if (active) { + setWhisperModelStatus(prev => ({ + ...prev, + downloaded: Boolean(res.downloaded), + cachedFilesCount: res.cachedFilesCount, + })); + } + } catch (err) { + console.warn('Error checking Whisper model status:', err); + } + }; + checkStatus(); + return () => { active = false; }; + }, [sttSettings.localModel, sttSettings.engine]); useEffect(() => { const loadModelStatus = async () => { @@ -353,9 +406,6 @@ export const AISettingsContent = ({ _onClose }) => { Enable AI Subsystem - - Toggle the global switch to enable or disable all background AI services, embeddings, and graph extraction. -