diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 7497f352dc..26fa32c460 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -42,8 +42,9 @@ use auth::{AuthStore, Plan}; use camera::{CameraPreviewManager, CameraPreviewState}; use cap_editor::{EditorInstance, EditorState}; use cap_project::{ - InstantRecordingMeta, ProjectConfiguration, RecordingMeta, RecordingMetaInner, SharingMeta, - StudioRecordingMeta, StudioRecordingStatus, UploadMeta, VideoUploadInfo, XY, ZoomSegment, + CursorEvents, InstantRecordingMeta, KeyboardEvent, ProjectConfiguration, RecordingMeta, + RecordingMetaInner, SharingMeta, StudioRecordingMeta, StudioRecordingStatus, UploadMeta, + VideoUploadInfo, XY, ZoomSegment, }; use cap_recording::{ RecordingMode, @@ -2030,6 +2031,63 @@ async fn generate_zoom_segments_from_clicks( Ok(zoom_segments) } +#[derive(Serialize, Deserialize, Type, Clone, Debug)] +#[serde(rename_all = "camelCase")] +struct SegmentKeyboardEvents { + segment_index: u32, + events: Vec, +} + +#[tauri::command] +#[specta::specta] +#[instrument(skip(editor_instance))] +async fn get_keyboard_events( + editor_instance: WindowEditorInstance, +) -> Result, String> { + let meta = editor_instance.meta(); + + let load_events = |path| { + let full_path = meta.path(path); + CursorEvents::load_from_file(&full_path) + .map(|events| events.keyboard) + .unwrap_or_else(|error| { + warn!( + path = %full_path.display(), + %error, + "Failed to load cursor events" + ); + Vec::new() + }) + }; + + let events = match &meta.inner { + RecordingMetaInner::Studio(meta) => match meta.as_ref() { + StudioRecordingMeta::SingleSegment { segment } => segment + .cursor + .as_ref() + .map(|path| { + vec![SegmentKeyboardEvents { + segment_index: 0, + events: load_events(path), + }] + }) + .unwrap_or_default(), + StudioRecordingMeta::MultipleSegments { inner, .. } => inner + .segments + .iter() + .enumerate() + .map(|(segment_index, segment)| SegmentKeyboardEvents { + segment_index: segment_index as u32, + events: segment.cursor.as_ref().map(load_events).unwrap_or_default(), + }) + .collect(), + }, + _ => Vec::new(), + }; + + Ok(events) +} + #[tauri::command] #[specta::specta] #[instrument] @@ -3019,6 +3077,7 @@ pub async fn run(recording_logging_handle: LoggingHandle, logs_dir: PathBuf) { set_project_config, update_project_config_in_memory, generate_zoom_segments_from_clicks, + get_keyboard_events, permissions::open_permission_settings, permissions::do_permissions_check, permissions::request_permission, diff --git a/apps/desktop/src/routes/editor/Player.tsx b/apps/desktop/src/routes/editor/Player.tsx index ae17000c2c..aae7fce771 100644 --- a/apps/desktop/src/routes/editor/Player.tsx +++ b/apps/desktop/src/routes/editor/Player.tsx @@ -2,9 +2,18 @@ import { Select as KSelect } from "@kobalte/core/select"; import { ToggleButton as KToggleButton } from "@kobalte/core/toggle-button"; import { createElementBounds } from "@solid-primitives/bounds"; import { debounce } from "@solid-primitives/scheduled"; +import { invoke } from "@tauri-apps/api/core"; import { Menu } from "@tauri-apps/api/menu"; import { cx } from "cva"; -import { createEffect, createSignal, onMount, Show } from "solid-js"; +import { + createEffect, + createMemo, + createResource, + createSignal, + For, + onMount, + Show, +} from "solid-js"; import Tooltip from "~/components/Tooltip"; import { captionsStore } from "~/store/captions"; @@ -444,8 +453,34 @@ const gridStyle = { }; function PreviewCanvas() { - const { latestFrame, canvasControls, performanceMode, setPerformanceMode } = - useEditorContext(); + const { + latestFrame, + canvasControls, + performanceMode, + setPerformanceMode, + project, + editorState, + } = useEditorContext(); + + type KeyboardOverlayEvent = { + active_modifiers: string[]; + key: string; + time_ms: number; + down: boolean; + }; + + type SegmentKeyboardEvents = { + segmentIndex: number; + events: KeyboardOverlayEvent[]; + }; + + const [keyboardEventsBySegment] = createResource(async () => { + try { + return await invoke("get_keyboard_events"); + } catch { + return []; + } + }); const hasRenderedFrame = () => canvasControls()?.hasRenderedFrame() ?? false; @@ -477,6 +512,204 @@ function PreviewCanvas() { height: 0, }); + const currentSourceTime = createMemo(() => { + const timeline = project.timeline?.segments ?? []; + if (timeline.length === 0) return null; + + const timelineTime = Math.max( + editorState.previewTime ?? editorState.playbackTime, + 0, + ); + let consumed = 0; + + for ( + let timelineIndex = 0; + timelineIndex < timeline.length; + timelineIndex++ + ) { + const segment = timeline[timelineIndex]; + if (!segment) continue; + const duration = (segment.end - segment.start) / segment.timescale; + + if (timelineTime <= consumed + duration) { + const elapsed = timelineTime - consumed; + return { + recordingSegmentIndex: segment.recordingSegment ?? timelineIndex, + sourceTimeSec: segment.start + elapsed * segment.timescale, + }; + } + + consumed += duration; + } + + const last = timeline[timeline.length - 1]; + if (!last) return null; + return { + recordingSegmentIndex: last.recordingSegment ?? timeline.length - 1, + sourceTimeSec: last.end, + }; + }); + + const activeShortcut = createMemo(() => { + const recentWindowMs = 850; + const modifierKeys = new Set([ + "Meta", + "MetaLeft", + "MetaRight", + "Command", + "Cmd", + "Ctrl", + "Control", + "ControlLeft", + "ControlRight", + "Alt", + "Option", + "Opt", + "AltLeft", + "AltRight", + "Shift", + "ShiftLeft", + "ShiftRight", + ]); + const modifierOrder = new Map([ + ["⌃", 0], + ["⌥", 1], + ["⇧", 2], + ["⌘", 3], + ]); + + const isModifierKey = (key: string) => modifierKeys.has(key); + + const normalizeModifier = ( + modifier: string, + ): "⌘" | "⌃" | "⌥" | "⇧" | null => { + switch (modifier) { + case "Meta": + case "Command": + case "Cmd": + case "Super": + case "Win": + return "⌘"; + case "Ctrl": + case "Control": + return "⌃"; + case "Alt": + case "Option": + case "Opt": + case "AltGraph": + return "⌥"; + case "Shift": + return "⇧"; + default: + return null; + } + }; + + const normalizeKey = (key: string) => { + switch (key) { + case "Left": + return "←"; + case "Right": + return "→"; + case "Up": + return "↑"; + case "Down": + return "↓"; + case "Enter": + case "Return": + return "Return"; + case "Escape": + return "Esc"; + case "Backspace": + return "Delete"; + case "Delete": + return "Del"; + case "CapsLock": + return "Caps"; + case "PageUp": + return "Page Up"; + case "PageDown": + return "Page Down"; + case "Space": + return "Space"; + default: + return key.toUpperCase(); + } + }; + + const source = currentSourceTime(); + const segments = keyboardEventsBySegment(); + if (!source || !segments) return null; + + const segmentEvents = + segments.find((s) => s.segmentIndex === source.recordingSegmentIndex) + ?.events ?? []; + if (segmentEvents.length === 0) return null; + + const nowMs = source.sourceTimeSec * 1000; + const active = new Map(); + let lastRecent: { label: string; downTime: number } | null = null; + + for (const event of segmentEvents) { + if (event.time_ms > nowMs) break; + if (isModifierKey(event.key)) continue; + + const normalizedModifiers = event.active_modifiers + .filter((modifier) => modifier !== event.key) + .map(normalizeModifier) + .filter( + (modifier): modifier is "⌘" | "⌃" | "⌥" | "⇧" => modifier !== null, + ) + .sort() + .filter((value, index, values) => values.indexOf(value) === index) + .sort( + (a, b) => + (modifierOrder.get(a) ?? Number.POSITIVE_INFINITY) - + (modifierOrder.get(b) ?? Number.POSITIVE_INFINITY), + ); + const hasShortcutModifier = normalizedModifiers.some( + (modifier) => modifier === "⌘" || modifier === "⌃" || modifier === "⌥", + ); + if (event.down && !hasShortcutModifier) continue; + const label = [...normalizedModifiers, normalizeKey(event.key)].join( + " + ", + ); + + if (event.down) { + const state = { label, downTime: event.time_ms }; + active.set(event.key, state); + if (!lastRecent || state.downTime > lastRecent.downTime) { + lastRecent = state; + } + } else { + active.delete(event.key); + } + } + + const activeValues = [...active.values()].sort( + (a, b) => b.downTime - a.downTime, + ); + if (activeValues.length > 0) { + return { + label: activeValues[0]?.label ?? "", + opacity: 1, + scale: 1.05, + }; + } + + if (lastRecent && nowMs - lastRecent.downTime <= recentWindowMs) { + const remaining = 1 - (nowMs - lastRecent.downTime) / recentWindowMs; + const clamped = Math.min(Math.max(remaining, 0), 1); + return { + label: lastRecent.label, + opacity: clamped, + scale: 1 + 0.05 * clamped, + }; + } + + return null; + }); + const updateDebouncedBounds = debounce( (width: number, height: number) => setDebouncedBounds({ width, height }), 100, @@ -557,6 +790,32 @@ function PreviewCanvas() { style={{ contain: "layout style" }} onContextMenu={handleContextMenu} > + + {(shortcut) => ( +
+
+
+ Shortcut +
+
+ + {(part) => ( + + {part} + + )} + +
+
+
+ )} +
, + pub key: String, + pub time_ms: f64, + pub down: bool, +} + +impl PartialOrd for KeyboardEvent { + fn partial_cmp(&self, other: &Self) -> Option { + self.time_ms.partial_cmp(&other.time_ms) + } +} + impl PartialOrd for CursorClickEvent { fn partial_cmp(&self, other: &Self) -> Option { self.time_ms.partial_cmp(&other.time_ms) @@ -53,6 +67,8 @@ pub struct CursorImage { pub struct CursorData { pub clicks: Vec, pub moves: Vec, + #[serde(default)] + pub keyboard: Vec, pub cursor_images: CursorImages, } @@ -67,6 +83,8 @@ impl CursorData { pub struct CursorEvents { pub clicks: Vec, pub moves: Vec, + #[serde(default)] + pub keyboard: Vec, } impl CursorEvents { @@ -262,6 +280,7 @@ impl From for CursorEvents { Self { clicks: value.clicks, moves: value.moves, + keyboard: value.keyboard, } } } @@ -312,6 +331,7 @@ mod tests { move_event(900.0, "pointer"), ], clicks: vec![click_event(250.0, "ibeam")], + keyboard: vec![], }; events.stabilize_short_lived_cursor_shapes( @@ -345,6 +365,7 @@ mod tests { move_event(1500.0, "pointer"), ], clicks: vec![click_event(400.0, "ibeam")], + keyboard: vec![], }; events.stabilize_short_lived_cursor_shapes( @@ -366,6 +387,7 @@ mod tests { move_event(1200.0, "pointer"), ], clicks: vec![click_event(250.0, "ibeam")], + keyboard: vec![], }; events.stabilize_short_lived_cursor_shapes(None, SHORT_CURSOR_SHAPE_DEBOUNCE_MS); diff --git a/crates/recording/src/cursor.rs b/crates/recording/src/cursor.rs index 796093d035..16ce225b2e 100644 --- a/crates/recording/src/cursor.rs +++ b/crates/recording/src/cursor.rs @@ -1,10 +1,10 @@ use cap_cursor_capture::CursorCropBounds; use cap_cursor_info::CursorShape; -use cap_project::{CursorClickEvent, CursorEvents, CursorMoveEvent, XY}; +use cap_project::{CursorClickEvent, CursorEvents, CursorMoveEvent, KeyboardEvent, XY}; use cap_timestamp::Timestamps; use futures::{FutureExt, future::Shared}; use std::{ - collections::HashMap, + collections::{HashMap, HashSet}, path::{Path, PathBuf}, time::Instant, }; @@ -28,6 +28,96 @@ pub struct CursorActorResponse { pub next_cursor_id: u32, pub moves: Vec, pub clicks: Vec, + pub keyboard: Vec, +} + +fn keycode_to_label(key: &device_query::Keycode) -> String { + use device_query::Keycode; + + match key { + Keycode::LShift | Keycode::RShift => "Shift".to_string(), + Keycode::LControl | Keycode::RControl => "Ctrl".to_string(), + Keycode::LAlt | Keycode::RAlt => "Alt".to_string(), + Keycode::LMeta | Keycode::RMeta | Keycode::Command | Keycode::RCommand => { + "Meta".to_string() + } + Keycode::Enter | Keycode::NumpadEnter => "Enter".to_string(), + Keycode::Space => "Space".to_string(), + Keycode::Escape => "Esc".to_string(), + Keycode::Backspace => "Backspace".to_string(), + Keycode::Tab => "Tab".to_string(), + Keycode::Up => "Up".to_string(), + Keycode::Down => "Down".to_string(), + Keycode::Left => "Left".to_string(), + Keycode::Right => "Right".to_string(), + _ => format!("{key:?}"), + } +} + +fn is_modifier(key: &device_query::Keycode) -> bool { + matches!( + key, + device_query::Keycode::LShift + | device_query::Keycode::RShift + | device_query::Keycode::LControl + | device_query::Keycode::RControl + | device_query::Keycode::LAlt + | device_query::Keycode::RAlt + | device_query::Keycode::LOption + | device_query::Keycode::ROption + | device_query::Keycode::LMeta + | device_query::Keycode::RMeta + | device_query::Keycode::Command + | device_query::Keycode::RCommand + ) +} + +fn active_modifiers(keys: &HashSet) -> Vec { + let mut modifiers = Vec::new(); + + if keys.iter().any(|key| { + matches!( + key, + device_query::Keycode::LMeta + | device_query::Keycode::RMeta + | device_query::Keycode::Command + | device_query::Keycode::RCommand + ) + }) { + modifiers.push("Meta".to_string()); + } + + if keys.iter().any(|key| { + matches!( + key, + device_query::Keycode::LControl | device_query::Keycode::RControl + ) + }) { + modifiers.push("Ctrl".to_string()); + } + + if keys.iter().any(|key| { + matches!( + key, + device_query::Keycode::LAlt + | device_query::Keycode::RAlt + | device_query::Keycode::LOption + | device_query::Keycode::ROption + ) + }) { + modifiers.push("Alt".to_string()); + } + + if keys.iter().any(|key| { + matches!( + key, + device_query::Keycode::LShift | device_query::Keycode::RShift + ) + }) { + modifiers.push("Shift".to_string()); + } + + modifiers } pub struct CursorActor { @@ -43,10 +133,16 @@ impl CursorActor { const CURSOR_FLUSH_INTERVAL_SECS: u64 = 5; -fn flush_cursor_data(output_path: &Path, moves: &[CursorMoveEvent], clicks: &[CursorClickEvent]) { +fn flush_cursor_data( + output_path: &Path, + moves: &[CursorMoveEvent], + clicks: &[CursorClickEvent], + keyboard: &[KeyboardEvent], +) { let events = CursorEvents { clicks: clicks.to_vec(), moves: moves.to_vec(), + keyboard: keyboard.to_vec(), }; if let Ok(json) = serde_json::to_string_pretty(&events) && let Err(e) = std::fs::write(output_path, json) @@ -93,11 +189,13 @@ pub fn spawn_cursor_recorder( next_cursor_id, moves: vec![], clicks: vec![], + keyboard: vec![], }; let mut last_flush = Instant::now(); let flush_interval = Duration::from_secs(CURSOR_FLUSH_INTERVAL_SECS); let mut last_cursor_id: Option = None; + let mut last_keys: HashSet = HashSet::new(); loop { let sleep = tokio::time::sleep(Duration::from_millis(16)); @@ -201,12 +299,55 @@ pub fn spawn_cursor_recorder( response.clicks.push(mouse_event); } + let current_keys: HashSet = + device_state.get_keys().into_iter().collect(); + + let mut pressed_keys = current_keys + .difference(&last_keys) + .copied() + .collect::>(); + pressed_keys.sort_by_key(|k| format!("{k:?}")); + + let mut released_keys = last_keys + .difference(¤t_keys) + .copied() + .collect::>(); + released_keys.sort_by_key(|k| format!("{k:?}")); + + for key in pressed_keys { + if is_modifier(&key) { + continue; + } + + response.keyboard.push(KeyboardEvent { + active_modifiers: active_modifiers(¤t_keys), + key: keycode_to_label(&key), + time_ms: elapsed, + down: true, + }); + } + + for key in released_keys { + if is_modifier(&key) { + continue; + } + + response.keyboard.push(KeyboardEvent { + active_modifiers: active_modifiers(&last_keys), + key: keycode_to_label(&key), + time_ms: elapsed, + down: false, + }); + } + + last_keys = current_keys; + last_mouse_state = mouse_state; if let Some(ref path) = output_path && last_flush.elapsed() >= flush_interval { - flush_cursor_data(path, &response.moves, &response.clicks); + flush_cursor_data(path, &response.moves, &response.clicks, &response.keyboard); last_flush = Instant::now(); } } @@ -214,7 +355,7 @@ pub fn spawn_cursor_recorder( info!("cursor recorder done"); if let Some(ref path) = output_path { - flush_cursor_data(path, &response.moves, &response.clicks); + flush_cursor_data(path, &response.moves, &response.clicks, &response.keyboard); } let _ = tx.send(response); diff --git a/crates/recording/src/studio_recording.rs b/crates/recording/src/studio_recording.rs index 4d575f6782..12422a2845 100644 --- a/crates/recording/src/studio_recording.rs +++ b/crates/recording/src/studio_recording.rs @@ -97,6 +97,7 @@ impl Actor { serde_json::to_string_pretty(&CursorEvents { clicks: res.clicks, moves: res.moves, + keyboard: res.keyboard, })?, )?; diff --git a/crates/rendering/src/layers/cursor.rs b/crates/rendering/src/layers/cursor.rs index 4ed98b0dd5..03ecb7143c 100644 --- a/crates/rendering/src/layers/cursor.rs +++ b/crates/rendering/src/layers/cursor.rs @@ -794,6 +794,7 @@ mod tests { .map(|(time, x, y)| move_event(*time, *x, *y)) .collect(), clicks: vec![], + keyboard: vec![], } } diff --git a/crates/rendering/src/layers/keyboard.rs b/crates/rendering/src/layers/keyboard.rs new file mode 100644 index 0000000000..3951f3d945 --- /dev/null +++ b/crates/rendering/src/layers/keyboard.rs @@ -0,0 +1,682 @@ +use std::collections::HashMap; + +use bytemuck::{Pod, Zeroable}; +use cap_project::CursorEvents; +use glyphon::{ + Attrs, Buffer, Cache, Color, Family, Metrics, Resolution, Shaping, SwashCache, TextArea, + TextAtlas, TextBounds, TextRenderer, Viewport, +}; +use wgpu::{ + BindGroup, BindGroupLayout, Device, Queue, RenderPipeline, include_wgsl, util::DeviceExt, +}; + +use crate::DecodedSegmentFrames; + +const RECENT_SHORTCUT_WINDOW_MS: f64 = 850.0; + +#[repr(C)] +#[derive(Clone, Copy, Pod, Zeroable)] +struct KeyboardOverlayUniforms { + output_size: [f32; 2], + _padding0: [f32; 2], + rect: [f32; 4], + fill_color: [f32; 4], + border_color: [f32; 4], + shadow_color: [f32; 4], + radius_feather: [f32; 2], + _padding1: [f32; 2], +} + +impl Default for KeyboardOverlayUniforms { + fn default() -> Self { + Self { + output_size: [1.0, 1.0], + _padding0: [0.0, 0.0], + rect: [0.0, 0.0, 1.0, 1.0], + fill_color: [0.0, 0.0, 0.0, 0.0], + border_color: [0.0, 0.0, 0.0, 0.0], + shadow_color: [0.0, 0.0, 0.0, 0.0], + radius_feather: [0.0, 1.0], + _padding1: [0.0, 0.0], + } + } +} + +struct OverlayStatics { + bind_group_layout: BindGroupLayout, + render_pipeline: RenderPipeline, +} + +struct OverlayInstance { + uniform_buffer: wgpu::Buffer, + bind_group: BindGroup, +} + +impl OverlayStatics { + fn new(device: &Device) -> Self { + let bind_group_layout: BindGroupLayout = + device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("Keyboard Overlay Bind Group Layout"), + entries: &[wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::VERTEX_FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }], + }); + + let shader = device.create_shader_module(include_wgsl!("../shaders/keyboard-overlay.wgsl")); + + let render_pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("Keyboard Overlay Pipeline"), + layout: Some( + &device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("Keyboard Overlay Pipeline Layout"), + bind_group_layouts: &[&bind_group_layout], + push_constant_ranges: &[], + }), + ), + vertex: wgpu::VertexState { + module: &shader, + entry_point: Some("vs_main"), + buffers: &[], + compilation_options: wgpu::PipelineCompilationOptions { + constants: &[], + zero_initialize_workgroup_memory: false, + }, + }, + fragment: Some(wgpu::FragmentState { + module: &shader, + entry_point: Some("fs_main"), + targets: &[Some(wgpu::ColorTargetState { + format: wgpu::TextureFormat::Rgba8UnormSrgb, + blend: Some(wgpu::BlendState::ALPHA_BLENDING), + write_mask: wgpu::ColorWrites::ALL, + })], + compilation_options: wgpu::PipelineCompilationOptions { + constants: &[], + zero_initialize_workgroup_memory: false, + }, + }), + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleStrip, + strip_index_format: None, + front_face: wgpu::FrontFace::Ccw, + cull_mode: None, + polygon_mode: wgpu::PolygonMode::Fill, + unclipped_depth: false, + conservative: false, + }, + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + multiview: None, + cache: None, + }); + + Self { + bind_group_layout, + render_pipeline, + } + } + + fn create_instance(&self, device: &Device) -> OverlayInstance { + let uniform_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("Keyboard Overlay Uniform Buffer"), + contents: bytemuck::cast_slice(&[KeyboardOverlayUniforms::default()]), + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + }); + + let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("Keyboard Overlay Bind Group"), + layout: &self.bind_group_layout, + entries: &[wgpu::BindGroupEntry { + binding: 0, + resource: uniform_buffer.as_entire_binding(), + }], + }); + + OverlayInstance { + uniform_buffer, + bind_group, + } + } +} + +#[derive(Clone)] +struct ShortcutState { + label: String, + down_time: f64, +} + +struct ShortcutPresentation { + label: String, + opacity: f32, +} + +pub struct KeyboardLayer { + overlay: OverlayStatics, + overlays: Vec, + font_system: glyphon::FontSystem, + swash_cache: SwashCache, + text_atlas: TextAtlas, + text_renderer: TextRenderer, + viewport: Viewport, + visible: bool, + current_label: Option, + current_font_size: f32, +} + +impl KeyboardLayer { + pub fn new(device: &Device, queue: &Queue) -> Self { + let overlay = OverlayStatics::new(device); + let font_system = glyphon::FontSystem::new(); + let swash_cache = SwashCache::new(); + let cache = Cache::new(device); + let viewport = Viewport::new(device, &cache); + let mut text_atlas = + TextAtlas::new(device, queue, &cache, wgpu::TextureFormat::Rgba8UnormSrgb); + let text_renderer = TextRenderer::new( + &mut text_atlas, + device, + wgpu::MultisampleState::default(), + None, + ); + + Self { + overlay, + overlays: Vec::new(), + font_system, + swash_cache, + text_atlas, + text_renderer, + viewport, + visible: false, + current_label: None, + current_font_size: 0.0, + } + } + + pub fn prepare( + &mut self, + device: &Device, + queue: &Queue, + cursor: &CursorEvents, + segment_frames: &DecodedSegmentFrames, + output_size: (u32, u32), + ) { + let time_ms = segment_frames.recording_time as f64 * 1000.0; + let presentation = active_shortcut_label(cursor, time_ms); + + self.visible = presentation.is_some(); + if !self.visible { + self.current_label = None; + self.overlays.clear(); + return; + } + + let Some(presentation) = presentation else { + return; + }; + + let text = presentation.label; + let key_parts = text.split(" + ").collect::>(); + if key_parts.is_empty() { + self.visible = false; + self.overlays.clear(); + return; + } + + let opacity = presentation.opacity.clamp(0.0, 1.0); + + let (width, height) = output_size; + let font_size = (height as f32 * 0.0225).clamp(15.0, 28.0); + self.current_label = Some(text.clone()); + self.current_font_size = font_size; + + self.viewport.update(queue, Resolution { width, height }); + + let horizontal_padding = font_size * 0.68; + let vertical_padding = font_size * 0.42; + let pill_height = font_size + vertical_padding * 2.0; + let plus_width = estimate_text_width("+", font_size); + let plus_gap = font_size * 0.42; + + let key_widths = key_parts + .iter() + .map(|part| { + (estimate_text_width(part, font_size) + horizontal_padding * 2.0) + .clamp(font_size * 1.8, width as f32 * 0.34) + }) + .collect::>(); + + let separators_width = if key_parts.len() > 1 { + (key_parts.len() as f32 - 1.0) * (plus_gap * 2.0 + plus_width) + } else { + 0.0 + }; + + let total_width = key_widths.iter().sum::() + separators_width; + + let x = ((width as f32 - total_width) * 0.5).max(12.0); + let y = (height as f32 * 0.87) + .min(height as f32 - pill_height - 12.0) + .max(12.0); + + while self.overlays.len() < key_parts.len() { + self.overlays.push(self.overlay.create_instance(device)); + } + self.overlays.truncate(key_parts.len()); + + let alpha = (opacity * 255.0).clamp(0.0, 255.0) as u8; + let outline_alpha = (opacity * 140.0).clamp(0.0, 255.0) as u8; + let outline_color = Color::rgba(20, 20, 20, outline_alpha); + let main_color = Color::rgba(255, 255, 255, alpha); + let plus_color = Color::rgba(220, 220, 220, (opacity * 220.0).clamp(0.0, 255.0) as u8); + + let mut key_buffers = Vec::::with_capacity(key_parts.len()); + let mut key_positions = Vec::<(f32, f32, f32, f32)>::with_capacity(key_parts.len()); + + let mut plus_buffers = Vec::::new(); + let mut plus_positions = Vec::<(f32, f32)>::new(); + + let metrics = Metrics::new(font_size, font_size * 1.25); + let mut cursor_x = x; + + for (index, part) in key_parts.iter().enumerate() { + if index > 0 { + let mut plus_buffer = Buffer::new(&mut self.font_system, metrics); + plus_buffer.set_size( + &mut self.font_system, + Some((plus_width * 1.5).max(font_size)), + None, + ); + plus_buffer.set_text( + &mut self.font_system, + "+", + &Attrs::new().family(Family::SansSerif).color(plus_color), + Shaping::Advanced, + ); + plus_buffer.shape_until_scroll(&mut self.font_system, false); + + let plus_x = cursor_x + plus_gap; + let plus_y = y + vertical_padding - 0.5; + plus_positions.push((plus_x, plus_y)); + plus_buffers.push(plus_buffer); + + cursor_x += plus_gap * 2.0 + plus_width; + } + + let key_width = key_widths[index]; + let text_left = cursor_x + horizontal_padding; + let text_top = y + vertical_padding - 0.8; + let inner_width = (key_width - horizontal_padding * 2.0).max(font_size); + let text_right = text_left + inner_width; + + let overlay_uniforms = KeyboardOverlayUniforms { + output_size: [width as f32, height as f32], + _padding0: [0.0, 0.0], + rect: [cursor_x, y, cursor_x + key_width, y + pill_height], + fill_color: [0.09, 0.09, 0.1, 0.78 * opacity], + border_color: [1.0, 1.0, 1.0, 0.14 * opacity], + shadow_color: [0.0, 0.0, 0.0, 0.33 * opacity], + radius_feather: [font_size * 0.52, 1.2], + _padding1: [0.0, 0.0], + }; + queue.write_buffer( + &self.overlays[index].uniform_buffer, + 0, + bytemuck::cast_slice(&[overlay_uniforms]), + ); + + let mut key_buffer = Buffer::new(&mut self.font_system, metrics); + key_buffer.set_size(&mut self.font_system, Some(inner_width), None); + key_buffer.set_text( + &mut self.font_system, + part, + &Attrs::new() + .family(Family::Monospace) + .color(Color::rgb(255, 255, 255)), + Shaping::Advanced, + ); + key_buffer.shape_until_scroll(&mut self.font_system, false); + + key_positions.push((text_left, text_top, text_right, text_top + font_size * 1.45)); + key_buffers.push(key_buffer); + + cursor_x += key_width; + } + + let mut text_areas = Vec::new(); + + for (index, buffer) in key_buffers.iter().enumerate() { + let (text_left, text_top, text_right, text_bottom) = key_positions[index]; + let bounds = TextBounds { + left: text_left as i32, + top: text_top as i32, + right: text_right as i32, + bottom: text_bottom as i32, + }; + + for (dx, dy) in [(-1.0, -1.0), (1.0, -1.0), (-1.0, 1.0), (1.0, 1.0)] { + text_areas.push(TextArea { + buffer, + left: text_left + dx, + top: text_top + dy, + scale: 1.0, + bounds, + default_color: outline_color, + custom_glyphs: &[], + }); + } + + text_areas.push(TextArea { + buffer, + left: text_left, + top: text_top, + scale: 1.0, + bounds, + default_color: main_color, + custom_glyphs: &[], + }); + } + + for (index, buffer) in plus_buffers.iter().enumerate() { + let (plus_x, plus_y) = plus_positions[index]; + let bounds = TextBounds { + left: plus_x as i32, + top: plus_y as i32, + right: (plus_x + plus_width * 1.4) as i32, + bottom: (plus_y + font_size * 1.4) as i32, + }; + + text_areas.push(TextArea { + buffer, + left: plus_x, + top: plus_y, + scale: 1.0, + bounds, + default_color: plus_color, + custom_glyphs: &[], + }); + } + + if self + .text_renderer + .prepare( + device, + queue, + &mut self.font_system, + &mut self.text_atlas, + &self.viewport, + text_areas, + &mut self.swash_cache, + ) + .is_err() + { + self.visible = false; + self.current_label = None; + self.overlays.clear(); + return; + } + } + + pub fn render<'a>(&'a self, pass: &mut wgpu::RenderPass<'a>) { + if !self.visible { + return; + } + + pass.set_pipeline(&self.overlay.render_pipeline); + for overlay in &self.overlays { + pass.set_bind_group(0, &overlay.bind_group, &[]); + pass.draw(0..4, 0..1); + } + + let _ = self + .text_renderer + .render(&self.text_atlas, &self.viewport, pass); + } +} + +fn estimate_text_width(text: &str, font_size: f32) -> f32 { + let mut width = 0.0; + + for ch in text.chars() { + width += match ch { + 'I' | 'J' | 'L' | '1' | '|' => font_size * 0.44, + 'M' | 'W' => font_size * 0.76, + ' ' => font_size * 0.32, + _ => font_size * 0.62, + }; + } + + width.max(font_size * 0.62) +} + +fn normalize_modifier(modifier: &str) -> Option<&'static str> { + match modifier { + "Meta" | "Command" | "Cmd" | "Super" | "Win" => Some("⌘"), + "Ctrl" | "Control" => Some("⌃"), + "Alt" | "Option" | "Opt" | "AltGraph" => Some("⌥"), + "Shift" => Some("⇧"), + _ => None, + } +} + +fn modifier_sort_key(symbol: &str) -> u8 { + match symbol { + "⌃" => 0, + "⌥" => 1, + "⇧" => 2, + "⌘" => 3, + _ => 255, + } +} + +fn is_modifier_key(key: &str) -> bool { + matches!( + key, + "Meta" + | "MetaLeft" + | "MetaRight" + | "Command" + | "Cmd" + | "Ctrl" + | "Control" + | "ControlLeft" + | "ControlRight" + | "Alt" + | "Option" + | "Opt" + | "AltLeft" + | "AltRight" + | "Shift" + | "ShiftLeft" + | "ShiftRight" + ) +} + +fn normalize_key_label(key: &str) -> String { + match key { + "Left" => "←".to_string(), + "Right" => "→".to_string(), + "Up" => "↑".to_string(), + "Down" => "↓".to_string(), + "Space" => "Space".to_string(), + "Enter" | "Return" => "Return".to_string(), + "Escape" => "Esc".to_string(), + "Backspace" => "Delete".to_string(), + "Tab" => "Tab".to_string(), + "CapsLock" => "Caps".to_string(), + "PageUp" => "Page Up".to_string(), + "PageDown" => "Page Down".to_string(), + "Delete" => "Del".to_string(), + other => other.to_uppercase(), + } +} + +fn active_shortcut_label(cursor: &CursorEvents, now_ms: f64) -> Option { + if cursor.keyboard.is_empty() { + return None; + } + + let mut active = HashMap::::new(); + let mut last_recent: Option = None; + + for event in cursor.keyboard.iter() { + if event.time_ms > now_ms { + break; + } + + let mut mods = event + .active_modifiers + .iter() + .filter(|modifier| modifier.as_str() != event.key) + .filter_map(|modifier| normalize_modifier(modifier)) + .map(ToOwned::to_owned) + .collect::>(); + mods.sort_by_key(|symbol| modifier_sort_key(symbol)); + mods.dedup(); + + if is_modifier_key(&event.key) { + continue; + } + + let has_shortcut_modifier = mods + .iter() + .any(|symbol| symbol == "⌘" || symbol == "⌃" || symbol == "⌥"); + if event.down && !has_shortcut_modifier { + continue; + } + + let mut parts = mods; + parts.push(normalize_key_label(&event.key)); + let label = parts.join(" + "); + + if event.down { + let state = ShortcutState { + label, + down_time: event.time_ms, + }; + + active.insert(event.key.clone(), state.clone()); + + if let Some(last) = &last_recent { + if state.down_time > last.down_time { + last_recent = Some(state); + } + } else { + last_recent = Some(state); + } + } else { + active.remove(&event.key); + } + } + + if let Some(current) = active.values().max_by(|a, b| { + a.down_time + .partial_cmp(&b.down_time) + .unwrap_or(std::cmp::Ordering::Equal) + }) { + return Some(ShortcutPresentation { + label: current.label.clone(), + opacity: 1.0, + }); + } + + if let Some(last) = last_recent + && now_ms - last.down_time <= RECENT_SHORTCUT_WINDOW_MS + { + let remaining = 1.0 - ((now_ms - last.down_time) / RECENT_SHORTCUT_WINDOW_MS); + return Some(ShortcutPresentation { + label: last.label, + opacity: remaining.clamp(0.0, 1.0) as f32, + }); + } + + None +} + +#[cfg(test)] +mod tests { + use super::*; + use cap_project::KeyboardEvent; + + fn key_event(active_modifiers: &[&str], key: &str, time_ms: f64, down: bool) -> KeyboardEvent { + KeyboardEvent { + active_modifiers: active_modifiers.iter().map(|v| (*v).to_string()).collect(), + key: key.to_string(), + time_ms, + down, + } + } + + #[test] + fn normalizes_modifier_symbols() { + assert_eq!(normalize_modifier("Meta"), Some("⌘")); + assert_eq!(normalize_modifier("Control"), Some("⌃")); + assert_eq!(normalize_modifier("Option"), Some("⌥")); + assert_eq!(normalize_modifier("Shift"), Some("⇧")); + assert_eq!(normalize_modifier("Unknown"), None); + } + + #[test] + fn normalizes_common_key_labels() { + assert_eq!(normalize_key_label("Left"), "←"); + assert_eq!(normalize_key_label("Return"), "Return"); + assert_eq!(normalize_key_label("Escape"), "Esc"); + assert_eq!(normalize_key_label("a"), "A"); + } + + #[test] + fn active_shortcut_has_full_opacity_when_key_is_down() { + let cursor = CursorEvents { + clicks: vec![], + moves: vec![], + keyboard: vec![key_event(&["Meta"], "k", 100.0, true)], + }; + + let presentation = active_shortcut_label(&cursor, 100.0).expect("shortcut should exist"); + assert_eq!(presentation.label, "⌘ + K"); + assert!((presentation.opacity - 1.0).abs() < f32::EPSILON); + } + + #[test] + fn modifiers_are_displayed_in_canonical_order() { + let cursor = CursorEvents { + clicks: vec![], + moves: vec![], + keyboard: vec![key_event( + &["Shift", "Meta", "Alt", "Ctrl"], + "k", + 100.0, + true, + )], + }; + + let presentation = active_shortcut_label(&cursor, 100.0).expect("shortcut should exist"); + assert_eq!(presentation.label, "⌃ + ⌥ + ⇧ + ⌘ + K"); + } + + #[test] + fn recently_released_shortcut_fades_then_disappears() { + let cursor = CursorEvents { + clicks: vec![], + moves: vec![], + keyboard: vec![ + key_event(&["Meta"], "k", 100.0, true), + key_event(&["Meta"], "k", 300.0, false), + ], + }; + + let fading = active_shortcut_label(&cursor, 700.0).expect("recent shortcut should fade"); + assert_eq!(fading.label, "⌘ + K"); + assert!(fading.opacity > 0.0 && fading.opacity < 1.0); + + let gone = active_shortcut_label(&cursor, 1300.0); + assert!(gone.is_none()); + } +} diff --git a/crates/rendering/src/layers/mod.rs b/crates/rendering/src/layers/mod.rs index 536fbb3bf6..7e727b27dc 100644 --- a/crates/rendering/src/layers/mod.rs +++ b/crates/rendering/src/layers/mod.rs @@ -4,6 +4,7 @@ mod camera; mod captions; mod cursor; mod display; +mod keyboard; mod mask; mod text; @@ -13,5 +14,6 @@ pub use camera::*; pub use captions::*; pub use cursor::*; pub use display::*; +pub use keyboard::*; pub use mask::*; pub use text::*; diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rs index 410a01c52a..cba8e450c3 100644 --- a/crates/rendering/src/lib.rs +++ b/crates/rendering/src/lib.rs @@ -11,7 +11,7 @@ use frame_pipeline::{RenderSession, finish_encoder, finish_encoder_nv12, flush_p use futures::future::OptionFuture; use layers::{ Background, BackgroundLayer, BlurLayer, CameraLayer, CaptionsLayer, CursorLayer, DisplayLayer, - MaskLayer, TextLayer, + KeyboardLayer, MaskLayer, TextLayer, }; use specta::Type; use spring_mass_damper::SpringMassDamperSimulationConfig; @@ -2517,6 +2517,7 @@ pub struct RendererLayers { background_blur: BlurLayer, display: DisplayLayer, cursor: CursorLayer, + keyboard: KeyboardLayer, camera: CameraLayer, camera_only: CameraLayer, mask: MaskLayer, @@ -2548,6 +2549,7 @@ impl RendererLayers { prefer_cpu_conversion, ), cursor: CursorLayer::new(device), + keyboard: KeyboardLayer::new(device, queue), camera: CameraLayer::new_with_all_shared_pipelines( device, shared_yuv_pipelines.clone(), @@ -2624,6 +2626,14 @@ impl RendererLayers { constants, ); + self.keyboard.prepare( + &constants.device, + &constants.queue, + cursor, + segment_frames, + (uniforms.output_size.0, uniforms.output_size.1), + ); + self.camera.prepare( &constants.device, &constants.queue, @@ -2703,6 +2713,14 @@ impl RendererLayers { constants, ); + self.keyboard.prepare( + &constants.device, + &constants.queue, + cursor, + segment_frames, + (uniforms.output_size.0, uniforms.output_size.1), + ); + self.camera.prepare_with_encoder( &constants.device, &constants.queue, @@ -2805,6 +2823,11 @@ impl RendererLayers { self.cursor.render(&mut pass); } + if should_render { + let mut pass = render_pass!(session.current_texture_view(), wgpu::LoadOp::Load); + self.keyboard.render(&mut pass); + } + // Render camera-only layer when transitioning with CameraOnly mode if uniforms.scene.is_transitioning_camera_only() { let mut pass = render_pass!(session.current_texture_view(), wgpu::LoadOp::Load); @@ -2986,6 +3009,7 @@ mod project_uniforms_tests { let events = CursorEvents { clicks: vec![], moves: vec![], + keyboard: vec![], }; let focus = ProjectUniforms::auto_zoom_focus(&events, 0.3, None, None); @@ -3003,6 +3027,7 @@ mod project_uniforms_tests { cursor_move(200.0, 0.55, 0.5), cursor_move(400.0, 0.6, 0.5), ], + keyboard: vec![], }; let smoothing = Some(default_smoothing()); @@ -3023,6 +3048,7 @@ mod project_uniforms_tests { let events = CursorEvents { clicks: vec![], moves: vec![cursor_move(0.0, 0.1, 0.5), cursor_move(40.0, 0.9, 0.5)], + keyboard: vec![], }; let smoothing = Some(default_smoothing()); diff --git a/crates/rendering/src/shaders/keyboard-overlay.wgsl b/crates/rendering/src/shaders/keyboard-overlay.wgsl new file mode 100644 index 0000000000..7f770e69dd --- /dev/null +++ b/crates/rendering/src/shaders/keyboard-overlay.wgsl @@ -0,0 +1,92 @@ +struct VertexOutput { + @builtin(position) position: vec4, + @location(0) uv: vec2, +}; + +struct Uniforms { + output_size: vec2, + _padding0: vec2, + rect: vec4, + fill_color: vec4, + border_color: vec4, + shadow_color: vec4, + radius_feather: vec2, + _padding1: vec2, +}; + +@group(0) @binding(0) +var uniforms: Uniforms; + +fn sdf_round_rect(point: vec2, center: vec2, half_size: vec2, radius: f32) -> f32 { + let q = abs(point - center) - (half_size - vec2(radius, radius)); + return length(max(q, vec2(0.0, 0.0))) + min(max(q.x, q.y), 0.0) - radius; +} + +@vertex +fn vs_main(@builtin(vertex_index) vertex_index: u32) -> VertexOutput { + var positions = array, 4>( + vec2(-1.0, -1.0), + vec2(-1.0, 1.0), + vec2(1.0, -1.0), + vec2(1.0, 1.0) + ); + + var uvs = array, 4>( + vec2(0.0, 0.0), + vec2(0.0, 1.0), + vec2(1.0, 0.0), + vec2(1.0, 1.0) + ); + + var out: VertexOutput; + out.position = vec4(positions[vertex_index], 0.0, 1.0); + out.uv = uvs[vertex_index]; + return out; +} + +@fragment +fn fs_main(input: VertexOutput) -> @location(0) vec4 { + let screen_pos = vec2( + input.uv.x * uniforms.output_size.x, + (1.0 - input.uv.y) * uniforms.output_size.y + ); + + let rect_min = uniforms.rect.xy; + let rect_max = uniforms.rect.zw; + let rect_center = (rect_min + rect_max) * 0.5; + let half_size = (rect_max - rect_min) * 0.5; + let radius = max(uniforms.radius_feather.x, 0.0); + let feather = max(uniforms.radius_feather.y, 0.001); + + let dist = sdf_round_rect(screen_pos, rect_center, half_size, radius); + + let fill_alpha = 1.0 - smoothstep(0.0, feather, dist); + let border_width = 1.0; + let border_alpha = (1.0 - smoothstep(-border_width, 0.0, dist)) * fill_alpha; + + let shadow_dist = sdf_round_rect( + screen_pos, + rect_center + vec2(0.0, 2.0), + half_size + vec2(1.0, 1.0), + radius + 1.0 + ); + let shadow_alpha = 1.0 - smoothstep(0.0, feather * 3.0, shadow_dist); + + let shadow = vec4( + uniforms.shadow_color.rgb, + uniforms.shadow_color.a * shadow_alpha + ); + + let fill = vec4( + uniforms.fill_color.rgb, + uniforms.fill_color.a * fill_alpha + ); + + let border = vec4( + uniforms.border_color.rgb, + uniforms.border_color.a * border_alpha + ); + + let base = shadow + fill * (1.0 - shadow.a); + return base + border * (1.0 - base.a); +}