From b6d211413d76e081cd1765d3615059a91561e654 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 28 Aug 2026 05:46:58 +0000 Subject: [PATCH 01/15] Add Cloud Agent environment for Rust 1.98 and mock gateway. The lockfile requires rustc 1.85+ (edition 2024). Pin 1.98.0, compile the workspace, and start the mock gateway on :8787 so Cloud Agents boot ready to work. Co-authored-by: Byte271 --- .cursor/environment.json | 5 +++++ .cursor/install.sh | 9 +++++++++ .cursor/start-gateway.sh | 29 +++++++++++++++++++++++++++++ .gitignore | 1 + 4 files changed, 44 insertions(+) create mode 100644 .cursor/environment.json create mode 100755 .cursor/install.sh create mode 100755 .cursor/start-gateway.sh diff --git a/.cursor/environment.json b/.cursor/environment.json new file mode 100644 index 0000000..b4af7ba --- /dev/null +++ b/.cursor/environment.json @@ -0,0 +1,5 @@ +{ + "name": "OpenLive", + "install": "bash .cursor/install.sh", + "start": "bash .cursor/start-gateway.sh" +} diff --git a/.cursor/install.sh b/.cursor/install.sh new file mode 100755 index 0000000..f354784 --- /dev/null +++ b/.cursor/install.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Cloud Agent install: pin Rust 1.98 (lockfile needs edition 2024 / rustc >= 1.85), +# then fetch and compile the workspace including test binaries. +set -euo pipefail + +rustup toolchain install 1.98.0 --component rustfmt --component clippy --no-self-update +rustup default 1.98.0 +cargo fetch --locked +cargo build --workspace --locked --all-targets diff --git a/.cursor/start-gateway.sh b/.cursor/start-gateway.sh new file mode 100755 index 0000000..8362054 --- /dev/null +++ b/.cursor/start-gateway.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Per-boot: mock OpenLive gateway on :8787. Idempotent; returns after /health is OK. +set -euo pipefail + +mkdir -p /tmp/openlive +if curl -sf --max-time 2 http://127.0.0.1:8787/health >/dev/null; then + echo openlive-gateway already healthy + exit 0 +fi +if [ ! -x ./target/debug/openlive-gateway ]; then + echo missing ./target/debug/openlive-gateway >&2 + exit 1 +fi +setsid ./target/debug/openlive-gateway \ + --listen 0.0.0.0:8787 \ + --provider mock \ + --web-dir apps/openlive-gateway/web \ + /tmp/openlive/gateway.log 2>&1 & +echo $! >/tmp/openlive/gateway.pid +for _ in $(seq 1 60); do + if curl -sf --max-time 2 http://127.0.0.1:8787/health >/dev/null; then + echo openlive-gateway ready + exit 0 + fi + sleep 0.25 +done +echo gateway failed to become healthy >&2 +cat /tmp/openlive/gateway.log >&2 || true +exit 1 diff --git a/.gitignore b/.gitignore index b62ddf6..bd77c14 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /target /data +apps/openlive-gateway/data/ *.zip .DS_Store From 2f7196f3b949331b7edf6f26ee12efc3f56ad411 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 28 Aug 2026 05:55:42 +0000 Subject: [PATCH 02/15] Satisfy Clippy 1.98 pedantic checks that fail CI. CI uses stable rustc 1.98 with -D warnings while the workspace denies clippy::all and clippy::pedantic, so map_unwrap_or, unnecessary_sort_by, result_large_err, and related lints now fail the rust job. Co-authored-by: Byte271 --- apps/openlive-gateway/src/main.rs | 30 ++++++++-------- apps/openlive-gateway/src/session.rs | 3 +- apps/openlive-gateway/src/session_registry.rs | 8 ++--- apps/openlive-gateway/src/session_state.rs | 2 +- apps/openlive-gateway/src/webrtc_media.rs | 2 +- apps/openlive-gateway/tests/task_lifecycle.rs | 36 ++++++++++--------- .../openlive-provider/src/headless_browser.rs | 8 ++--- crates/openlive-provider/src/knowledge.rs | 3 +- crates/openlive-provider/src/llm_bridge.rs | 4 ++- crates/openlive-provider/src/memory_store.rs | 3 +- crates/openlive-provider/src/mock.rs | 4 +-- .../src/openai_compatible.rs | 2 ++ .../openlive-provider/src/openai_realtime.rs | 3 ++ .../openlive-provider/src/pending_actions.rs | 3 +- crates/openlive-provider/src/tools.rs | 5 ++- crates/openlive-provider/src/user_profile.rs | 3 +- crates/openlive-runtime/src/persistence.rs | 3 +- 17 files changed, 59 insertions(+), 63 deletions(-) diff --git a/apps/openlive-gateway/src/main.rs b/apps/openlive-gateway/src/main.rs index de4a6f4..661714e 100644 --- a/apps/openlive-gateway/src/main.rs +++ b/apps/openlive-gateway/src/main.rs @@ -1288,7 +1288,7 @@ async fn sandbox_test_run(State(state): State, headers: HeaderMap) -> let read = sandbox_read_file(path); tests.push(serde_json::json!({ "name": "sandbox_write_read", - "ok": write.is_ok() && read.as_ref().map(|t| t.contains("openlive-ok")).unwrap_or(false), + "ok": write.is_ok() && read.as_ref().is_ok_and(|t| t.contains("openlive-ok")), "detail": format!("{:?} / {:?}", write, read.as_ref().map(|s| s.chars().take(40).collect::())), })); // 2. Calculator / identity via agent @@ -1345,7 +1345,7 @@ async fn sandbox_test_run(State(state): State, headers: HeaderMap) -> let after = sandbox_read_file(cpath); tests.push(serde_json::json!({ "name": "pending_confirm_write", - "ok": approved.is_ok() && after.as_ref().map(|t| t.contains("v2-approved")).unwrap_or(false), + "ok": approved.is_ok() && after.as_ref().is_ok_and(|t| t.contains("v2-approved")), "detail": format!("{:?} / {:?}", approved, after), })); // 5. Lab note + browse wiki summary @@ -1361,7 +1361,7 @@ async fn sandbox_test_run(State(state): State, headers: HeaderMap) -> .await; tests.push(serde_json::json!({ "name": "browse_wikipedia", - "ok": browse.as_ref().map(|(t, _)| t.to_ascii_lowercase().contains("agent")).unwrap_or(false), + "ok": browse.as_ref().is_ok_and(|(t, _)| t.to_ascii_lowercase().contains("agent")), "detail": browse.as_ref().map_or_else(std::clone::Clone::clone, |(t, c)| format!("{} @ {}", t.chars().take(80).collect::(), c.url)), })); // 6. Durable profile @@ -2000,20 +2000,19 @@ async fn session_transcript( })); } } - "user_transcript_delta" => { + "user_transcript_delta" if payload .get("is_final") .and_then(serde_json::Value::as_bool) - .unwrap_or(false) - { - if let Some(text) = payload.get("text").and_then(|t| t.as_str()) { - turns.push(serde_json::json!({ - "role": "user", - "text": text, - "sequence": row.sequence, - "event_id": row.event_id, - })); - } + .unwrap_or(false) => + { + if let Some(text) = payload.get("text").and_then(|t| t.as_str()) { + turns.push(serde_json::json!({ + "role": "user", + "text": text, + "sequence": row.sequence, + "event_id": row.event_id, + })); } } _ => {} @@ -2390,6 +2389,5 @@ fn feature_flags(state: &AppState) -> serde_json::Value { fn now_ms() -> u64 { SystemTime::now() .duration_since(UNIX_EPOCH) - .map(|d| u64::try_from(d.as_millis()).unwrap_or(u64::MAX)) - .unwrap_or(0) + .map_or(0, |d| u64::try_from(d.as_millis()).unwrap_or(u64::MAX)) } diff --git a/apps/openlive-gateway/src/session.rs b/apps/openlive-gateway/src/session.rs index 4221f42..221c8da 100644 --- a/apps/openlive-gateway/src/session.rs +++ b/apps/openlive-gateway/src/session.rs @@ -707,8 +707,7 @@ impl SessionCoordinator { let conversation_version = self .leases .active() - .map(|lease| lease.conversation_version) - .unwrap_or_default(); + .map_or_else(Default::default, |lease| lease.conversation_version); self.engine.mark_response_started(generation_id); // Bind every pending (unbound) task to this generation. Tasks // admitted between turns now get attached to the upcoming diff --git a/apps/openlive-gateway/src/session_registry.rs b/apps/openlive-gateway/src/session_registry.rs index 4b5a44f..8a64921 100644 --- a/apps/openlive-gateway/src/session_registry.rs +++ b/apps/openlive-gateway/src/session_registry.rs @@ -63,12 +63,11 @@ impl SessionRegistry { pub fn list(&self) -> Vec { self.inner .lock() - .map(|guard| guard.values().cloned().collect()) - .unwrap_or_default() + .map_or_else(|_| Vec::new(), |guard| guard.values().cloned().collect()) } pub fn active_count(&self) -> usize { - self.inner.lock().map(|g| g.len()).unwrap_or(0) + self.inner.lock().map_or(0, |g| g.len()) } pub fn opened_total(&self) -> u64 { @@ -79,8 +78,7 @@ impl SessionRegistry { fn now_ms() -> u64 { SystemTime::now() .duration_since(UNIX_EPOCH) - .map(|d| u64::try_from(d.as_millis()).unwrap_or(u64::MAX)) - .unwrap_or(0) + .map_or(0, |d| u64::try_from(d.as_millis()).unwrap_or(u64::MAX)) } #[cfg(test)] diff --git a/apps/openlive-gateway/src/session_state.rs b/apps/openlive-gateway/src/session_state.rs index c5374bb..bd07a88 100644 --- a/apps/openlive-gateway/src/session_state.rs +++ b/apps/openlive-gateway/src/session_state.rs @@ -781,7 +781,7 @@ mod tests { let first = orchestrator .admit(sample_request(task_id, "Remind me"), None, 0) .expect("first admit"); - assert!(first.task_id == task_id); + assert_eq!(first.task_id, task_id); assert!(orchestrator .admit(sample_request(task_id, "Remind me again"), None, 0) .is_none()); diff --git a/apps/openlive-gateway/src/webrtc_media.rs b/apps/openlive-gateway/src/webrtc_media.rs index ef8f291..08cc30f 100644 --- a/apps/openlive-gateway/src/webrtc_media.rs +++ b/apps/openlive-gateway/src/webrtc_media.rs @@ -99,7 +99,7 @@ impl WebRtcHub { #[must_use] pub fn peer_count(&self) -> usize { - self.peers.lock().map(|g| g.len()).unwrap_or(0) + self.peers.lock().map_or(0, |g| g.len()) } /// Answer a browser SDP offer. Returns (`answer_sdp`, session). diff --git a/apps/openlive-gateway/tests/task_lifecycle.rs b/apps/openlive-gateway/tests/task_lifecycle.rs index bd520aa..5ea6193 100644 --- a/apps/openlive-gateway/tests/task_lifecycle.rs +++ b/apps/openlive-gateway/tests/task_lifecycle.rs @@ -2,10 +2,10 @@ //! //! This test spawns the `openlive-gateway` binary with the mock provider, //! opens a WebSocket connection, and exercises the complete task lifecycle: -//! 1. capability_offer → capability_selected (resume_supported = true) -//! 2. task_requested → task_acknowledged (latency < 500ms) -//! 3. task_cancel → task_outcome (result = cancelled) -//! 4. session_resume → replay of buffered task_acknowledged +//! 1. `capability_offer` → `capability_selected` (`resume_supported` = true) +//! 2. `task_requested` → `task_acknowledged` (latency < 500ms) +//! 3. `task_cancel` → `task_outcome` (result = cancelled) +//! 4. `session_resume` → replay of buffered `task_acknowledged` //! //! The test uses `tokio-tungstenite` as the WebSocket client and reads //! binary envelopes via the same framing the browser uses. It is the @@ -54,9 +54,10 @@ async fn spawn_gateway() -> (String, tokio::process::Child) { // Wait for the gateway to start listening. let deadline = Instant::now() + Duration::from_secs(5); loop { - if Instant::now() > deadline { - panic!("gateway did not start listening on {listen} within 5s"); - } + assert!( + Instant::now() <= deadline, + "gateway did not start listening on {listen} within 5s" + ); if tokio::net::TcpStream::connect(&listen).await.is_ok() { break; } @@ -112,9 +113,7 @@ async fn wait_for_event( ) -> EventEnvelope { let deadline = Instant::now() + Duration::from_secs(3); loop { - if Instant::now() > deadline { - panic!("timed out waiting for event"); - } + assert!(Instant::now() <= deadline, "timed out waiting for event"); let envelope = recv_envelope(socket).await; if predicate(&envelope.event) { return envelope; @@ -123,6 +122,7 @@ async fn wait_for_event( } #[tokio::test] +#[allow(clippy::too_many_lines)] async fn task_lifecycle_request_acknowledge_cancel_resume() { let (url, mut child) = spawn_gateway().await; @@ -269,7 +269,7 @@ async fn task_lifecycle_request_acknowledge_cancel_resume() { return envelope; } RealtimeEvent::Pong => return envelope, - _ => continue, + _ => {} } } }) @@ -488,7 +488,9 @@ async fn duplicate_task_id_is_rejected() { /// If these thresholds regress, the orchestrator's `admit()` path has grown /// non-trivial work and needs profiling. #[tokio::test] +#[allow(clippy::doc_markdown)] async fn task_acknowledgement_latency_benchmark() { + const SAMPLES: usize = 50; let (url, mut child) = spawn_gateway().await; let (mut socket, _response) = tokio_tungstenite::connect_async(&url) @@ -530,9 +532,8 @@ async fn task_acknowledgement_latency_benchmark() { }) .await; - // Send 50 task_requested events, then collect 50 task_acknowledged + // Send 50 `task_requested` events, then collect 50 `task_acknowledged` // events, measuring the round-trip for each. - const SAMPLES: usize = 50; let mut task_ids = Vec::with_capacity(SAMPLES); let mut send_times = Vec::with_capacity(SAMPLES); @@ -566,15 +567,16 @@ async fn task_acknowledgement_latency_benchmark() { .expect("timed out waiting for ack") .expect("stream closed") .expect("ws error"); - let text = match envelope { - Message::Text(t) => t, - _ => continue, + let Message::Text(text) = envelope else { + continue; }; let envelope: EventEnvelope = serde_json::from_str(&text).expect("deserialize"); if let RealtimeEvent::TaskAcknowledged(ack) = &envelope.event { if let Some(idx) = task_ids.iter().position(|id| *id == ack.task_id) { if acked_ids.insert(ack.task_id) { - latencies.push(send_times[idx].elapsed().as_millis() as u64); + latencies.push( + u64::try_from(send_times[idx].elapsed().as_millis()).unwrap_or(u64::MAX), + ); } } } diff --git a/crates/openlive-provider/src/headless_browser.rs b/crates/openlive-provider/src/headless_browser.rs index b9d0325..f865c9a 100644 --- a/crates/openlive-provider/src/headless_browser.rs +++ b/crates/openlive-provider/src/headless_browser.rs @@ -230,8 +230,7 @@ fn safe_shot_name(url: &str) -> String { .collect(); let ts = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or(0); + .map_or(0, |d| d.as_secs()); format!("{host}-{ts}.png") } @@ -342,8 +341,7 @@ fn safe_pdf_name(url: &str) -> String { .collect(); let ts = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or(0); + .map_or(0, |d| d.as_secs()); format!("{host}-{ts}.pdf") } @@ -478,7 +476,7 @@ pub fn list_lab_media(limit: usize) -> Vec { } } } - items.sort_by(|a, b| b.modified_ms.cmp(&a.modified_ms)); + items.sort_by_key(|a| std::cmp::Reverse(a.modified_ms)); items.truncate(limit); items } diff --git a/crates/openlive-provider/src/knowledge.rs b/crates/openlive-provider/src/knowledge.rs index fd26e19..185f590 100644 --- a/crates/openlive-provider/src/knowledge.rs +++ b/crates/openlive-provider/src/knowledge.rs @@ -233,8 +233,7 @@ mod tests { "openlive-know-{}", std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_nanos()) - .unwrap_or(0) + .map_or(0, |d| d.as_nanos()) )); fs::create_dir_all(&dir).unwrap(); let mut file = fs::File::create(dir.join("notes.md")).unwrap(); diff --git a/crates/openlive-provider/src/llm_bridge.rs b/crates/openlive-provider/src/llm_bridge.rs index db1adfb..aa38118 100644 --- a/crates/openlive-provider/src/llm_bridge.rs +++ b/crates/openlive-provider/src/llm_bridge.rs @@ -118,7 +118,9 @@ impl LlmBridge { #[must_use] pub fn settings(&self) -> LlmSettings { - self.settings.read().map(|g| g.clone()).unwrap_or_default() + self.settings + .read() + .map_or_else(|_| LlmSettings::default(), |g| g.clone()) } pub fn update_settings(&self, partial: LlmSettings) { diff --git a/crates/openlive-provider/src/memory_store.rs b/crates/openlive-provider/src/memory_store.rs index cd14a20..c37cb9d 100644 --- a/crates/openlive-provider/src/memory_store.rs +++ b/crates/openlive-provider/src/memory_store.rs @@ -68,8 +68,7 @@ pub fn append_memory(role: &str, text: &str, tags: Vec) -> Result, generation_id: uuid::Uuid, diff --git a/crates/openlive-provider/src/openai_compatible.rs b/crates/openlive-provider/src/openai_compatible.rs index 3b35fd5..3819c78 100644 --- a/crates/openlive-provider/src/openai_compatible.rs +++ b/crates/openlive-provider/src/openai_compatible.rs @@ -676,6 +676,7 @@ pub(crate) async fn checked_response(response: Response) -> Result, Stri )) } +#[allow(clippy::result_large_err)] async fn send_state( sender: &mpsc::Sender, generation_id: Uuid, @@ -709,6 +710,7 @@ async fn send_pipeline_error( .await; } +#[allow(clippy::result_large_err)] async fn send( sender: &mpsc::Sender, generation_id: Option, diff --git a/crates/openlive-provider/src/openai_realtime.rs b/crates/openlive-provider/src/openai_realtime.rs index 273bbf2..b655d2c 100644 --- a/crates/openlive-provider/src/openai_realtime.rs +++ b/crates/openlive-provider/src/openai_realtime.rs @@ -236,6 +236,7 @@ async fn run_realtime_session( let _ = websocket_sender.close().await; } +#[allow(clippy::result_large_err)] async fn handle_input( input: ProviderInput, active: &mut Option, @@ -460,6 +461,7 @@ async fn emit_provider_error( .await; } +#[allow(clippy::result_large_err)] async fn emit( sender: &mpsc::Sender, response: &ActiveResponse, @@ -494,6 +496,7 @@ async fn send_error( .await; } +#[allow(clippy::result_large_err)] async fn send_json(sender: &mut S, value: &Value) -> Result<(), WebSocketError> where S: Sink + Unpin, diff --git a/crates/openlive-provider/src/pending_actions.rs b/crates/openlive-provider/src/pending_actions.rs index e721d13..fabb22c 100644 --- a/crates/openlive-provider/src/pending_actions.rs +++ b/crates/openlive-provider/src/pending_actions.rs @@ -43,8 +43,7 @@ fn store() -> &'static Mutex> { fn now_ms() -> u64 { SystemTime::now() .duration_since(UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0) + .map_or(0, |d| d.as_millis() as u64) } fn purge(map: &mut HashMap) { diff --git a/crates/openlive-provider/src/tools.rs b/crates/openlive-provider/src/tools.rs index 9dbb83c..90f58da 100644 --- a/crates/openlive-provider/src/tools.rs +++ b/crates/openlive-provider/src/tools.rs @@ -1261,7 +1261,7 @@ pub async fn web_search_with_sources( ranked.push((title_rank(&c, &cleaned), c)); } } - ranked.sort_by(|a, b| b.0.cmp(&a.0)); + ranked.sort_by_key(|a| std::cmp::Reverse(a.0)); let candidates: Vec = ranked.into_iter().map(|(_, t)| t).collect(); for title in candidates.into_iter().take(8) { @@ -2080,8 +2080,7 @@ pub fn simple_eval(expr: &str) -> Result { fn chrono_lite_now() -> String { let secs = SystemTime::now() .duration_since(UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or(0); + .map_or(0, |d| d.as_secs()); let tod = secs % 86_400; let h = tod / 3600; let m = (tod % 3600) / 60; diff --git a/crates/openlive-provider/src/user_profile.rs b/crates/openlive-provider/src/user_profile.rs index 528235f..c68bba9 100644 --- a/crates/openlive-provider/src/user_profile.rs +++ b/crates/openlive-provider/src/user_profile.rs @@ -60,8 +60,7 @@ fn profile_path() -> PathBuf { fn now_ms() -> u64 { std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0) + .map_or(0, |d| d.as_millis() as u64) } #[must_use] diff --git a/crates/openlive-runtime/src/persistence.rs b/crates/openlive-runtime/src/persistence.rs index 72c6382..c7f3fef 100644 --- a/crates/openlive-runtime/src/persistence.rs +++ b/crates/openlive-runtime/src/persistence.rs @@ -204,8 +204,7 @@ mod tests { fn tmp_store() -> SessionStore { let stamp = SystemTime::now() .duration_since(UNIX_EPOCH) - .map(|d| d.as_nanos()) - .unwrap_or(0); + .map_or(0, |d| d.as_nanos()); let path = std::env::temp_dir().join(format!("openlive-persist-{stamp}")); SessionStore::open(path).expect("store") } From f8b57b93c2dbbf45e4bd580262bc73bb09b1921e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 28 Aug 2026 06:17:44 +0000 Subject: [PATCH 03/15] Ship a call-first OpenLive surface with Bloub and neural TTS. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Default chrome is the listening orb plus Mute / End / Settings. Vendors Bloub (MIT) and Morphicons + Lucide. Auto TTS is Piper or silence — no formant pad. Desktop shows the orb while the gateway spawns. Co-authored-by: Byte271 --- THIRD_PARTY_NOTICES.md | 14 + apps/openlive-desktop/README.md | 5 +- apps/openlive-desktop/splash/index.html | 82 + apps/openlive-desktop/src/main.rs | 31 +- apps/openlive-desktop/tauri.conf.json | 3 +- apps/openlive-gateway/src/config.rs | 2 +- apps/openlive-gateway/src/main.rs | 31 +- apps/openlive-gateway/web/app.js | 181 +- apps/openlive-gateway/web/audio-session.js | 12 +- apps/openlive-gateway/web/bloub-orb.js | 268 +++ apps/openlive-gateway/web/call-controls.js | 59 + apps/openlive-gateway/web/call-state.js | 40 + apps/openlive-gateway/web/demo-voice.js | 56 + apps/openlive-gateway/web/index.html | 71 +- apps/openlive-gateway/web/package-lock.json | 503 ++++- apps/openlive-gateway/web/package.json | 8 +- .../web/scripts/vendor-ui.mjs | 26 + apps/openlive-gateway/web/setup-store.js | 6 +- apps/openlive-gateway/web/styles.css | 118 ++ .../web/tests/call-state.test.js | 35 + .../web/tests/demo-voice.test.js | 41 + .../web/tests/protocol.test.js | 2 + apps/openlive-gateway/web/tts-client.js | 20 +- apps/openlive-gateway/web/ui.js | 40 +- .../openlive-gateway/web/vendor/bloub/LICENSE | 21 + apps/openlive-gateway/web/vendor/bloub/NOTICE | 6 + .../web/vendor/bloub/engine.js | 1628 +++++++++++++++++ .../web/vendor/bloub/entry.js | 4 + .../web/vendor/bloub/src/cycles.ts | 225 +++ .../web/vendor/bloub/src/decor.ts | 285 +++ .../web/vendor/bloub/src/engine.ts | 558 ++++++ .../web/vendor/bloub/src/expressions.ts | 192 ++ .../web/vendor/bloub/src/eyefit.ts | 455 +++++ .../web/vendor/bloub/src/face.ts | 179 ++ .../web/vendor/bloub/src/math.ts | 42 + .../web/vendor/bloub/src/profiles.ts | 22 + .../web/vendor/bloub/src/repere.ts | 31 + .../web/vendor/bloub/src/shape.ts | 310 ++++ .../web/vendor/bloub/src/skins.ts | 147 ++ .../web/vendor/bloub/src/states.ts | 616 +++++++ .../web/vendor/licenses/lucide-LICENSE | 43 + .../web/vendor/licenses/morphicons-LICENSE | 21 + .../web/vendor/morphicons-entry.js | 2 + .../openlive-gateway/web/vendor/morphicons.js | 1349 ++++++++++++++ apps/openlive-gateway/web/visual-state.js | 10 +- apps/openlive-gateway/web/voice-visualizer.js | 443 +---- 46 files changed, 7677 insertions(+), 566 deletions(-) create mode 100644 apps/openlive-desktop/splash/index.html create mode 100644 apps/openlive-gateway/web/bloub-orb.js create mode 100644 apps/openlive-gateway/web/call-controls.js create mode 100644 apps/openlive-gateway/web/call-state.js create mode 100644 apps/openlive-gateway/web/demo-voice.js create mode 100644 apps/openlive-gateway/web/scripts/vendor-ui.mjs create mode 100644 apps/openlive-gateway/web/tests/call-state.test.js create mode 100644 apps/openlive-gateway/web/tests/demo-voice.test.js create mode 100644 apps/openlive-gateway/web/vendor/bloub/LICENSE create mode 100644 apps/openlive-gateway/web/vendor/bloub/NOTICE create mode 100644 apps/openlive-gateway/web/vendor/bloub/engine.js create mode 100644 apps/openlive-gateway/web/vendor/bloub/entry.js create mode 100644 apps/openlive-gateway/web/vendor/bloub/src/cycles.ts create mode 100644 apps/openlive-gateway/web/vendor/bloub/src/decor.ts create mode 100644 apps/openlive-gateway/web/vendor/bloub/src/engine.ts create mode 100644 apps/openlive-gateway/web/vendor/bloub/src/expressions.ts create mode 100644 apps/openlive-gateway/web/vendor/bloub/src/eyefit.ts create mode 100644 apps/openlive-gateway/web/vendor/bloub/src/face.ts create mode 100644 apps/openlive-gateway/web/vendor/bloub/src/math.ts create mode 100644 apps/openlive-gateway/web/vendor/bloub/src/profiles.ts create mode 100644 apps/openlive-gateway/web/vendor/bloub/src/repere.ts create mode 100644 apps/openlive-gateway/web/vendor/bloub/src/shape.ts create mode 100644 apps/openlive-gateway/web/vendor/bloub/src/skins.ts create mode 100644 apps/openlive-gateway/web/vendor/bloub/src/states.ts create mode 100644 apps/openlive-gateway/web/vendor/licenses/lucide-LICENSE create mode 100644 apps/openlive-gateway/web/vendor/licenses/morphicons-LICENSE create mode 100644 apps/openlive-gateway/web/vendor/morphicons-entry.js create mode 100644 apps/openlive-gateway/web/vendor/morphicons.js diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 93fb97d..548fd71 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -68,6 +68,20 @@ binaries and update this notice. --- +## Call surface UI + +Vendored under `apps/openlive-gateway/web/vendor/`. Keep the license files beside the bundles when redistributing. + +| Project | License | Use in OpenLive | +|---------|---------|-----------------| +| **[Bloub](https://github.com/jeremy-prt/bloub)** (Jérémy Perret) | MIT | Framework-free face/orb engine (`engine.sample(t)`). Design imitates the x.ai avatar. **OpenLive is not affiliated with xAI, x.ai, or OpenAI.** | +| **[Morphicons](https://github.com/guillermolg00/morphicons)** | MIT | Spring morphs for Mute / End / Settings (`createMorph` + lucide path data) | +| **[Lucide](https://lucide.dev/)** | ISC | Icon path data consumed by Morphicons (not `lucide-react`) | + +Morphicons are **not** used as the speaking face. + +--- + ## Fonts (web UI) | Family | Source | License | diff --git a/apps/openlive-desktop/README.md b/apps/openlive-desktop/README.md index c6b5730..28aa718 100644 --- a/apps/openlive-desktop/README.md +++ b/apps/openlive-desktop/README.md @@ -49,7 +49,8 @@ cargo tauri build ## Notes -- The desktop shell loads the same web UI as the browser version. -- The gateway server must be running locally for the voice surface to work. +- The desktop shell loads a local listening-orb splash immediately, then + navigates to the gateway UI once `http://127.0.0.1:12345/health` is up. + Spawning the gateway must not block first paint. - Replace `icons/icon.ico` and `icons/icon.icns` with branded assets before publishing. diff --git a/apps/openlive-desktop/splash/index.html b/apps/openlive-desktop/splash/index.html new file mode 100644 index 0000000..5ddc3f2 --- /dev/null +++ b/apps/openlive-desktop/splash/index.html @@ -0,0 +1,82 @@ + + + + + + OpenLive + + + +
+ + + + + +
+

Listening

+

Speak freely.

+
+
+ + + diff --git a/apps/openlive-desktop/src/main.rs b/apps/openlive-desktop/src/main.rs index 5fafcc1..af6f145 100644 --- a/apps/openlive-desktop/src/main.rs +++ b/apps/openlive-desktop/src/main.rs @@ -3,6 +3,9 @@ // Wraps the openlive-gateway web surface in a Tauri webview. The gateway // server is spawned as a child process on startup, kept alive for the // lifetime of the app, and killed on exit. +// +// The listening orb is shown immediately from a local splash page. Gateway +// spawn must not block first paint. #![cfg_attr(not(debug_assertions), windows_subsystem = "windows")] @@ -12,7 +15,7 @@ use std::sync::Mutex; use std::time::{Duration, Instant}; use tauri::Manager; -/// Port the gateway listens on (kept in sync with tauri.conf.json devUrl). +/// Port the gateway listens on (kept in sync with splash + web UI). const GATEWAY_PORT: u16 = 12345; static GATEWAY_CHILD: Mutex> = Mutex::new(None); @@ -20,20 +23,15 @@ static GATEWAY_CHILD: Mutex> = Mutex::new(None); fn main() { tauri::Builder::default() .setup(|app| { - // Resolve the gateway binary and web assets, then spawn the gateway. - // We do this inside setup() so we can use app.path().resource_dir() - // for bundled resources and fall back to the source tree in dev mode. let (gateway_exe, web_dir) = resolve_gateway_and_web(app.handle()); spawn_gateway(&gateway_exe, web_dir.as_deref()); - wait_for_gateway_ready(); - // Create the main window only after the gateway is ready so the - // webview never sees an ERR_CONNECTION_REFUSED page. - let url = format!("http://127.0.0.1:{GATEWAY_PORT}"); - tauri::WebviewWindowBuilder::new( + // Show the listening orb immediately. Splash polls /health, and a + // background thread also navigates once TCP is up. + let window = tauri::WebviewWindowBuilder::new( app, "main", - tauri::WebviewUrl::External(url.parse().unwrap()), + tauri::WebviewUrl::App("index.html".into()), ) .title("OpenLive") .inner_size(1280.0, 820.0) @@ -41,6 +39,15 @@ fn main() { .center() .build()?; + let window_for_poll = window.clone(); + std::thread::spawn(move || { + wait_for_gateway_ready(); + let js = format!( + "window.location.replace('http://127.0.0.1:{GATEWAY_PORT}/')" + ); + let _ = window_for_poll.eval(&js); + }); + Ok(()) }) .on_window_event(|_window, event| { @@ -83,7 +90,7 @@ fn resolve_gateway_and_web(handle: &tauri::AppHandle) -> (Option, Optio // 2. Dev mode: find the project root by walking up from the executable. let exe_dir = std::env::current_exe() .ok() - .and_then(|p| p.parent().map(|d| d.to_path_buf())) + .and_then(|p| p.parent().map(Path::to_path_buf)) .unwrap_or_else(|| PathBuf::from(".")); if let Some(project_root) = find_project_root(&exe_dir) { @@ -137,7 +144,7 @@ fn spawn_gateway(gateway_exe: &Option, web_dir: Option<&Path>) { match cmd.spawn() { Ok(child) => { eprintln!("[openlive-desktop] Gateway spawned (pid {})", child.id()); - *GATEWAY_CHILD.lock().unwrap() = Some(child); + *GATEWAY_CHILD.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(child); } Err(e) => { eprintln!("[openlive-desktop] Failed to spawn gateway: {e}"); diff --git a/apps/openlive-desktop/tauri.conf.json b/apps/openlive-desktop/tauri.conf.json index 1879b00..0a6f916 100644 --- a/apps/openlive-desktop/tauri.conf.json +++ b/apps/openlive-desktop/tauri.conf.json @@ -6,8 +6,7 @@ "build": { "beforeDevCommand": "", "beforeBuildCommand": "cargo build -p openlive-gateway --release", - "frontendDist": "http://127.0.0.1:12345", - "devUrl": "http://127.0.0.1:12345" + "frontendDist": "splash" }, "app": { "windows": [], diff --git a/apps/openlive-gateway/src/config.rs b/apps/openlive-gateway/src/config.rs index 982b77b..4013113 100644 --- a/apps/openlive-gateway/src/config.rs +++ b/apps/openlive-gateway/src/config.rs @@ -183,7 +183,7 @@ pub fn provider_catalog() -> serde_json::Value { { "id": "mock", "class": "mock", - "summary": "Offline formant duplex for demos (no external services)", + "summary": "Offline duplex for local UI (no external services; no formant on the demo path)", "cli": "--provider mock" }, { diff --git a/apps/openlive-gateway/src/main.rs b/apps/openlive-gateway/src/main.rs index 661714e..3581c63 100644 --- a/apps/openlive-gateway/src/main.rs +++ b/apps/openlive-gateway/src/main.rs @@ -299,9 +299,9 @@ async fn tts_status() -> Json { let st = piper_status(DEFAULT_PIPER_VOICE); Json(serde_json::json!({ "piper": st, - "fallback": "formant", + "fallback": "silence until Piper (formant only if engine=formant)", "browser_tts": "optional client-side", - "preferred": if st.available { "piper" } else { "formant" }, + "preferred": if st.available { "piper" } else { "silence" }, })) } @@ -334,7 +334,7 @@ async fn tts_speak( .and_then(serde_json::Value::as_str) .unwrap_or("auto"); - // Prefer Piper when installed; otherwise formant. + // Prefer Piper. Auto never falls through to formant (demo path = zero fake voice). if prefer != "formant" { match piper_synthesize(text, voice_id) { Ok((pcm, rate)) => { @@ -353,19 +353,16 @@ async fn tts_speak( .into_response(); } Err(e) => { - if prefer == "piper" { - let st = piper_status(voice_id); - return ( - StatusCode::SERVICE_UNAVAILABLE, - Json(serde_json::json!({ - "error": e, - "piper": st, - "hint": "Copy install_command_windows or install_command_unix from /v1/tts/status", - })), - ) - .into_response(); - } - // fall through to formant + let st = piper_status(voice_id); + return ( + StatusCode::SERVICE_UNAVAILABLE, + Json(serde_json::json!({ + "error": e, + "piper": st, + "hint": "Copy install_command_windows or install_command_unix from /v1/tts/status. Formant is only used when engine=formant.", + })), + ) + .into_response(); } } } @@ -1739,7 +1736,7 @@ async fn list_voices(State(state): State) -> Json { "active": active, "engine": if piper.available { "piper+formant" } else { "openlive-formant" }, "piper": piper, - "note": "Prefer open-source Piper when installed; formant is always available offline." + "note": "Prefer Piper when installed. Formant is available only when engine=formant." })) } diff --git a/apps/openlive-gateway/web/app.js b/apps/openlive-gateway/web/app.js index d38a09a..85ce0dd 100644 --- a/apps/openlive-gateway/web/app.js +++ b/apps/openlive-gateway/web/app.js @@ -8,7 +8,7 @@ * Architecture: * - `socket` is the binary WebSocket to /v1/realtime on the gateway. * - `audio` is the AudioSession (mic capture + playback worklets). - * - `visualizer` is the canvas orb renderer. + * - `visualizer` is the bloub orb (x.ai-style face; not affiliated with xAI). * - `transcript` is the in-memory TranscriptLog. * - `telemetry` is the ConnectionTelemetry rolling window. * - `settings` is the persisted UI preferences. @@ -87,6 +87,12 @@ import { } from "./speech-tts.js"; import { exportMemory, saveMemoryItem } from "./memory-client.js"; import { fetchTtsStatus, piperInstallUi, speakOpenLive } from "./tts-client.js"; +import { + allowInboundProviderPcm, + demoAllowsFormant, + isRealTtsReady, +} from "./demo-voice.js"; +import { installCallMorphs } from "./call-controls.js"; import { BACKCHANNEL_TOKENS, identityReply, @@ -243,9 +249,17 @@ const BUILTIN_PROVIDER_DETAILS = { custom: { id: "custom", name: "Custom", base_url: "http://127.0.0.1:8000/v1", default_model: "default", models: [], free_tier: true, description: "Any OpenAI-compatible base URL. Enter base URL, then pick or type a model id." }, }; -const visualizer = new VoiceVisualizer(controls.voiceOrb); +const visualizer = new VoiceVisualizer(controls.bloubOrb || controls.voiceOrb); visualizer.setMotionScale(settings.motionScale); +/** Morphicons on Mute / End / Settings. Installed after DOM listeners. */ +let callMorphs = { + setMuted() {}, + setSettingsOpen() {}, +}; +/** Piper (or other neural TTS) is ready — demo path stays silent until then. */ +let realTtsReady = false; + const mediaCapture = new MediaCaptureSession({ onState: handleCaptureState, onError: ({ message }) => showNotice(message), @@ -411,7 +425,7 @@ controls.settings?.addEventListener("click", () => { if (!controls.settingsVoice?.options?.length) { fillVoiceSelect(voices.length ? voices : OFFLINE_VOICES); } - toggleSettings(); + setCallSettingsOpen(); // Scroll settings body to top on open so the user always sees the first section. const settingsBody = document.querySelector(".settings-body"); if (settingsBody) { @@ -424,7 +438,7 @@ controls.settings?.addEventListener("click", () => { }); controls.closeSettings?.addEventListener("click", () => { playClick("soft"); - toggleSettings(false); + setCallSettingsOpen(false); }); controls.backchannels?.addEventListener("change", (event) => persistField("backchannels", event.target.value, /* reconfigure */ true), @@ -625,7 +639,7 @@ installShortcuts({ toggleMute: handleMuteToggle, toggleTranscript: () => toggleTranscript(), toggleDiagnostics: () => toggleDiagnostics(), - toggleSettings: () => toggleSettings(), + toggleSettings: () => setCallSettingsOpen(), toggleInstructions: () => { const open = toggleInstructions(); if (open) renderInstructionsPanel(customInstructions, onInstructionAxisChange); @@ -649,6 +663,8 @@ installShortcuts({ endConversation: endConversation, closeOverlays: () => { closeOverlays(); + callMorphs.setSettingsOpen(false); + controls.settings?.setAttribute("aria-expanded", "false"); }, showOnboarding: () => setOnboardingOpen(true), }); @@ -699,26 +715,48 @@ void bootstrapLlmUi().then(() => { // Paint Settings → Runtime as soon as the page loads (don't wait for Start). void refreshRuntimeStatus(); -if (!isSetupComplete()) { - openSetupWizard({ force: true }); -} else { - setSetupOpen(false); - if (!settings.onboardingDismissed) { - setOnboardingOpen(true); - } - void pushLlmConfig(setup).catch(() => {}); -} +// First paint is the call surface — do not force the setup wizard. +setSetupOpen(false); +void pushLlmConfig(setup).catch(() => {}); +void refreshTtsWarmup(); +window.setInterval(() => { + void refreshTtsWarmup(); +}, 4000); // Failsafe: dismiss splash after 3s even if bootstrapLlmUi hangs. setTimeout(dismissBootSplash, 3000); // Install ripple click feedback on all interactive elements. installRippleFeedback(); +callMorphs = installCallMorphs(); /* --------------------------------------------------------------------------- Primary action / conversation lifecycle --------------------------------------------------------------------------- */ +function setCallSettingsOpen(force) { + const next = toggleSettings(force); + callMorphs.setSettingsOpen(next); + controls.settings?.setAttribute("aria-expanded", String(next)); + return next; +} + +async function refreshTtsWarmup() { + try { + const status = await fetchTtsStatus(); + realTtsReady = isRealTtsReady(status); + if ( + !conversationActive && + (mode === VoiceMode.IDLE || mode === VoiceMode.STARTING) && + !realTtsReady + ) { + setVoiceMode(VoiceMode.IDLE, "Listening…"); + } + } catch { + realTtsReady = false; + } +} + function isInteractionBlocked() { return document.body.classList.contains("setup-open"); } @@ -771,6 +809,7 @@ function handleMuteToggle() { audio.stopMicrophone(); microphoneActive = false; setConversationActive(true, false, settings.entryMode === "ptt"); + callMorphs.setMuted(true); transition(VoiceMode.MUTED); stopSpeechRecognition(); return; @@ -778,6 +817,7 @@ function handleMuteToggle() { audio.startMicrophone().then(() => { microphoneActive = true; setConversationActive(true, true, settings.entryMode === "ptt"); + callMorphs.setMuted(false); hideNotice(); transition(VoiceMode.LISTENING); startSpeechRecognition(); @@ -840,6 +880,7 @@ async function beginConversation() { conversationActive = true; setConversationActive(true, true, settings.entryMode === "ptt"); + callMorphs.setMuted(false); transition(VoiceMode.LISTENING); startSpeechRecognition(); transcript.append("system", "Conversation started."); @@ -894,6 +935,7 @@ function endConversation() { renderTranscript(transcript.entries); resetExperience(); visualizer.setMode(VoiceMode.IDLE); + callMorphs.setMuted(false); telemetry.reset(); addTimeline("session", "Conversation ended"); } @@ -1949,11 +1991,20 @@ const CRITICAL_CONTROL_TYPES = new Set([ function handleMedia(packet) { observeServerSequence(packet.sequence); - // Prefer the gateway's native TTS pipeline (Piper/formant) when it is - // actively streaming PCM. This is more reliable than browser TTS alone - // and avoids the intermittent silence/hang issues seen on Windows - // Chrome/Edge with speechSynthesis. + // Prefer the gateway's native TTS pipeline (Piper) when it is + // actively streaming PCM. Mock formant frames are dropped on the + // demo path unless the user explicitly selected engine=formant. if (packet?.pcm?.length > 0 && audio) { + if (!allowInboundProviderPcm(loadSetup(), activeProvider)) { + if (!handleMedia._droppedMock) { + handleMedia._droppedMock = true; + addTimeline( + "tts", + "Ignored mock formant PCM — demo path uses Piper or silence", + ); + } + return; + } receivedMediaForGeneration = true; audio.enqueue(packet).catch((error) => { console.warn("Failed to enqueue server PCM:", error); @@ -2146,7 +2197,7 @@ function handleControl(envelope) { assistantText = ""; transcript.finalizeByGeneration(envelope.generation_id, finalText); renderTranscript(transcript.entries); - // Robust TTS: prefer gateway Piper/formant PCM, fall back to browser. + // Robust TTS: Piper (or configured engine). Auto never pads with formant. setup = loadSetup(); const isSoftAck = /^(mm-?hmm|mhmm|mhm)\.?$/i.test(finalText); // Dedupe: gateway can emit the same final twice under race conditions. @@ -2929,9 +2980,33 @@ function wireSetupSettingsBindings() { void withLoading(controls.settingsProbeAgent, probeAgentFromForm("settings")), ); controls.reopenSetup?.addEventListener("click", () => { - toggleSettings(false); + setCallSettingsOpen(false); openSetupWizard({ force: true }); }); + controls.settingsCamera?.addEventListener("click", () => { + setCallSettingsOpen(false); + void toggleCamera(); + }); + controls.settingsScreen?.addEventListener("click", () => { + setCallSettingsOpen(false); + void toggleScreenShare(); + }); + controls.settingsTranscript?.addEventListener("click", () => { + setCallSettingsOpen(false); + toggleTranscript(true); + }); + controls.settingsModes?.addEventListener("click", () => { + setCallSettingsOpen(false); + toggleModePicker(true); + }); + controls.settingsTasks?.addEventListener("click", () => { + setCallSettingsOpen(false); + setTaskRailVisible(true); + }); + controls.settingsDiagnostics?.addEventListener("click", () => { + setCallSettingsOpen(false); + toggleDiagnostics(true); + }); } async function bootstrapLlmUi() { @@ -3765,9 +3840,9 @@ function speechOpts(extra = {}) { } /** - * Speak assistant text with the best available engine. - * Tries gateway TTS (Piper/formant) first, then browser TTS as fallback. - * Handles failures gracefully and transitions back to listening when done. + * Speak assistant text with the configured engine. + * Auto/Piper: neural TTS only — silence + on-screen text if Piper is not ready. + * Formant and browser run only when the user explicitly selected them. */ async function speakAssistant(text, speakTurn, isSoftAck = false) { const localSetup = loadSetup(); @@ -3799,8 +3874,8 @@ async function speakAssistant(text, speakTurn, isSoftAck = false) { } } - // Fallback to browser TTS if gateway TTS is disabled or failed. - if (!gatewayOk && localSetup.browserTts !== false && browserTtsAvailable()) { + // Browser TTS only when the user picked that engine — never as an auto pad. + if (!gatewayOk && ttsEngine === "browser" && browserTtsAvailable()) { try { const fullySpoken = await speakBrowser( text, @@ -3820,9 +3895,8 @@ async function speakAssistant(text, speakTurn, isSoftAck = false) { } } - // If neither engine could speak, at least keep the conversation alive. - if (!gatewayOk && (localSetup.browserTts === false || !browserTtsAvailable())) { - addTimeline("tts", "No TTS engine available; text shown only"); + if (!gatewayOk && ttsEngine !== "browser") { + addTimeline("tts", "No neural TTS yet; showing text only"); } // Transition back to listening when appropriate. @@ -3886,8 +3960,8 @@ function showPiperInstallModal(ui) { } /** - * Always try to speak assistant text. - * Prefers open-source Piper → formant → browser. + * Speak assistant text for previews / explicit out-loud requests. + * Auto never pads with formant or browser TTS. * @param {string} text * @param {object} [extra] * @returns {Promise} @@ -3922,7 +3996,6 @@ async function speakAssistantOutLoud(text, extra = {}) { } } - // Prefer open-source Piper → formant → browser (browser quality is last resort). const spoken = await speakOpenLive(line, { voiceId: setup.voiceId || selectedVoice?.id, voiceURI: setup.browserVoiceURI || null, @@ -3934,14 +4007,18 @@ async function speakAssistantOutLoud(text, extra = {}) { const ok = !!spoken.ok; if (ok) { hideNotice(); - } else if (spoken.piper && !spoken.piper.available) { - const ui = piperInstallUi({ piper: spoken.piper }); - showPiperInstallModal(ui); + } else if ( + setup.ttsEngine === "piper" && + spoken.piper && + !spoken.piper.available && + !speakAssistantOutLoud._shownInstall + ) { + speakAssistantOutLoud._shownInstall = true; + showPiperInstallModal(piperInstallUi({ piper: spoken.piper })); + } else if ((setup.ttsEngine || "auto") === "auto") { + hideNotice(); } else { - showNotice( - spoken.error || - "Speech failed. Settings → TTS: try Formant or install Piper (open-source).", - ); + showNotice(spoken.error || "Speech failed."); } if (conversationActive && microphoneActive && !userEnded) { @@ -4037,7 +4114,8 @@ async function previewSelectedVoice() { setup = saveSetup({ browserVoiceURI: uri }); if (controls.settingsSystemVoice) controls.settingsSystemVoice.value = uri; } - if (setup.browserTts !== false && browserTtsAvailable() && catalog.length) { + const engine = setup.ttsEngine || "auto"; + if (engine === "browser" && setup.browserTts !== false && browserTtsAvailable() && catalog.length) { showNotice(`Playing: ${catalog.find((c) => c.id === uri)?.name || "system voice"}…`); stopBrowserSpeech(); const ok = await speakBrowser( @@ -4049,15 +4127,26 @@ async function previewSelectedVoice() { return; } } - // Always offer formant backup on preview failure. - showNotice("Browser voice failed — playing backup formant voice…"); - const data = await previewVoice(id, line); - await playPcmBase64(data.pcm_base64, data.sample_rate || 24000); + const spoken = await speakOpenLive(line, { + voiceId: id, + voiceURI: uri || null, + ttsEngine: engine, + }); + if (spoken.ok) { + hideNotice(); + return; + } + if (demoAllowsFormant(setup)) { + showNotice("Playing formant preview…"); + const data = await previewVoice(id, line); + await playPcmBase64(data.pcm_base64, data.sample_rate || 24000); + hideNotice(); + return; + } hideNotice(); showNotice( - catalog.length - ? "Backup voice works. For natural speech, pick a System voice and try Preview again (Edge works best)." - : "No system voices found. Install Windows Speech voices, or keep using backup formant voice.", + spoken.error || + "Preview silent until Piper is installed. Settings → TTS can switch to Formant.", ); } catch (e) { showNotice(e?.message || "Preview failed"); diff --git a/apps/openlive-gateway/web/audio-session.js b/apps/openlive-gateway/web/audio-session.js index 71d9b9f..c34a88b 100644 --- a/apps/openlive-gateway/web/audio-session.js +++ b/apps/openlive-gateway/web/audio-session.js @@ -8,13 +8,23 @@ import { } from "./audio-utils.js"; import { EmotionDetector } from "./emotion-detector.js"; +/** Local-first barge-in order. VAD ducks before any server round trip. */ +export const BARGE_IN_CHAIN = Object.freeze([ + "local_duck", + "soft_duck", + "hard_yield", + "cancel", +]); + /** * Openlive 26.7.16 — AudioSession * * Owns the AudioContext, the microphone capture worklet, the playback * worklet, and the output gain node. Bridges binary PCM frames between * the WebSocket and the worklets. Local-first interruption (the reversible - * duck before any server round trip) lives here. + * duck before any server round trip) lives here: + * local_duck → soft_duck → hard_yield → cancel generation + * VAD ducks playback before waiting on server RTT. * * Phase 1 client-side intelligence chain: * mic → RNNoise worklet → Silero VAD worklet → capture worklet diff --git a/apps/openlive-gateway/web/bloub-orb.js b/apps/openlive-gateway/web/bloub-orb.js new file mode 100644 index 0000000..cfc66d5 --- /dev/null +++ b/apps/openlive-gateway/web/bloub-orb.js @@ -0,0 +1,268 @@ +/** + * SVG host for the vendored bloub engine (MIT, Jérémy Perret). + * Eyes are mask holes in the body, matching the x.ai-style face. + * Not affiliated with xAI. + */ + +import { + BotEngine, + DEMI_VIEWBOX, + NOTIF_BLUE, + RAYON, +} from "./vendor/bloub/engine.js"; +import { bloubShouldDim, bloubStateFor } from "./call-state.js"; + +const VB = DEMI_VIEWBOX; +const PAPER = "#f4f4f2"; +const INK = "#0a0a0c"; + +export class BloubOrb { + /** + * @param {SVGSVGElement} svg + */ + constructor(svg) { + this.svg = svg; + this.engine = new BotEngine(RAYON, "idle"); + this.clock = 0; + this.last = 0; + this.mode = "idle"; + this.uid = `bloub-${Math.random().toString(36).slice(2, 8)}`; + this.reducedMotion = matchMedia("(prefers-reduced-motion: reduce)").matches; + this.motionScale = 1; + this.frame = null; + this.mount(); + this.tick = (now) => this.draw(now); + this.frame = requestAnimationFrame(this.tick); + } + + mount() { + const svg = this.svg; + svg.setAttribute("viewBox", `${-VB} ${-VB} ${VB * 2} ${VB * 2}`); + svg.setAttribute("role", "img"); + svg.setAttribute( + "aria-label", + "OpenLive face — inspired by the x.ai avatar; not affiliated with xAI", + ); + const ns = "http://www.w3.org/2000/svg"; + svg.replaceChildren(); + const defs = document.createElementNS(ns, "defs"); + const mask = document.createElementNS(ns, "mask"); + mask.id = `${this.uid}-mask`; + mask.setAttribute("maskUnits", "userSpaceOnUse"); + mask.setAttribute("x", String(-VB)); + mask.setAttribute("y", String(-VB)); + mask.setAttribute("width", String(VB * 2)); + mask.setAttribute("height", String(VB * 2)); + this.bodyMask = document.createElementNS(ns, "path"); + this.bodyMask.setAttribute("fill", "#fff"); + this.eyeMasks = [0, 1].map(() => { + const p = document.createElementNS(ns, "path"); + p.setAttribute("fill", "#000"); + return p; + }); + this.notch = document.createElementNS(ns, "circle"); + this.notch.setAttribute("fill", "#000"); + this.notch.setAttribute("visibility", "hidden"); + mask.append(this.bodyMask, ...this.eyeMasks, this.notch); + defs.append(mask); + this.gradRoot = document.createElementNS(ns, "g"); + defs.append(this.gradRoot); + + this.arcBack = document.createElementNS(ns, "g"); + this.arcBack.setAttribute("fill", "none"); + this.arcBack.setAttribute("stroke-linecap", "round"); + this.dotsBehind = document.createElementNS(ns, "g"); + this.bodyGroup = document.createElementNS(ns, "g"); + this.paperPath = document.createElementNS(ns, "path"); + this.paperPath.setAttribute("fill", PAPER); + const inkGroup = document.createElementNS(ns, "g"); + inkGroup.setAttribute("mask", `url(#${mask.id})`); + const ink = document.createElementNS(ns, "rect"); + ink.setAttribute("x", String(-VB)); + ink.setAttribute("y", String(-VB)); + ink.setAttribute("width", String(VB * 2)); + ink.setAttribute("height", String(VB * 2)); + ink.setAttribute("fill", INK); + inkGroup.append(ink); + this.bodyGroup.append(this.paperPath, inkGroup); + this.dotsFront = document.createElementNS(ns, "g"); + this.notif = document.createElementNS(ns, "circle"); + this.notif.setAttribute("fill", NOTIF_BLUE); + this.notif.setAttribute("visibility", "hidden"); + this.arcFront = document.createElementNS(ns, "g"); + this.arcFront.setAttribute("fill", "none"); + this.arcFront.setAttribute("stroke-linecap", "round"); + + svg.append( + defs, + this.arcBack, + this.dotsBehind, + this.bodyGroup, + this.dotsFront, + this.notif, + this.arcFront, + ); + } + + /** + * @param {string} mode + */ + setMode(mode) { + this.mode = mode; + const state = bloubStateFor(mode); + this.engine.setState(state, this.clock); + this.svg.classList.toggle("is-dimmed", bloubShouldDim(mode)); + } + + /** + * @param {number} input + * @param {number} output + */ + setSignals(input, output) { + const speaking = output > 0.04; + const listening = input > 0.12 && !speaking; + this.engine.setLook( + { + yaw: (input - 0.5) * 18, + pitch: speaking ? -8 : listening ? 6 : 0, + mix: speaking || listening ? 0.35 : 0, + spin: 0, + wander: bloubShouldDim(this.mode) ? 0.15 : 1, + }, + this.clock, + ); + } + + fireBargeIn() { + this.engine.setState("burst", this.clock); + } + + /** + * @param {number} scale + */ + setMotionScale(scale) { + this.motionScale = Math.max(0, Math.min(1, scale)); + } + + destroy() { + if (this.frame) cancelAnimationFrame(this.frame); + this.frame = null; + } + + draw(now) { + if (!this.frame) return; + this.frame = requestAnimationFrame(this.tick); + const dt = this.last ? (now - this.last) / 1000 : 0; + this.last = now; + const advance = this.reducedMotion ? 0 : dt * this.motionScale; + this.clock += advance; + this.paint(this.engine.sample(this.clock)); + } + + /** + * @param {import("./vendor/bloub/engine.js").BotFrame} frame + */ + paint(frame) { + this.bodyMask.setAttribute("d", frame.bodyPath); + this.paperPath.setAttribute("d", frame.bodyPath); + this.bodyGroup.setAttribute("opacity", String(frame.bodyAlpha)); + for (let i = 0; i < 2; i++) { + const eye = frame.eyes[i]; + const node = this.eyeMasks[i]; + if (!eye) { + node.setAttribute("d", ""); + continue; + } + node.setAttribute("d", eye.d); + node.setAttribute("transform", eye.matrix); + node.setAttribute("opacity", String(eye.alpha)); + } + if (frame.notch) { + this.notch.setAttribute("visibility", "visible"); + this.notch.setAttribute("cx", String(frame.notch.x)); + this.notch.setAttribute("cy", String(frame.notch.y)); + this.notch.setAttribute("r", String(frame.notch.r)); + } else { + this.notch.setAttribute("visibility", "hidden"); + } + if (frame.notif) { + this.notif.setAttribute("visibility", "visible"); + this.notif.setAttribute("cx", String(frame.notif.x)); + this.notif.setAttribute("cy", String(frame.notif.y)); + this.notif.setAttribute("r", String(frame.notif.r)); + } else { + this.notif.setAttribute("visibility", "hidden"); + } + this.paintDots(frame); + this.paintArcs(frame); + } + + paintDots(frame) { + const host = frame.dotsBehind ? this.dotsBehind : this.dotsFront; + const other = frame.dotsBehind ? this.dotsFront : this.dotsBehind; + other.replaceChildren(); + const ns = "http://www.w3.org/2000/svg"; + while (host.childNodes.length > frame.dots.length) { + host.lastChild.remove(); + } + frame.dots.forEach((dot, i) => { + let node = host.childNodes[i]; + const isPath = Boolean(dot.d); + if (!node || node.tagName !== (isPath ? "path" : "circle")) { + node = document.createElementNS(ns, isPath ? "path" : "circle"); + if (host.childNodes[i]) host.replaceChild(node, host.childNodes[i]); + else host.append(node); + } + node.setAttribute("fill", INK); + node.setAttribute("opacity", String(dot.opacity)); + if (isPath) { + node.setAttribute("d", dot.d); + node.setAttribute( + "transform", + `translate(${dot.x} ${dot.y}) rotate(${dot.rot ?? 0}) scale(${RAYON})`, + ); + } else { + node.setAttribute("cx", String(dot.x)); + node.setAttribute("cy", String(dot.y)); + node.setAttribute("r", String(dot.r)); + node.removeAttribute("transform"); + } + }); + } + + paintArcs(frame) { + const ns = "http://www.w3.org/2000/svg"; + this.gradRoot.replaceChildren(); + this.arcBack.replaceChildren(); + this.arcFront.replaceChildren(); + for (const arc of frame.arcs) { + const gid = `${this.uid}-${arc.id}`; + const grad = document.createElementNS(ns, "linearGradient"); + grad.id = gid; + grad.setAttribute("gradientUnits", "userSpaceOnUse"); + grad.setAttribute("x1", String(arc.grad.x1)); + grad.setAttribute("y1", String(arc.grad.y1)); + grad.setAttribute("x2", String(arc.grad.x2)); + grad.setAttribute("y2", String(arc.grad.y2)); + const stops = arc.grad.stops || []; + stops.forEach((c, i) => { + const stop = document.createElementNS(ns, "stop"); + stop.setAttribute("offset", String(stops.length > 1 ? i / (stops.length - 1) : 0)); + stop.setAttribute("stop-color", c); + grad.append(stop); + }); + this.gradRoot.append(grad); + for (const [host, d] of [ + [this.arcBack, arc.back], + [this.arcFront, arc.front], + ]) { + const path = document.createElementNS(ns, "path"); + path.setAttribute("d", d); + path.setAttribute("stroke", `url(#${gid})`); + path.setAttribute("stroke-width", String(arc.width)); + path.setAttribute("opacity", String(arc.opacity)); + host.append(path); + } + } + } +} diff --git a/apps/openlive-gateway/web/call-controls.js b/apps/openlive-gateway/web/call-controls.js new file mode 100644 index 0000000..1046947 --- /dev/null +++ b/apps/openlive-gateway/web/call-controls.js @@ -0,0 +1,59 @@ +/** + * Morphicons (MIT) on the default call chrome: Mute, End, Settings. + * Uses lucide data exports, not lucide-react. + */ + +import { createMorph, Mic, MicOff, PhoneOff, Settings, X } from "./vendor/morphicons.js"; + +function svgPathHost() { + const ns = "http://www.w3.org/2000/svg"; + const svg = document.createElementNS(ns, "svg"); + svg.setAttribute("viewBox", "0 0 24 24"); + svg.setAttribute("aria-hidden", "true"); + svg.setAttribute("fill", "none"); + svg.setAttribute("stroke", "currentColor"); + svg.setAttribute("stroke-width", "1.8"); + svg.setAttribute("stroke-linecap", "round"); + svg.setAttribute("stroke-linejoin", "round"); + const path = document.createElementNS(ns, "path"); + svg.append(path); + return { svg, path }; +} + +/** + * @param {HTMLElement | null} host + * @param {unknown} icon + */ +function mountMorph(host, icon) { + if (!host) return null; + host.replaceChildren(); + const { svg, path } = svgPathHost(); + host.append(svg); + return createMorph(path, icon, { reducedMotion: "user" }); +} + +/** + * Wire Mute / End / Settings morphs. Safe to call once after DOM is ready. + * + * @returns {{ + * setMuted: (muted: boolean) => void, + * setSettingsOpen: (open: boolean) => void, + * }} + */ +export function installCallMorphs() { + const mute = mountMorph(document.querySelector('[data-morph="mute"]'), Mic); + mountMorph(document.querySelector('[data-morph="end"]'), PhoneOff); + const settings = mountMorph( + document.querySelector('[data-morph="settings"]'), + Settings, + ); + + return { + setMuted(muted) { + mute?.morphTo(muted ? MicOff : Mic, "snappy"); + }, + setSettingsOpen(open) { + settings?.morphTo(open ? X : Settings, "snappy"); + }, + }; +} diff --git a/apps/openlive-gateway/web/call-state.js b/apps/openlive-gateway/web/call-state.js new file mode 100644 index 0000000..06e441a --- /dev/null +++ b/apps/openlive-gateway/web/call-state.js @@ -0,0 +1,40 @@ +/** + * Map OpenLive voice modes onto bloub engine states. + * + * Warming / listening → idle (gaze drift + blink) + * Speaking → orbit + * Interrupted / barge-in → burst, then back to listening + * Muted → idle (renderer dims the face) + */ + +import { VoiceMode } from "./visual-state.js"; + +const MAP = Object.freeze({ + [VoiceMode.IDLE]: "idle", + [VoiceMode.STARTING]: "idle", + [VoiceMode.LISTENING]: "idle", + [VoiceMode.THINKING]: "thinking", + [VoiceMode.SPEAKING]: "orbit", + [VoiceMode.YIELDING]: "wink", + [VoiceMode.INTERRUPTED]: "burst", + [VoiceMode.MUTED]: "idle", + [VoiceMode.RECONNECTING]: "swirl", + [VoiceMode.CONNECTION_ERROR]: "idle", + [VoiceMode.ERROR]: "idle", +}); + +/** + * @param {string} mode + * @returns {string} + */ +export function bloubStateFor(mode) { + return MAP[mode] || "idle"; +} + +/** + * @param {string} mode + * @returns {boolean} + */ +export function bloubShouldDim(mode) { + return mode === VoiceMode.MUTED; +} diff --git a/apps/openlive-gateway/web/demo-voice.js b/apps/openlive-gateway/web/demo-voice.js new file mode 100644 index 0000000..d85da20 --- /dev/null +++ b/apps/openlive-gateway/web/demo-voice.js @@ -0,0 +1,56 @@ +/** + * OpenLive demo-path voice policy. + * + * Happy path (ttsEngine === "auto"): speak only with real neural TTS (Piper). + * Never pad with formant/mock — the fake→real switch is an audible tell. + * Formant is allowed only when the user explicitly selects that engine + * (first-run / last-resort fallback). + */ + +export const DEMO_TTS_POLICY = "neural-or-silence"; + +/** + * @param {{ ttsEngine?: string } | null | undefined} setup + * @returns {boolean} + */ +export function demoAllowsFormant(setup) { + return (setup?.ttsEngine || "auto") === "formant"; +} + +/** + * Engine string sent to `/v1/tts/speak`. Auto never requests formant. + * + * @param {{ ttsEngine?: string } | null | undefined} setup + * @returns {"auto" | "piper" | "formant" | "browser"} + */ +export function neuralSpeakEngine(setup) { + const engine = setup?.ttsEngine || "auto"; + if (engine === "formant" || engine === "browser" || engine === "piper") { + return engine; + } + return "piper"; +} + +/** + * @param {{ piper?: { available?: boolean }, preferred?: string } | null | undefined} status + * @returns {boolean} + */ +export function isRealTtsReady(status) { + return Boolean(status?.piper?.available); +} + +/** + * Whether inbound provider PCM may play on the demo path. + * Mock formant frames are dropped unless the user opted into formant TTS. + * + * @param {{ ttsEngine?: string } | null | undefined} setup + * @param {{ id?: string, provider_class?: string } | null | undefined} provider + * @returns {boolean} + */ +export function allowInboundProviderPcm(setup, provider) { + if (demoAllowsFormant(setup)) return true; + const id = String(provider?.id || ""); + const klass = String(provider?.provider_class || ""); + const isMock = klass === "mock" || id.includes("mock"); + return !isMock; +} diff --git a/apps/openlive-gateway/web/index.html b/apps/openlive-gateway/web/index.html index c74667c..480f633 100644 --- a/apps/openlive-gateway/web/index.html +++ b/apps/openlive-gateway/web/index.html @@ -59,7 +59,7 @@ -