diff --git a/Cargo.lock b/Cargo.lock index dfdb243..4e6f592 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3072,6 +3072,7 @@ dependencies = [ "ratatui-image", "reqwest", "rmcp", + "rpassword", "runlet", "serde", "serde_json", @@ -4508,6 +4509,27 @@ dependencies = [ "url", ] +[[package]] +name = "rpassword" +version = "7.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2da316a15f47e3d053de9cb2c439650bd8fa4aaeb9365f2e5f27f492ff73c196" +dependencies = [ + "libc", + "rtoolbox", + "windows-sys 0.61.2", +] + +[[package]] +name = "rtoolbox" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a1efe12a1469752d0e6ff5ebec0b6ef4924cc5c4c71046b0ec730040535819d" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "runlet" version = "0.6.0" diff --git a/Cargo.toml b/Cargo.toml index d2d975f..182a334 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,6 +77,7 @@ ratatui = { version = "=0.30.2", default-features = false, features = ["crosster ratatui-image = { version = "=11.0.8", default-features = false, features = ["crossterm"], optional = true } reqwest = { version = "=0.13.5", default-features = false, features = ["blocking", "form", "rustls", "stream"] } rmcp = { version = "=3.2.0", default-features = false, features = ["auth", "client"] } +rpassword = "=7.5.4" serde = { version = "=1.0.229", features = ["derive"] } serde_json = "=1.0.151" shlex = { version = "=2.0.1", optional = true } diff --git a/docs/user/evaluations.md b/docs/user/evaluations.md new file mode 100644 index 0000000..c870940 --- /dev/null +++ b/docs/user/evaluations.md @@ -0,0 +1,80 @@ +# Evaluate content with named questions + +Use `eval` to classify content, rate it against a rubric, or estimate whether a +condition holds. One call evaluates multiple independent questions against the +same supplied state and returns named answers. + +## Enable evaluations + +Evaluations are off by default. Enable them in your user configuration only: + +```sh +kit config set experimental.eval true +kit config set credential_store keychain +kit auth login typesafe +``` + +These settings persist in `~/.kit/config.toml`: evaluations are enabled and +login and subsequent Kit runs use the same persistent `keychain` credential +store. Do not override that store when starting Kit. See +[TypeSafe authentication](getting-started-and-configuration.md#typesafe-api-key) +for the API key prerequisite and the file-store alternative; if using that +alternative, persist both `credential_store` and `credential_dir` with +`kit config set` so login and subsequent runs share the same store. A project's +configuration cannot enable evaluations. You can use `TYPESAFE_API_KEY` instead +of a saved key; a nonempty environment value takes precedence. Restart Kit after +enabling the feature or adding a key. Without both opt-in and an available key, +`eval` is not offered to the agent. + +**Each evaluation sends the supplied state and questions to TypeSafe and uses +your TypeSafe quota.** Supply only content you intend to share. Kit does not +choose content automatically, compact conversations, or filter other tool +results through evaluations. Evaluations introduce no content logging or +telemetry. As with other tool calls, inputs and results are part of the session. + +Disable with `kit config set experimental.eval false` and restart Kit. + +## Ask several questions in one call + +`eval` is a hidden Compose tool, not a separate Runlet expression: + +```text +return eval({ + state: "My invoice was charged twice. Please help.", + questions: { + urgent: { type: "noul", instructions: "Does this require immediate attention?" }, + department: { + type: "choice", + instructions: "Which department should handle this?", + criteria: { billing: "Payments and invoices", support: "Product help" } + }, + frustration: { + type: "score", + instructions: "How frustrated is the customer?", + criteria: ["Calm", "Frustrated", "Very angry"] + } + } +}) +``` + +- **Noul** returns `noul`, the probability of yes from 0 to 1. Optional `criteria` + can describe `true` and `false`. +- **Choice** returns `choice`, option `probabilities`, and `confidence`. Supply + 1–255 named options; a description can be `null`. +- **Score** returns `score`, level `probabilities`, `legend`, and `confidence`. + Supply 2–10 ordered levels. Levels start at zero; scores can fall between them. + +State and instructions accept text, objects, or arrays. Each result includes the +resolved Jev model version and input/output token usage. The selected model is +`jev-latest`. Answers are probabilistic assessments, not guarantees. This tool +does not generate arbitrary JSON schemas. + +A call accepts up to 64 questions and 256 KiB of encoded input. Results are +limited to 1 MiB. Up to four evaluations run concurrently per runtime; each +call has a 30-second deadline, including time waiting to start. Kit sends one +request per call, does not split or batch calls, and never automatically retries. +Cancellation stops waiting but cannot undo an evaluation already submitted; +a cancelled, timed-out, or failed call may still use quota. + +See the [TypeSafe API reference](https://docs.typesafe.ai/api.md) for the question +and answer definitions. diff --git a/docs/user/getting-started-and-configuration.md b/docs/user/getting-started-and-configuration.md index 51f9bc6..37491a9 100644 --- a/docs/user/getting-started-and-configuration.md +++ b/docs/user/getting-started-and-configuration.md @@ -87,6 +87,36 @@ kit prompt --provider openrouter \ Kit uses the CLI or TOML `model`; `OPENROUTER_MODEL` does not override that selection. The adapter also accepts `OPENROUTER_BASE_URL`, `OPENROUTER_APP_NAME`, `OPENROUTER_SITE_URL`, `OPENROUTER_MAX_COMPLETION_TOKENS`, `OPENROUTER_TEMPERATURE`, and `OPENROUTER_REASONING_EFFORT`. Its model-catalog lookup for the selected model's context length is best-effort, so a catalog failure does not by itself prevent normal provider usage. +### TypeSafe API key + +TypeSafe authentication is BYOK (bring your own key). Create a key at +, then enter it at the hidden terminal prompt: + +```sh +kit auth login typesafe --credential-store keychain +kit auth status typesafe --credential-store keychain +kit auth logout typesafe --credential-store keychain +``` + +Use the same storage option for all three commands. You can instead +use `--credential-store file --credential-dir /path/to/private-directory`. +Login requires `keychain` or `file` to save your key. Enter the key only at the +hidden prompt, not in command-line arguments or `config.toml`. + +Status gives a nonempty `TYPESAFE_API_KEY` environment variable precedence over a +stored key; an empty variable does not override stored credentials. Login still +prompts and saves a key when the variable is set, and warns about this precedence. +Login checks the key with TypeSafe before saving it, without running inference +or incurring inference charges. If authentication fails, your previously saved +key is unchanged. Successful login confirms authentication. Status reports +whether a key is configured; it does not check whether that key is still valid. + +Logout removes only the stored key and warns if `TYPESAFE_API_KEY` remains active. +Revoke the key separately at ; `--local-only` +suppresses that reminder. These commands manage credentials only: TypeSafe is not +a model provider, and this does not enable Jev inference, compaction, filtering, +or behavior configuration. + ### Speakeasy AI Control Plane Sign in through the Speakeasy dashboard, then use the same persistent credential @@ -369,3 +399,10 @@ or task deadline. The setting is resolved once per Kit process. Native Kit ACP children and the TUI's Kit server receive the exact resolved value as a CLI argument, for both new and resumed sessions; their local TOML cannot override it. + +### Experimental evaluations + +`experimental.eval` defaults to `false`. Enable it in your user configuration to +ask named classification, scoring, and yes/no questions with TypeSafe. A TypeSafe +key is also required. Each call shares its supplied content with TypeSafe and +uses your quota. See [Evaluations](evaluations.md) for setup, examples, and limits. diff --git a/src/docs.rs b/src/docs.rs index 6822e56..56a7ad2 100644 --- a/src/docs.rs +++ b/src/docs.rs @@ -416,6 +416,7 @@ mod tests { vec![ "docs/user/agent-plugins.md", "docs/user/compose-and-local-tools.md", + "docs/user/evaluations.md", "docs/user/getting-started-and-configuration.md", "docs/user/mcp.md", "docs/user/migrating-from-claude-code-and-codex.md", diff --git a/src/main.rs b/src/main.rs index e35d47f..7698e10 100644 --- a/src/main.rs +++ b/src/main.rs @@ -458,7 +458,7 @@ impl AuthAction { fn augment_command(command: clap::Command) -> clap::Command { let command = command.subcommand({ let command = clap::Command::new("login") - .about("Authenticate a model provider in the configured credential store"); + .about("Authenticate a service in the configured credential store"); let command = command.group( clap::ArgGroup::new("Login") .multiple(true) @@ -473,8 +473,7 @@ impl AuthAction { ) }); let command = command.subcommand({ - let command = - clap::Command::new("status").about("Show model-provider authentication status"); + let command = clap::Command::new("status").about("Show service authentication status"); let command = command.group( clap::ArgGroup::new("Status") .multiple(true) @@ -490,7 +489,7 @@ impl AuthAction { }); command.subcommand({ let command = clap::Command::new("logout") - .about("Remove model-provider credentials, revoking them when supported"); + .about("Remove service credentials, revoking them when supported"); let command = command.group( clap::ArgGroup::new("Logout") .multiple(true) @@ -1246,13 +1245,19 @@ impl ValueEnum for AcpProtocolVersion { impl ValueEnum for AuthProvider { fn value_variants<'a>() -> &'a [Self] { - &[Self::Openai, Self::Openrouter, Self::Speakeasy] + &[ + Self::Openai, + Self::Openrouter, + Self::Speakeasy, + Self::Typesafe, + ] } fn to_possible_value(&self) -> Option { Some(clap::builder::PossibleValue::new(match self { Self::Openai => "openai", Self::Openrouter => "openrouter", Self::Speakeasy => "speakeasy", + Self::Typesafe => "typesafe", })) } } @@ -1314,7 +1319,6 @@ fn migrate_config(mut config: toml::Table) -> toml::Table { #[derive(Debug, Default, Deserialize)] struct Config { - #[cfg(feature = "tui")] #[serde(default, deserialize_with = "deserialize_experimental_config")] experimental: ExperimentalConfig, request_budget_seconds: Option, @@ -1344,7 +1348,6 @@ struct Config { config_path: Option, } -#[cfg(feature = "tui")] fn deserialize_experimental_config<'de, D>(deserializer: D) -> Result where D: serde::Deserializer<'de>, @@ -1353,9 +1356,11 @@ where table.try_into().map_err(serde::de::Error::custom) } -#[cfg(feature = "tui")] #[derive(Debug, Default, Deserialize)] struct ExperimentalConfig { + #[serde(default)] + eval: bool, + #[cfg(feature = "tui")] #[serde(default)] voice: bool, } @@ -1578,24 +1583,15 @@ enum AuthProvider { Openai, Openrouter, Speakeasy, -} - -impl AuthProvider { - const fn provider_kind(self) -> kit::ProviderKind { - match self { - Self::Openai => kit::ProviderKind::OpenAiSubscription, - Self::Openrouter => kit::ProviderKind::OpenRouter, - Self::Speakeasy => kit::ProviderKind::Speakeasy, - } - } + Typesafe, } enum AuthAction { - /// Authenticate a model provider in the configured credential store. + /// Authenticate a service in the configured credential store. Login { provider: AuthProvider }, - /// Show model-provider authentication status. + /// Show service authentication status. Status { provider: AuthProvider }, - /// Remove model-provider credentials, revoking them when supported. + /// Remove service credentials, revoking them when supported. Logout { provider: AuthProvider, /// Remove local credentials without attempting remote revocation. @@ -1831,6 +1827,7 @@ async fn execute_auth( OpenAi(kit::provider::OpenAiAuthCommand), OpenRouter(kit::provider::OpenRouterAuthCommand), Speakeasy(kit::provider::SpeakeasyAuthCommand), + TypeSafe(kit::provider::TypeSafeAuthCommand), Logout(kit::ProviderKind, bool), } let command = match action { @@ -1842,6 +1839,9 @@ async fn execute_auth( AuthProvider::Speakeasy => { Execution::Speakeasy(kit::provider::SpeakeasyAuthCommand::Login) } + AuthProvider::Typesafe => { + Execution::TypeSafe(kit::provider::TypeSafeAuthCommand::Login) + } }, AuthAction::Status { provider } => match provider { AuthProvider::Openai => Execution::OpenAi(kit::provider::OpenAiAuthCommand::Status), @@ -1851,11 +1851,27 @@ async fn execute_auth( AuthProvider::Speakeasy => { Execution::Speakeasy(kit::provider::SpeakeasyAuthCommand::Status) } + AuthProvider::Typesafe => { + Execution::TypeSafe(kit::provider::TypeSafeAuthCommand::Status) + } }, AuthAction::Logout { provider, local_only, - } => Execution::Logout(provider.provider_kind(), *local_only), + } => match provider { + AuthProvider::Openai => { + Execution::Logout(kit::ProviderKind::OpenAiSubscription, *local_only) + } + AuthProvider::Openrouter => { + Execution::Logout(kit::ProviderKind::OpenRouter, *local_only) + } + AuthProvider::Speakeasy => Execution::Logout(kit::ProviderKind::Speakeasy, *local_only), + AuthProvider::Typesafe => { + Execution::TypeSafe(kit::provider::TypeSafeAuthCommand::Logout { + local_only: *local_only, + }) + } + }, }; let output = tokio::task::spawn_blocking(move || match command { Execution::OpenAi(command) => kit::provider::execute_openai_auth(command, &storage), @@ -1867,6 +1883,7 @@ async fn execute_auth( .map(|(key, source)| (key, *source)), ), Execution::Speakeasy(command) => kit::provider::execute_speakeasy_auth(command, &storage), + Execution::TypeSafe(command) => kit::provider::execute_typesafe_auth(command, &storage), Execution::Logout(provider, local_only) => kit::provider::execute_provider_logout( provider, &storage, @@ -2194,6 +2211,7 @@ async fn run_cli(cli: Cli) -> Result<(), Box> { openrouter_api_key.as_ref().map(|(key, _)| key.clone()), )?, }; + let runtime = kit::Runtime::with_eval(runtime, config.experimental.eval)?; let runtime = kit::Runtime::with_plugin_runtime(runtime, plugins)?; let runtime = kit::Runtime::with_telemetry(runtime, telemetry_settings.clone())?; let (harnesses, default_harness) = config.harnesses()?; @@ -2291,6 +2309,7 @@ async fn run_cli(cli: Cli) -> Result<(), Box> { openrouter_api_key.as_ref().map(|(key, _)| key.clone()), )?, }; + let runtime = kit::Runtime::with_eval(runtime, config.experimental.eval)?; let runtime = kit::Runtime::with_plugin_runtime(runtime, plugins)?; let runtime = kit::Runtime::with_telemetry(runtime, telemetry_settings.clone())?; let runtime = kit::Runtime::with_depth(runtime, subagent_depth)?; @@ -2358,6 +2377,7 @@ async fn run_cli(cli: Cli) -> Result<(), Box> { reasoning_effort, openrouter_api_key.as_ref().map(|(key, _)| key.clone()), )?; + let runtime = kit::Runtime::with_eval(runtime, config.experimental.eval)?; let runtime = kit::Runtime::with_plugin_runtime(runtime, plugins)?; let runtime = kit::Runtime::with_telemetry(runtime, telemetry_settings.clone())?; let (harnesses, default_harness) = config.harnesses()?; @@ -2538,6 +2558,31 @@ mod tests { assert!(help.contains("session picker without an ID")); } + #[test] + fn config_experimental_eval_defaults_strict_and_writer() { + assert!(!Config::default().experimental.eval); + for (text, expected) in [ + ("", false), + ("[experimental]", false), + ("[experimental]\neval = true", true), + ("[experimental]\neval = false", false), + ("[experimental]\nfuture = true", false), + ] { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("config.toml"); + std::fs::write(&path, text).unwrap(); + assert_eq!(Config::load(&path).unwrap().experimental.eval, expected); + kit::config_editor::set(&path, "experimental.eval", "true").unwrap(); + assert!(Config::load(&path).unwrap().experimental.eval); + kit::config_editor::unset(&path, "experimental.eval").unwrap(); + assert!(!Config::load(&path).unwrap().experimental.eval); + } + for value in ["'true'", "1", "[]", "{}"] { + let text = format!("[experimental]\neval = {value}"); + assert!(toml::from_str::(&text).is_err()); + } + } + #[cfg(feature = "tui")] #[test] fn config_experimental_voice_defaults_and_strict_boolean() { @@ -3847,6 +3892,15 @@ future_option = true #[test] fn auth_commands_parse_without_runtime_arguments() { + for action in ["login", "status", "logout"] { + assert!(Cli::try_parse_from(["kit", "auth", action, "typesafe"]).is_ok()); + } + assert!(Cli::try_parse_from(["kit", "auth", "logout", "typesafe", "--local-only"]).is_ok()); + assert!( + Cli::try_parse_from(["kit", "auth", "login", "typesafe", "--api-key", "secret"]) + .is_err() + ); + assert!(Cli::try_parse_from(["kit", "prompt", "--provider", "typesafe", "hello"]).is_err()); assert!(Cli::try_parse_from(["kit", "auth", "login", "openai"]).is_ok()); assert!( Cli::try_parse_from([ @@ -3891,6 +3945,7 @@ future_option = true AuthProvider::Openai, AuthProvider::Openrouter, AuthProvider::Speakeasy, + AuthProvider::Typesafe, ] { let login = AuthAction::Login { provider }; assert!(validate_auth_storage(&login, &CredentialStorage::Memory).is_err()); diff --git a/src/provider/mod.rs b/src/provider/mod.rs index ccf1d44..5a8e31f 100644 --- a/src/provider/mod.rs +++ b/src/provider/mod.rs @@ -3,6 +3,10 @@ pub mod chatgpt; pub(crate) mod openai_auth; mod openrouter_auth; mod speakeasy_auth; +pub(crate) mod typesafe_auth; + +#[doc(hidden)] +pub use typesafe_auth::{TypeSafeAuthCommand, execute_typesafe_auth}; pub(crate) use adapter::authentication_method_id; pub use adapter::{ diff --git a/src/provider/typesafe_auth.rs b/src/provider/typesafe_auth.rs new file mode 100644 index 0000000..ea2d460 --- /dev/null +++ b/src/provider/typesafe_auth.rs @@ -0,0 +1,486 @@ +use std::time::Duration; + +use serde::{Deserialize, Serialize}; +use zeroize::{Zeroize, ZeroizeOnDrop, Zeroizing}; + +use crate::credentials::CredentialStorage; + +const NAMESPACE: &str = "typesafe"; +const IDENTITY: &str = "default"; +const KEYS_URL: &str = "https://console.typesafe.ai/keys"; +const MODELS_URL: &str = "https://api.typesafe.ai/v1/models"; +const LOGIN_TIMEOUT: Duration = Duration::from_secs(15); +const MAX_RECORD_BYTES: usize = 16 * 1024; + +#[doc(hidden)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum TypeSafeAuthCommand { + Login, + Status, + Logout { local_only: bool }, +} + +// A new, isolated credential record, not a model-provider configuration. +#[derive(Deserialize, Serialize, Zeroize, ZeroizeOnDrop)] +#[serde(deny_unknown_fields)] +pub(crate) struct Credentials { + api_key: String, +} + +impl Credentials { + pub(crate) fn api_key(&self) -> &str { + &self.api_key + } +} + +/// Resolve the active key without making a network request. +pub(crate) fn resolve_api_key(storage: &CredentialStorage) -> Result, String> { + let environment = match std::env::var("TYPESAFE_API_KEY") { + Ok(value) => Some(Zeroizing::new(value)), + Err(std::env::VarError::NotPresent) => None, + Err(std::env::VarError::NotUnicode(_)) => { + return Err("Invalid TYPESAFE_API_KEY. Check the key and try again.".into()); + } + }; + resolve_key(storage, environment.as_deref().map(String::as_str)) +} + +fn resolve_key( + storage: &CredentialStorage, + environment: Option<&str>, +) -> Result, String> { + if let Some(key) = environment.filter(|key| !key.is_empty()) { + let record = Credentials { + api_key: key.into(), + }; + validate(&record)?; + Ok(Some(record)) + } else { + load(storage) + } +} + +impl std::fmt::Debug for Credentials { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("TypeSafeCredentials([REDACTED])") + } +} + +#[doc(hidden)] +pub fn execute_typesafe_auth( + command: TypeSafeAuthCommand, + storage: &CredentialStorage, +) -> Result { + let environment_active = + std::env::var_os("TYPESAFE_API_KEY").is_some_and(|value| !value.is_empty()); + execute(command, storage, environment_active) +} + +fn execute( + command: TypeSafeAuthCommand, + storage: &CredentialStorage, + environment_active: bool, +) -> Result { + match command { + TypeSafeAuthCommand::Login => { + if !storage.is_persistent() { + return Err("To save your TypeSafe key, use --credential-store keychain or --credential-store file.".into()); + } + eprintln!("Create a TypeSafe API key at {KEYS_URL}"); + let record = Credentials { + api_key: rpassword::prompt_password("TypeSafe API key (hidden): ") + .map_err(|_| "could not read TypeSafe API key from the terminal".to_string())?, + }; + complete_login( + storage, + &record, + environment_active, + MODELS_URL, + LOGIN_TIMEOUT, + ) + } + TypeSafeAuthCommand::Status => { + if environment_active { + Ok("TypeSafe: configured via TYPESAFE_API_KEY.\n".into()) + } else if load(storage)?.is_some() { + Ok("TypeSafe: configured.\n".into()) + } else { + Ok("TypeSafe: not configured.\n".into()) + } + } + TypeSafeAuthCommand::Logout { local_only } => { + let removed = storage + .entry(NAMESPACE, IDENTITY) + .delete() + .map_err(|_| "Could not remove your TypeSafe key. Try again.".to_string())?; + let mut output = if removed { + "TypeSafe key removed.\n".to_string() + } else { + "TypeSafe: no saved key.\n".to_string() + }; + if !local_only { + output.push_str(&format!("Revoke TypeSafe API keys at {KEYS_URL}.\n")); + } + if environment_active { + output.push_str("Warning: TYPESAFE_API_KEY remains active after logout.\n"); + } + Ok(output) + } + } +} + +// https://docs.typesafe.ai/models documents this authenticated, non-inference GET. +// Keep redirects disabled so validation cannot forward the key to another URL. +fn complete_login( + storage: &CredentialStorage, + record: &Credentials, + environment_active: bool, + models_url: &str, + timeout: Duration, +) -> Result { + validate(record)?; + let unavailable = || "Could not authenticate with TypeSafe. Try again later.".to_string(); + let client = reqwest::blocking::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .connect_timeout(timeout.min(Duration::from_secs(5))) + .timeout(timeout) + .build() + .map_err(|_| unavailable())?; + let response = client + .get(models_url) + .bearer_auth(&record.api_key) + .send() + .map_err(|_| unavailable())?; + match response.status() { + reqwest::StatusCode::OK => {} + reqwest::StatusCode::UNAUTHORIZED | reqwest::StatusCode::FORBIDDEN => { + return Err("TypeSafe rejected this API key. Check the key and try again.".into()); + } + _ => return Err(unavailable()), + } + // Only the authenticated status matters; never read or expose response bodies. + save(storage, record)?; + let mut output = "TypeSafe: authenticated. API key saved.\n".to_string(); + if environment_active { + output.push_str("Warning: TYPESAFE_API_KEY overrides your saved key.\n"); + } + Ok(output) +} + +fn validate(record: &Credentials) -> Result<(), String> { + if record.api_key.is_empty() + || record + .api_key + .chars() + .any(|c| c.is_whitespace() || c.is_control()) + { + return Err("Invalid TypeSafe API key. Copy the key and try again.".into()); + } + if record.api_key.len() > MAX_RECORD_BYTES { + return Err("Invalid TypeSafe API key. Copy the key and try again.".into()); + } + Ok(()) +} + +fn load(storage: &CredentialStorage) -> Result, String> { + let Some(bytes) = storage + .entry(NAMESPACE, IDENTITY) + .load() + .map_err(|_| "Could not read your TypeSafe key. Log in again.".to_string())? + else { + return Ok(None); + }; + if bytes.len() > MAX_RECORD_BYTES { + return Err("Could not read your TypeSafe key. Log in again.".into()); + } + let record: Credentials = serde_json::from_slice(&bytes) + .map_err(|_| "Could not read your TypeSafe key. Log in again.".to_string())?; + validate(&record)?; + Ok(Some(record)) +} + +fn save(storage: &CredentialStorage, record: &Credentials) -> Result<(), String> { + validate(record)?; + let bytes = Zeroizing::new( + serde_json::to_vec(record) + .map_err(|_| "Could not save your TypeSafe key. Try again.".to_string())?, + ); + if bytes.len() > MAX_RECORD_BYTES { + return Err("Invalid TypeSafe API key. Copy the key and try again.".into()); + } + storage + .entry(NAMESPACE, IDENTITY) + .save(&bytes) + .map_err(|_| "Could not save your TypeSafe key. Try again.".to_string()) +} + +#[cfg(test)] +#[allow(clippy::disallowed_methods, clippy::disallowed_macros)] +mod tests { + use super::*; + + use std::io::{Read, Write}; + use std::net::TcpListener; + + // A real HTTP boundary: capture the request and return an upstream response. + fn serve(response: String, delay: Duration) -> (String, std::thread::JoinHandle) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}/v1/models", listener.local_addr().unwrap()); + let task = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + let mut request = Vec::new(); + let mut byte = [0]; + while !request.ends_with(b"\r\n\r\n") { + if stream.read(&mut byte).unwrap() == 0 { + break; + } + request.push(byte[0]); + assert!(request.len() < 32 * 1024); + } + std::thread::sleep(delay); + let _ = stream.write_all(response.as_bytes()); + String::from_utf8(request).unwrap() + }); + (url, task) + } + + #[test] + fn key_resolution_prefers_nonempty_environment_and_redacts_keys() { + let directory = tempfile::tempdir().unwrap(); + let storage = CredentialStorage::Filesystem(directory.path().into()); + assert!(resolve_key(&storage, None).unwrap().is_none()); + save( + &storage, + &Credentials { + api_key: "stored-secret".into(), + }, + ) + .unwrap(); + for environment in [None, Some("")] { + assert_eq!( + resolve_key(&storage, environment) + .unwrap() + .unwrap() + .api_key(), + "stored-secret" + ); + } + let key = resolve_key(&storage, Some("environment-secret")) + .unwrap() + .unwrap(); + assert_eq!(key.api_key(), "environment-secret"); + assert!(!format!("{key:?}").contains("environment-secret")); + assert!(resolve_key(&storage, Some("invalid secret")).is_err()); + storage.entry(NAMESPACE, IDENTITY).save(b"invalid").unwrap(); + assert!(resolve_key(&storage, Some("environment-secret")).is_ok()); + } + + #[test] + fn login_validates_before_saving_and_redacts_failures() { + for status in [200, 201, 204, 301, 302, 307, 401, 403, 429, 500, 503] { + let directory = tempfile::tempdir().unwrap(); + let storage = CredentialStorage::Filesystem(directory.path().into()); + save( + &storage, + &Credentials { + api_key: "original".into(), + }, + ) + .unwrap(); + let body = "new-secret upstream details"; + let response = format!( + "HTTP/1.1 {status} Test\r\nLocation: /redirected\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + let (url, task) = serve(response, Duration::ZERO); + let result = complete_login( + &storage, + &Credentials { + api_key: "new-secret".into(), + }, + true, + &url, + Duration::from_secs(2), + ); + let request = task.join().unwrap().to_ascii_lowercase(); + assert!(request.starts_with("get /v1/models http/1.1\r\n")); + assert!(request.contains("authorization: bearer new-secret\r\n")); + if status == 200 { + let output = result.unwrap(); + assert!(output.starts_with("TypeSafe: authenticated. API key saved.")); + assert!(output.contains("TYPESAFE_API_KEY overrides your saved key")); + assert!(!output.contains("new-secret")); + assert_eq!(load(&storage).unwrap().unwrap().api_key, "new-secret"); + } else { + let error = result.unwrap_err(); + assert!(!error.contains("new-secret")); + assert!(!error.contains("upstream")); + assert!(!error.contains("authenticated")); + assert_eq!(error.contains("rejected"), status == 401 || status == 403); + assert_eq!(load(&storage).unwrap().unwrap().api_key, "original"); + } + } + } + + #[test] + fn successful_authentication_does_not_hide_save_failure() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("not-a-directory"); + std::fs::write(&path, b"untouched").unwrap(); + let storage = CredentialStorage::Filesystem(path.clone()); + let (url, task) = serve( + "HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n".into(), + Duration::ZERO, + ); + let error = complete_login( + &storage, + &Credentials { + api_key: "new-secret".into(), + }, + false, + &url, + Duration::from_secs(2), + ) + .unwrap_err(); + task.join().unwrap(); + assert_eq!(error, "Could not save your TypeSafe key. Try again."); + assert_eq!(std::fs::read(path).unwrap(), b"untouched"); + } + + #[test] + fn network_failure_and_timeout_preserve_previous_key() { + let directory = tempfile::tempdir().unwrap(); + let storage = CredentialStorage::Filesystem(directory.path().into()); + save( + &storage, + &Credentials { + api_key: "original".into(), + }, + ) + .unwrap(); + for (response, delay) in [ + (String::new(), Duration::ZERO), + ( + "HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n".into(), + Duration::from_millis(500), + ), + ] { + let (url, task) = serve(response, delay); + let error = complete_login( + &storage, + &Credentials { + api_key: "new-secret".into(), + }, + false, + &url, + Duration::from_millis(100), + ) + .unwrap_err(); + task.join().unwrap(); + assert_eq!( + error, + "Could not authenticate with TypeSafe. Try again later." + ); + assert_eq!(load(&storage).unwrap().unwrap().api_key, "original"); + } + } + + #[test] + fn stored_key_round_trip_status_and_logout() { + let directory = tempfile::tempdir().unwrap(); + let storage = CredentialStorage::Filesystem(directory.path().into()); + assert!( + execute(TypeSafeAuthCommand::Status, &storage, false) + .unwrap() + .contains("not configured") + ); + let record = Credentials { + api_key: "secret-test-key".into(), + }; + save(&storage, &record).unwrap(); + assert_eq!(load(&storage).unwrap().unwrap().api_key, record.api_key); + let bytes = storage.entry(NAMESPACE, IDENTITY).load().unwrap().unwrap(); + assert_eq!(bytes.as_slice(), br#"{"api_key":"secret-test-key"}"#); + let status = execute(TypeSafeAuthCommand::Status, &storage, false).unwrap(); + assert_eq!(status, "TypeSafe: configured.\n"); + assert!(!status.contains(&record.api_key)); + assert!(!format!("{record:?}").contains(&record.api_key)); + let logout = execute( + TypeSafeAuthCommand::Logout { local_only: false }, + &storage, + true, + ) + .unwrap(); + assert!(logout.contains(KEYS_URL)); + assert!(logout.contains("TYPESAFE_API_KEY remains active")); + assert!(load(&storage).unwrap().is_none()); + let logout = execute( + TypeSafeAuthCommand::Logout { local_only: true }, + &storage, + false, + ) + .unwrap(); + assert!(logout.contains("no saved key")); + assert!(!logout.contains(KEYS_URL)); + } + + #[test] + fn malformed_records_are_redacted_and_environment_takes_precedence() { + let directory = tempfile::tempdir().unwrap(); + let storage = CredentialStorage::Filesystem(directory.path().into()); + for bytes in [ + br#"{"api_key":"secret","extra":true}"#.as_slice(), + br#"{"api_key":""}"#, + br#"{"api_key":"secret with spaces"}"#, + br#"{"api_key":42}"#, + b"secret-not-json", + b"{}", + ] { + storage.entry(NAMESPACE, IDENTITY).save(bytes).unwrap(); + let error = load(&storage).unwrap_err(); + assert!(!error.contains("secret")); + let status = execute(TypeSafeAuthCommand::Status, &storage, true).unwrap(); + assert!(status.contains("configured via TYPESAFE_API_KEY")); + } + storage + .entry(NAMESPACE, IDENTITY) + .save(&vec![b'x'; MAX_RECORD_BYTES + 1]) + .unwrap(); + assert!(load(&storage).unwrap_err().contains("Log in again")); + } + + #[test] + fn invalid_key_does_not_overwrite_existing_credentials() { + let directory = tempfile::tempdir().unwrap(); + let storage = CredentialStorage::Filesystem(directory.path().into()); + save( + &storage, + &Credentials { + api_key: "original".into(), + }, + ) + .unwrap(); + for key in [ + String::new(), + " ".into(), + "secret\n".into(), + "secret\0".into(), + "x".repeat(MAX_RECORD_BYTES), + ] { + assert!(save(&storage, &Credentials { api_key: key }).is_err()); + assert_eq!(load(&storage).unwrap().unwrap().api_key, "original"); + } + assert!( + execute( + TypeSafeAuthCommand::Login, + &CredentialStorage::Memory, + false + ) + .is_err() + ); + } +} diff --git a/src/runtime.rs b/src/runtime.rs index eb3759e..139f267 100644 --- a/src/runtime.rs +++ b/src/runtime.rs @@ -504,6 +504,7 @@ impl std::fmt::Display for LogoutAuthenticationError { } pub struct Runtime { + eval: Option, root: PathBuf, adapter: SelectableAdapter, provider: ProviderKind, @@ -622,6 +623,7 @@ impl Runtime { max_subagent_depth, ); Ok(Arc::new(Self { + eval: None, root, adapter, provider, @@ -749,6 +751,20 @@ impl Runtime { Ok(Arc::new(runtime)) } + /// Enables evaluation only after explicit user opt-in and credential resolution. + pub fn with_eval(runtime: Arc, enabled: bool) -> Result, String> { + let mut runtime = Arc::try_unwrap(runtime) + .map_err(|_| "could not configure evaluation after runtime was shared".to_string())?; + runtime.eval = if crate::tools::EvalTool::available(enabled, &runtime.credential_storage) { + Some(crate::tools::EvalTool::new( + runtime.credential_storage.clone(), + )) + } else { + None + }; + Ok(Arc::new(runtime)) + } + pub fn root(&self) -> &Path { &self.root } @@ -1219,6 +1235,9 @@ impl Runtime { .with(Observed::new(DocsTool::new())) .with(Observed::new(ShellTool::new(self.root.clone()))) .with(Observed::new(EditTool::new(self.root.clone())).with_root(self.root.clone())); + if let Some(eval) = &self.eval { + children.register(Observed::new(eval.clone())); + } if depth < self.max_subagent_depth { children .register(Observed::new(SubagentTool::new(subagents.clone(), depth))) diff --git a/src/tools/eval.rs b/src/tools/eval.rs new file mode 100644 index 0000000..f9bae7d --- /dev/null +++ b/src/tools/eval.rs @@ -0,0 +1,459 @@ +//! Explicit, opt-in evaluation. One invocation is one paid submission. +use crate::{credentials::CredentialStorage, provider::typesafe_auth}; +use agentkit_core::{ToolOutput, ToolResultPart}; +use agentkit_tools_core::{ + Tool, ToolContext, ToolError, ToolName, ToolRequest, ToolResult, ToolSpec, +}; +use async_trait::async_trait; +use serde_json::{Map, Value}; +use std::{sync::Arc, time::Duration}; +use tokio::sync::Semaphore; +const MAX_BYTES: usize = 256 * 1024; +const MAX_RESPONSE: usize = 1024 * 1024; +const TIMEOUT: Duration = Duration::from_secs(30); +const URL: &str = "https://api.typesafe.ai/v1/systemone"; +#[derive(Clone)] +pub struct EvalTool { + storage: CredentialStorage, + // Shared by every clone/Compose registry in this runtime. The only writer + // is invoke: an RAII permit spans submission and response validation. Error, + // timeout and cancellation drop it; no secondary locks or counters exist. + permits: Arc, + spec: ToolSpec, +} +impl EvalTool { + pub(crate) fn available(enabled: bool, storage: &CredentialStorage) -> bool { + enabled && typesafe_auth::resolve_api_key(storage).is_ok_and(|key| key.is_some()) + } + pub(crate) fn new(storage: CredentialStorage) -> Self { + Self { + storage, + permits: Arc::new(Semaphore::new(4)), + spec: ToolSpec::new(ToolName::new("eval"), + "Evaluate supplied state with named independent questions in one TypeSafe Jev request. Noul answers yes/no with a probability; Choice selects a named option; Score rates ordered levels. Sends state and questions to TypeSafe and consumes its quota. Maximum 256 KiB input, 64 questions, 30 seconds, four concurrent evaluations. Never automatically retries or batches; a cancelled or failed evaluation may still consume quota. Results are assessments, not facts. No automatic compaction or filtering.", + input_schema()) + .with_output_schema(output_schema()), + } + } +} +fn failure(message: &str) -> ToolError { + ToolError::ExecutionFailed(message.into()) +} +fn content(value: &Value) -> bool { + value.is_string() || value.is_object() || value.is_array() +} +fn payload(input: Value) -> Result { + let invalid = || { + ToolError::InvalidInput("Use state and 1–64 named Noul, Choice, or Score questions; input must not exceed 256 KiB.".into()) + }; + if serde_json::to_vec(&input).map_err(|_| invalid())?.len() > MAX_BYTES { + return Err(invalid()); + } + let fields = input.as_object().ok_or_else(invalid)?; + if fields.len() != 2 || !content(&input["state"]) { + return Err(invalid()); + } + let questions = input["questions"].as_object().ok_or_else(invalid)?; + if questions.is_empty() || questions.len() > 64 { + return Err(invalid()); + } + for (name, question) in questions { + let q = question.as_object().ok_or_else(invalid)?; + if name.is_empty() + || name.chars().count() > 128 + || !content(&question["instructions"]) + || q.keys() + .any(|key| !["type", "instructions", "criteria"].contains(&key.as_str())) + { + return Err(invalid()); + } + let valid = match question["type"].as_str() { + Some("noul") => q.get("criteria").is_none_or(|v| { + v.as_object().is_some_and(|c| { + c.iter() + .all(|(k, v)| ["true", "false"].contains(&k.as_str()) && content(v)) + }) + }), + Some("choice") => question["criteria"].as_object().is_some_and(|c| { + !c.is_empty() && c.len() <= 255 && c.values().all(|v| v.is_null() || content(v)) + }), + Some("score") => question["criteria"] + .as_array() + .is_some_and(|c| (2..=10).contains(&c.len()) && c.iter().all(content)), + _ => false, + }; + if !valid { + return Err(invalid()); + } + } + Ok(object([ + ("model", Value::from("jev-latest")), + ("state", input["state"].clone()), + ("questions", input["questions"].clone()), + ])) +} +fn probability(value: &Value) -> bool { + value.as_f64().is_some_and(|n| (0.0..=1.0).contains(&n)) +} +fn validate_response(value: Value, body: &Value) -> Result { + let invalid = || failure("The evaluation returned an invalid result. No retry was made."); + if !value["model"] + .as_str() + .is_some_and(|s| !s.is_empty() && s.len() <= 128 && s != "jev-latest") + || value["usage"]["input_tokens"].as_u64().is_none() + || value["usage"]["output_tokens"].as_u64().is_none() + { + return Err(invalid()); + } + let answers = value["answers"].as_object().ok_or_else(invalid)?; + let questions = body["questions"].as_object().ok_or_else(invalid)?; + if answers.len() != questions.len() { + return Err(invalid()); + } + for (name, q) in questions { + let a = answers.get(name).ok_or_else(invalid)?; + if a["type"] != q["type"] { + return Err(invalid()); + } + if q["type"] == "noul" { + if !probability(&a["noul"]) { + return Err(invalid()); + } + continue; + } + if !probability(&a["confidence"]) { + return Err(invalid()); + } + let probabilities = a["probabilities"].as_object().ok_or_else(invalid)?; + if !probabilities.values().all(probability) + || (probabilities + .values() + .filter_map(Value::as_f64) + .sum::() + - 1.0) + .abs() + > 0.001 + { + return Err(invalid()); + } + if q["type"] == "choice" { + let criteria = q["criteria"].as_object().ok_or_else(invalid)?; + if probabilities.len() != criteria.len() + || !criteria.keys().all(|k| probabilities.contains_key(k)) + || !a["choice"] + .as_str() + .is_some_and(|s| criteria.contains_key(s)) + { + return Err(invalid()); + } + } else { + let levels = q["criteria"].as_array().ok_or_else(invalid)?.len(); + let legend = a["legend"].as_object().ok_or_else(invalid)?; + if probabilities.len() != levels + || legend.len() != levels + || !(0..levels).all(|i| { + probabilities.contains_key(&i.to_string()) + && legend.get(&i.to_string()).is_some_and(Value::is_string) + }) + || !a["score"] + .as_f64() + .is_some_and(|s| (0.0..=(levels - 1) as f64).contains(&s)) + { + return Err(invalid()); + } + } + } + Ok(value) +} +fn client() -> Result { + reqwest::Client::builder() + .retry(reqwest::retry::never()) + .redirect(reqwest::redirect::Policy::none()) + .connect_timeout(Duration::from_secs(10)) + .timeout(TIMEOUT) + .build() + .map_err(|_| failure("Evaluation is temporarily unavailable.")) +} +async fn submit( + client: &reqwest::Client, + url: &str, + key: &str, + body: &Value, +) -> Result { + let mut response = client + .post(url) + .bearer_auth(key) + .json(body) + .send() + .await + .map_err(|_| { + failure("Evaluation did not complete. No retry was made; quota may have been used.") + })?; + if !response.status().is_success() { + return Err(failure(match response.status().as_u16() { + 401 | 403 => { + "TypeSafe could not accept your key. Use kit auth login typesafe to update it." + } + 429 => "Your TypeSafe evaluation limit was reached. Try again later.", + _ => "Evaluation did not complete. No retry was made; quota may have been used.", + })); + } + let mut bytes = Vec::new(); + if response + .content_length() + .is_some_and(|n| n > MAX_RESPONSE as u64) + { + return Err(failure("Evaluation result exceeds 1 MiB.")); + } + while let Some(chunk) = response + .chunk() + .await + .map_err(|_| failure("Evaluation result was interrupted. No retry was made."))? + { + if bytes.len() + chunk.len() > MAX_RESPONSE { + return Err(failure("Evaluation result exceeds 1 MiB.")); + } + bytes.extend_from_slice(&chunk); + } + let value = serde_json::from_slice(&bytes) + .map_err(|_| failure("The evaluation returned an invalid result. No retry was made."))?; + validate_response(value, body) +} +#[async_trait] +impl Tool for EvalTool { + fn spec(&self) -> &ToolSpec { + &self.spec + } + async fn invoke( + &self, + request: ToolRequest, + context: &mut ToolContext<'_>, + ) -> Result { + let work = async { + let body = payload(request.input)?; + let _permit = self + .permits + .acquire() + .await + .map_err(|_| failure("Evaluation is unavailable."))?; + let storage = self.storage.clone(); + let credentials = + tokio::task::spawn_blocking(move || typesafe_auth::resolve_api_key(&storage)) + .await + .map_err(|_| failure("TypeSafe key could not be loaded."))? + .map_err(|_| failure("TypeSafe key could not be loaded."))? + .ok_or_else(|| failure("Use kit auth login typesafe before evaluating."))?; + let value = submit(&client()?, URL, credentials.api_key(), &body).await?; + Ok(ToolResult::new(ToolResultPart::success( + request.call_id, + ToolOutput::structured(value), + ))) + }; + let cancelled = std::pin::pin!(async { + match &context.cancellation { + Some(c) => c.cancelled().await, + None => std::future::pending().await, + } + }); + let work = std::pin::pin!(tokio::time::timeout(TIMEOUT, work)); + match futures_util::future::select(cancelled, work).await { + futures_util::future::Either::Left(((), _)) => Err(failure( + "Evaluation cancelled. Quota may have been used; no retry was made.", + )), + futures_util::future::Either::Right((result, _)) => result.unwrap_or_else(|_| { + Err(failure( + "Evaluation timed out. Quota may have been used; no retry was made.", + )) + }), + } + } +} + +#[cfg(test)] +#[allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::disallowed_methods, + clippy::disallowed_macros +)] +mod tests; + +fn object(fields: [(&str, Value); N]) -> Value { + Value::Object(Map::from_iter( + fields + .into_iter() + .map(|(key, value)| (key.to_owned(), value)), + )) +} +fn strings(values: &[&str]) -> Value { + Value::Array(values.iter().map(|s| Value::from(*s)).collect()) +} +fn typed(kind: &str) -> Value { + object([("type", Value::from(kind))]) +} +fn record(properties: Value, required: &[&str]) -> Value { + object([ + ("type", Value::from("object")), + ("properties", properties), + ("required", strings(required)), + ("additionalProperties", Value::Bool(false)), + ]) +} +fn content_schema() -> Value { + object([("type", strings(&["string", "object", "array"]))]) +} +fn question_schema(kind: &str, criteria: Value, required: bool) -> Value { + record( + object([ + ("type", object([("const", Value::from(kind))])), + ("instructions", content_schema()), + ("criteria", criteria), + ]), + if required { + &["type", "instructions", "criteria"] + } else { + &["type", "instructions"] + }, + ) +} +fn input_schema() -> Value { + let questions = object([( + "oneOf", + Value::Array(vec![ + question_schema( + "noul", + record( + object([("true", content_schema()), ("false", content_schema())]), + &[], + ), + false, + ), + question_schema( + "choice", + object([ + ("type", Value::from("object")), + ("minProperties", Value::from(1)), + ("maxProperties", Value::from(255)), + ( + "additionalProperties", + object([("type", strings(&["string", "object", "array", "null"]))]), + ), + ]), + true, + ), + question_schema( + "score", + object([ + ("type", Value::from("array")), + ("minItems", Value::from(2)), + ("maxItems", Value::from(10)), + ("items", content_schema()), + ]), + true, + ), + ]), + )]); + record( + object([ + ("state", content_schema()), + ( + "questions", + object([ + ("type", Value::from("object")), + ("minProperties", Value::from(1)), + ("maxProperties", Value::from(64)), + ( + "propertyNames", + object([ + ("minLength", Value::from(1)), + ("maxLength", Value::from(128)), + ]), + ), + ("additionalProperties", questions), + ]), + ), + ]), + &["state", "questions"], + ) +} +fn output_schema() -> Value { + let probability = object([ + ("type", Value::from("number")), + ("minimum", Value::from(0)), + ("maximum", Value::from(1)), + ]); + let probabilities = object([ + ("type", Value::from("object")), + ("additionalProperties", probability.clone()), + ]); + let answers = object([( + "oneOf", + Value::Array(vec![ + record( + object([ + ("type", object([("const", Value::from("noul"))])), + ("noul", probability.clone()), + ]), + &["type", "noul"], + ), + record( + object([ + ("type", object([("const", Value::from("choice"))])), + ("choice", typed("string")), + ("probabilities", probabilities.clone()), + ("confidence", probability.clone()), + ]), + &["type", "choice", "probabilities", "confidence"], + ), + record( + object([ + ("type", object([("const", Value::from("score"))])), + ("score", typed("number")), + ("probabilities", probabilities), + ("confidence", probability), + ( + "legend", + object([ + ("type", Value::from("object")), + ("additionalProperties", typed("string")), + ]), + ), + ]), + &["type", "score", "probabilities", "confidence", "legend"], + ), + ]), + )]); + // Preserve forward-compatible response metadata rather than filtering it. + object([ + ("type", Value::from("object")), + ( + "properties", + object([ + ("model", typed("string")), + ( + "answers", + object([ + ("type", Value::from("object")), + ("additionalProperties", answers), + ]), + ), + ( + "usage", + object([ + ("type", Value::from("object")), + ( + "properties", + object([ + ("input_tokens", typed("integer")), + ("output_tokens", typed("integer")), + ]), + ), + ("required", strings(&["input_tokens", "output_tokens"])), + ]), + ), + ]), + ), + ("required", strings(&["model", "answers", "usage"])), + ]) +} diff --git a/src/tools/eval/tests.rs b/src/tools/eval/tests.rs new file mode 100644 index 0000000..f03c146 --- /dev/null +++ b/src/tools/eval/tests.rs @@ -0,0 +1,236 @@ +use super::*; +use serde_json::json; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +fn input() -> Value { + json!({"state":{"ticket":"Please help"},"questions":{ + "urgent":{"type":"noul","instructions":"Urgent?","criteria":{"true":"Now"}}, + "route":{"type":"choice","instructions":["Route"],"criteria":{"billing":null,"support":"Help"}}, + "severity":{"type":"score","instructions":{"question":"How severe?"},"criteria":["Low","High"]} + }}) +} +fn response() -> Value { + json!({"model":"jev-1.13.0","answers":{ + "urgent":{"type":"noul","noul":0.7}, + "route":{"type":"choice","choice":"support","probabilities":{"billing":0.2,"support":0.8},"confidence":0.6}, + "severity":{"type":"score","score":0.8,"legend":{"0":"Low","1":"High"},"probabilities":{"0":0.2,"1":0.8},"confidence":0.6} + },"usage":{"input_tokens":30,"output_tokens":20}}) +} +#[test] +fn validates_three_question_types_and_bounds() { + let tool = EvalTool::new(CredentialStorage::default()); + let schema = jsonschema::validator_for(&tool.spec().input_schema).unwrap(); + assert!(schema.is_valid(&input())); + let body = payload(input()).unwrap(); + assert_eq!(body["model"], "jev-latest"); + assert_eq!(validate_response(response(), &body).unwrap(), response()); + for bad in [ + json!(null), + json!({"state":1,"questions":{}}), + json!({"state":"x","questions":{"a":{"type":"json","instructions":"x"}}}), + ] { + assert!(payload(bad).is_err()); + } + let mut large = input(); + large["state"] = Value::String("x".repeat(MAX_BYTES)); + assert!(payload(large).is_err()); + for q in [ + json!({"type":"score","instructions":"x","criteria":["one"]}), + json!({"type":"noul","instructions":"x","criteria":{"maybe":"x"}}), + json!({"type":"choice","instructions":"x","criteria":{}}), + ] { + assert!(payload(json!({"state":"x","questions":{"q":q}})).is_err()); + } +} +#[test] +fn question_name_bounds_count_unicode_characters() { + for (length, accepted) in [(0, false), (65, true), (128, true), (129, false)] { + let name = "é".repeat(length); + let value = json!({ + "state": "x", + "questions": {name: {"type": "noul", "instructions": "Urgent?"}} + }); + assert_eq!(payload(value).is_ok(), accepted, "{length} characters"); + } +} + +#[test] +fn rejects_malformed_answers() { + let body = payload(input()).unwrap(); + for (pointer, bad) in [ + ("/model", json!("jev-latest")), + ("/usage/input_tokens", json!(-1)), + ("/answers/urgent/noul", json!(1.1)), + ("/answers/route/choice", json!("unknown")), + ("/answers/route/confidence", json!(null)), + ("/answers/route/probabilities/support", json!(0.1)), + ("/answers/severity/score", json!(2)), + ("/answers/severity/legend", json!({})), + ("/answers/urgent/type", json!("choice")), + ("/answers", json!({})), + ] { + let mut value = response(); + *value.pointer_mut(pointer).unwrap() = bad; + assert!(validate_response(value, &body).is_err(), "{pointer}"); + } +} + +// An actual local HTTP peer: tests never read a real key or contact TypeSafe. +async fn peer(status: u16, body: String) -> (String, tokio::task::JoinHandle) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let task = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut bytes = Vec::new(); + let (offset, length) = loop { + let mut buffer = [0; 4096]; + let n = socket.read(&mut buffer).await.unwrap(); + assert!(n > 0); + bytes.extend_from_slice(&buffer[..n]); + if let Some(offset) = bytes.windows(4).position(|w| w == b"\r\n\r\n") { + let headers = String::from_utf8_lossy(&bytes[..offset]).to_lowercase(); + assert!(headers.starts_with("post /v1/systemone ")); + assert!(headers.contains("authorization: bearer fake-key")); + let length: usize = headers + .lines() + .find_map(|line| line.strip_prefix("content-length: ")) + .unwrap() + .parse() + .unwrap(); + break (offset + 4, length); + } + }; + while bytes.len() < offset + length { + let mut buffer = [0; 4096]; + let n = socket.read(&mut buffer).await.unwrap(); + assert!(n > 0); + bytes.extend_from_slice(&buffer[..n]); + } + let request = serde_json::from_slice(&bytes[offset..offset + length]).unwrap(); + let reply = format!( + "HTTP/1.1 {status} Result\r\nContent-Length: {}\r\nContent-Type: application/json\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + socket.write_all(reply.as_bytes()).await.unwrap(); + request + }); + (format!("http://{address}/v1/systemone"), task) +} +#[tokio::test] +async fn sends_all_questions_once_and_preserves_results() { + let expected = response(); + let (url, peer) = peer(200, expected.to_string()).await; + let body = payload(input()).unwrap(); + assert_eq!( + submit(&client().unwrap(), &url, "fake-key", &body) + .await + .unwrap(), + expected + ); + assert_eq!(peer.await.unwrap(), body); +} +#[tokio::test] +async fn safe_errors_for_rejected_keys_limits_and_malformed_results() { + for (status, body, message) in [ + (401, "secret state", "could not accept your key"), + (403, "secret state", "could not accept your key"), + (429, "secret state", "limit was reached"), + (500, "secret state", "did not complete"), + (200, "secret state", "invalid result"), + ] { + let (url, peer) = peer(status, body.into()).await; + let error = submit( + &client().unwrap(), + &url, + "fake-key", + &payload(input()).unwrap(), + ) + .await + .unwrap_err() + .to_string(); + assert!(error.contains(message)); + assert!(!error.contains("secret state")); + peer.await.unwrap(); + } +} +#[test] +fn disabled_gate_never_needs_credentials() { + assert!(!EvalTool::available(false, &CredentialStorage::default())); +} + +#[test] +fn credential_gate_uses_only_explicit_sources() { + const MARKER: &str = "KIT_TEST_EVAL_GATE"; + if let Ok(case) = std::env::var(MARKER) { + let directory = tempfile::tempdir().unwrap(); + let storage = CredentialStorage::Filesystem(directory.path().to_path_buf()); + if case == "stored" { + storage + .entry("typesafe", "default") + .save(br#"{"api_key":"fake-stored-key"}"#) + .unwrap(); + } + assert!(!EvalTool::available(false, &storage)); + assert_eq!( + EvalTool::available(true, &storage), + case == "stored" || case == "environment" + ); + return; + } + for case in ["missing", "stored", "environment", "invalid"] { + let mut command = std::process::Command::new(std::env::current_exe().unwrap()); + command + .args([ + "--exact", + "tools::eval::tests::credential_gate_uses_only_explicit_sources", + ]) + .env(MARKER, case) + .env_remove("TYPESAFE_API_KEY"); + if case == "environment" { + command.env("TYPESAFE_API_KEY", "fake-environment-key"); + } + if case == "invalid" { + command.env("TYPESAFE_API_KEY", "invalid key"); + } + let output = command.output().unwrap(); + assert!( + output.status.success(), + "{case}: {}", + String::from_utf8_lossy(&output.stderr) + ); + } +} + +#[tokio::test] +async fn cancellation_while_waiting_does_not_consume_permits() { + use agentkit_core::{CancellationController, MetadataMap, SessionId, TurnId}; + use agentkit_tools_core::{AllowAllPermissions, OwnedToolContext}; + let tool = EvalTool::new(CredentialStorage::default()); + let held = tool.permits.acquire_many(4).await.unwrap(); + let controller = CancellationController::new(); + let context = OwnedToolContext { + session_id: SessionId::new("eval-test"), + turn_id: TurnId::new("turn"), + metadata: MetadataMap::new(), + permissions: Arc::new(AllowAllPermissions), + resources: Arc::new(()), + cancellation: Some(controller.handle().checkpoint()), + execution_scope: None, + approved_request: None, + }; + let request = ToolRequest::new("call", "eval", input(), "eval-test", "turn"); + let mut borrowed = context.borrowed(); + let mut invocation = Box::pin(tool.invoke(request, &mut borrowed)); + // Poll the real API until it waits for an admission permit, then cancel. + assert!(futures_util::poll!(invocation.as_mut()).is_pending()); + controller.interrupt(); + assert!( + invocation + .await + .unwrap_err() + .to_string() + .contains("cancelled") + ); + drop(held); + assert!(tool.permits.try_acquire_many(4).is_ok()); +} diff --git a/src/tools/mod.rs b/src/tools/mod.rs index 0246fc1..c37ed81 100644 --- a/src/tools/mod.rs +++ b/src/tools/mod.rs @@ -24,3 +24,6 @@ pub use subagent::{ mod image_gen; pub use image_gen::ImageGenTool; + +mod eval; +pub(crate) use eval::EvalTool; diff --git a/tests/cli.rs b/tests/cli.rs index 15165e7..79fd006 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -12,6 +12,33 @@ use std::{fs, path::Path, process::Command}; use agentkit_core::{Item, ItemKind}; +#[test] +fn typesafe_auth_environment_status_and_logout() { + let home = tempfile::tempdir().unwrap(); + for (action, key, expected) in [ + ( + "status", + "test-secret", + "TypeSafe: configured via TYPESAFE_API_KEY.", + ), + ("status", "", "TypeSafe: not configured."), + ("logout", "test-secret", "TYPESAFE_API_KEY remains active"), + ("logout", "", "no saved key"), + ] { + let output = Command::new(env!("CARGO_BIN_EXE_kit")) + .env("HOME", home.path()) + .env("TYPESAFE_API_KEY", key) + .args(["auth", action, "typesafe", "--credential-store", "memory"]) + .output() + .unwrap(); + assert!(output.status.success(), "{output:?}"); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(stdout.contains(expected), "{stdout}"); + assert!(!stdout.contains("test-secret")); + assert!(!String::from_utf8_lossy(&output.stderr).contains("test-secret")); + } +} + fn write_session(home: &Path, root: &Path, id: &str) -> std::path::PathBuf { let root = root.canonicalize().unwrap(); let identity = blake3::hash(root.as_os_str().as_encoded_bytes());