From 7e3234799f73c8911395b89a4e81adbab1ae4b7f Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Thu, 3 Sep 2026 19:13:45 +0300 Subject: [PATCH 1/2] feat(cli): report distinct exit codes for benchmark, auth and upload failures --- src/cli/auth.rs | 10 +-- src/cli/exec/mod.rs | 1 + src/cli/mod.rs | 7 +- src/cli/run/mod.rs | 1 + src/executor/memory/executor.rs | 5 +- src/executor/orchestrator.rs | 11 ++- src/executor/valgrind/measure.rs | 5 +- src/executor/wall_time/executor.rs | 5 +- src/exit_code.rs | 103 ++++++++++++++++++++++++++ src/lib.rs | 1 + src/main.rs | 5 +- src/run_environment/local/provider.rs | 7 +- src/upload/uploader.rs | 13 +++- 13 files changed, 154 insertions(+), 20 deletions(-) create mode 100644 src/exit_code.rs diff --git a/src/cli/auth.rs b/src/cli/auth.rs index 07905dc7a..6f4c6a328 100644 --- a/src/cli/auth.rs +++ b/src/cli/auth.rs @@ -1,6 +1,7 @@ use std::io::Read; use std::time::Duration; +use super::status::{check_mark, cross_mark}; use crate::api_client::{ Authentication, CodSpeedAPIClient, RepositoryOverviewPayload, SessionAndRepositoryOverviewError, SessionAndRepositoryOverviewVars, SessionError, @@ -10,14 +11,13 @@ use crate::cli::run::helpers::{ ParsedRepository, find_repository_root, parse_repository_from_remote, }; use crate::config::CodSpeedConfig; +use crate::exit_code::auth_failed; use crate::prelude::*; use clap::{Args, Subcommand}; use console::style; use git2::Repository; use tokio::time::{Instant, sleep}; -use super::status::{check_mark, cross_mark}; - #[derive(Debug, Args)] pub struct AuthArgs { #[command(subcommand)] @@ -118,9 +118,9 @@ async fn login( .session() .await .map_err(|err| match err { - SessionError::Unauthenticated => { - anyhow!("Invalid token. The token is either malformed or has expired.") - } + SessionError::Unauthenticated => auth_failed(anyhow!( + "Invalid token. The token is either malformed or has expired." + )), SessionError::Other(err) => err, })?; diff --git a/src/cli/exec/mod.rs b/src/cli/exec/mod.rs index 4a757f74b..8396aa3cb 100644 --- a/src/cli/exec/mod.rs +++ b/src/cli/exec/mod.rs @@ -18,6 +18,7 @@ pub mod multi_targets; pub const DEFAULT_REPOSITORY_NAME: &str = "local-runs"; #[derive(Args, Debug)] +#[command(after_long_help = crate::exit_code::help_text())] pub struct ExecArgs { #[command(flatten)] pub shared: ExecAndRunSharedArgs, diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 2a9218ddc..169c78fea 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -39,7 +39,12 @@ fn create_styles() -> Styles { } #[derive(Parser, Debug)] -#[command(version, about = "The CodSpeed CLI tool", styles = create_styles())] +#[command( + version, + about = "The CodSpeed CLI tool", + styles = create_styles(), + after_long_help = crate::exit_code::help_text(), +)] pub struct Cli { /// The URL of the CodSpeed GraphQL API #[arg(long, env = "CODSPEED_API_URL", global = true, hide = true)] diff --git a/src/cli/run/mod.rs b/src/cli/run/mod.rs index a218155c7..ec5b2c832 100644 --- a/src/cli/run/mod.rs +++ b/src/cli/run/mod.rs @@ -16,6 +16,7 @@ pub mod helpers; pub mod logger; #[derive(Args, Debug)] +#[command(after_long_help = crate::exit_code::help_text())] pub struct RunArgs { #[command(flatten)] pub shared: ExecAndRunSharedArgs, diff --git a/src/executor/memory/executor.rs b/src/executor/memory/executor.rs index b8c9a3985..98cddb4be 100644 --- a/src/executor/memory/executor.rs +++ b/src/executor/memory/executor.rs @@ -11,6 +11,7 @@ use crate::executor::helpers::run_with_sudo::is_root_user; use crate::executor::memory::tunables::MemoryTunables; use crate::executor::shared::fifo::RunnerFifo; use crate::executor::{ExecutionContext, Executor}; +use crate::exit_code::benchmark_failed; use crate::instruments::mongo_tracer::MongoTracer; use crate::prelude::*; use crate::runner_mode::RunnerMode; @@ -188,7 +189,9 @@ impl Executor for MemoryExecutor { debug!("cmd exit status: {status:?}"); if !status.success() { - bail!("failed to execute memory tracker process: {status}"); + return Err(benchmark_failed(anyhow!( + "failed to execute memory tracker process: {status}" + ))); } Ok(()) diff --git a/src/executor/orchestrator.rs b/src/executor/orchestrator.rs index ca2dbdf4f..5d8055d7a 100644 --- a/src/executor/orchestrator.rs +++ b/src/executor/orchestrator.rs @@ -7,6 +7,7 @@ use crate::cli::run::logger::Logger; use crate::executor::config::BenchmarkTarget; use crate::executor::config::OrchestratorConfig; use crate::executor::helpers::profile_folder::create_profile_folder; +use crate::exit_code::{auth_failed, upload_failed}; use crate::prelude::*; use crate::run_environment::{self, RunEnvironment, RunEnvironmentProvider}; use crate::runner_mode::RunnerMode; @@ -223,7 +224,10 @@ impl Orchestrator { if !skip_upload { start_group!("Uploading results"); - let last_upload_result = self.upload_all(&mut completed_runs, api_client).await?; + let last_upload_result = self + .upload_all(&mut completed_runs, api_client) + .await + .map_err(upload_failed)?; end_group!(); if self.is_local() { @@ -267,7 +271,10 @@ impl Orchestrator { let total_runs = completed_runs.len(); for (run_part_index, (ctx, executor_name)) in completed_runs.iter_mut().enumerate() { // OIDC tokens can expire quickly, so refresh just before each upload - self.provider.set_oidc_token(api_client).await?; + self.provider + .set_oidc_token(api_client) + .await + .map_err(auth_failed)?; if run_part_index == 0 { // After the mint, so this names the token the upload actually uses diff --git a/src/executor/valgrind/measure.rs b/src/executor/valgrind/measure.rs index 62807b925..df81ccaa2 100644 --- a/src/executor/valgrind/measure.rs +++ b/src/executor/valgrind/measure.rs @@ -6,6 +6,7 @@ use crate::executor::helpers::get_bench_command::get_bench_command; use crate::executor::helpers::run_command_with_log_pipe::run_command_with_log_pipe; use crate::executor::valgrind::helpers::ignored_objects_path::get_objects_path_to_ignore; use crate::executor::valgrind::helpers::python::is_free_threaded_python; +use crate::exit_code::benchmark_failed; use crate::instruments::mongo_tracer::MongoTracer; use crate::prelude::*; use log::log_enabled; @@ -220,7 +221,9 @@ pub async fn measure( }; debug!("Program exit code = {cmd_status}"); if cmd_status != 0 { - bail!("failed to execute the benchmark process, exit code: {cmd_status}"); + return Err(benchmark_failed(anyhow!( + "failed to execute the benchmark process, exit code: {cmd_status}" + ))); } Ok(()) diff --git a/src/executor/wall_time/executor.rs b/src/executor/wall_time/executor.rs index a419f7bc3..72d3d675d 100644 --- a/src/executor/wall_time/executor.rs +++ b/src/executor/wall_time/executor.rs @@ -18,6 +18,7 @@ use crate::executor::helpers::run_with_sudo::wrap_with_sudo; use crate::executor::shared::fifo::FifoBenchmarkData; use crate::executor::shared::fifo::RunnerFifo; use crate::executor::{ExecutionContext, ExecutorName, ExecutorSupport}; +use crate::exit_code::benchmark_failed; use crate::instruments::mongo_tracer::MongoTracer; use crate::prelude::*; use crate::runner_mode::RunnerMode; @@ -177,7 +178,9 @@ impl Executor for WallTimeExecutor { debug!("cmd exit status: {status:?}"); if !status.success() { - bail!("failed to execute the benchmark process: {status}"); + return Err(benchmark_failed(anyhow!( + "failed to execute the benchmark process: {status}" + ))); } Ok(()) diff --git a/src/exit_code.rs b/src/exit_code.rs new file mode 100644 index 000000000..4e56bdc7b --- /dev/null +++ b/src/exit_code.rs @@ -0,0 +1,103 @@ +use std::fmt; + +use crate::prelude::*; + +/// Generic failure. Something not covered by other codes. +pub const FAILURE: i32 = 1; + +/// Invalid command-line usage (from Clap). +pub const USAGE: i32 = 2; + +/// Benchmark command exited with a non-zero status. +pub const BENCHMARK_FAILED: i32 = 3; + +/// Could not authenticate against CodSpeed. +/// Retrying without fixing the credentials will not help. +pub const AUTH_FAILED: i32 = 4; + +/// Benchmarks ran successfully, but their results could not be uploaded to CodSpeed +/// (e.g. network error, server error, etc.). +pub const UPLOAD_FAILED: i32 = 5; + +/// An error tagged with the exit code to report for it. +#[derive(Debug)] +pub struct Marked { + code: i32, + error: Error, +} + +impl Marked { + fn wrap(code: i32, error: Error) -> Error { + let code = code_of(&error).unwrap_or(code); + Error::new(Marked { code, error }) + } +} + +impl fmt::Display for Marked { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.error, f) + } +} + +impl std::error::Error for Marked { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + self.error.source() + } +} + +pub fn benchmark_failed(error: Error) -> Error { + Marked::wrap(BENCHMARK_FAILED, error) +} + +pub fn auth_failed(error: Error) -> Error { + Marked::wrap(AUTH_FAILED, error) +} + +pub fn upload_failed(error: Error) -> Error { + Marked::wrap(UPLOAD_FAILED, error) +} + +pub fn help_text() -> String { + format!( + "Exit codes:\n \ + 0 Success\n \ + {FAILURE} Failure\n \ + {USAGE} Invalid command-line usage\n \ + {BENCHMARK_FAILED} The benchmark command itself failed\n \ + {AUTH_FAILED} Authentication or authorization failed\n \ + {UPLOAD_FAILED} The benchmarks ran, but their results could not be uploaded" + ) +} + +fn code_of(error: &Error) -> Option { + error + .chain() + .find_map(|cause| cause.downcast_ref::()) + .map(|marked| marked.code) +} + +pub fn exit_code_for(error: &Error) -> i32 { + code_of(error).unwrap_or(FAILURE) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_exit_codes() { + assert_eq!(exit_code_for(&anyhow!("something went wrong")), FAILURE); + assert_eq!( + exit_code_for(&benchmark_failed(anyhow!("exit status: 101"))), + BENCHMARK_FAILED + ); + assert_eq!( + exit_code_for(&auth_failed(anyhow!("Invalid token"))), + AUTH_FAILED + ); + assert_eq!( + exit_code_for(&upload_failed(anyhow!("connection reset")).context("Uploading results")), + UPLOAD_FAILED + ); + } +} diff --git a/src/lib.rs b/src/lib.rs index fe86e3e32..284a41d64 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,6 +6,7 @@ mod binary_pins; pub mod cli; mod config; mod executor; +pub mod exit_code; mod instruments; mod local_logger; pub mod logger; diff --git a/src/main.rs b/src/main.rs index 518b8d3ef..2ca1b53f7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,4 @@ -use codspeed_runner::{clean_logger, cli}; +use codspeed_runner::{clean_logger, cli, exit_code}; use console::style; use log::log_enabled; @@ -6,6 +6,7 @@ use log::log_enabled; async fn main() { let res = cli::run().await; if let Err(err) = res { + let code = exit_code::exit_code_for(&err); // Show the primary error let mut chain = err.chain(); if let Some(primary) = chain.next() { @@ -22,6 +23,6 @@ async fn main() { } } clean_logger(); - std::process::exit(1); + std::process::exit(code); } } diff --git a/src/run_environment/local/provider.rs b/src/run_environment/local/provider.rs index ab67f69b5..f48bbe80a 100644 --- a/src/run_environment/local/provider.rs +++ b/src/run_environment/local/provider.rs @@ -11,6 +11,7 @@ use crate::api_client::{ use crate::cli::run::helpers::{find_repository_root, parse_repository_from_remote}; use crate::executor::config::OrchestratorConfig; use crate::executor::config::RepositoryOverride; +use crate::exit_code::auth_failed; use crate::local_logger::get_local_logger; use crate::prelude::*; use crate::run_environment::interfaces::{ @@ -271,9 +272,9 @@ impl LocalProvider { }) .await .map_err(|err| match err { - SessionAndRepositoryOverviewError::Unauthenticated => { - anyhow!("Invalid token. Run `codspeed auth login` to re-authenticate.") - } + SessionAndRepositoryOverviewError::Unauthenticated => auth_failed(anyhow!( + "Invalid token. Run `codspeed auth login` to re-authenticate." + )), SessionAndRepositoryOverviewError::Other(err) => err, })?; diff --git a/src/upload/uploader.rs b/src/upload/uploader.rs index c3a7139b6..415447fa1 100644 --- a/src/upload/uploader.rs +++ b/src/upload/uploader.rs @@ -1,7 +1,10 @@ +use super::interfaces::{UploadData, UploadMetadata}; +use super::profile_archive::ProfileArchive; use crate::api_client::CodSpeedAPIClient; use crate::executor::ExecutionContext; use crate::executor::ExecutorName; use crate::executor::Orchestrator; +use crate::exit_code::auth_failed; use crate::run_environment::RunEnvironment; use crate::upload::{UploadError, profile_archive::ProfileArchiveContent}; use crate::{ @@ -21,9 +24,6 @@ use tokio::fs::File; use tokio::io::AsyncWriteExt; use tokio_tar::Builder; -use super::interfaces::{UploadData, UploadMetadata}; -use super::profile_archive::ProfileArchive; - fn bytes_to_mib(bytes: u64) -> u64 { bytes / (1024 * 1024) } @@ -175,13 +175,18 @@ async fn retrieve_upload_data( upload_metadata.run_environment_metadata.repository ); - bail!( + let error = anyhow!( "Failed to retrieve upload data: {}\n -> {} {}", status, style("Reason:").bold(), // we have to manually apply the style to the error message, because nesting styles is not supported by the console crate: https://github.com/console-rs/console/issues/106 style(error_message).red() ); + return Err(if status == StatusCode::UNAUTHORIZED { + auth_failed(error) + } else { + error + }); } Ok(response.json().await?) From b539880ac353e923022c77bdac3dcb039be53de6 Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Thu, 3 Sep 2026 20:14:43 +0300 Subject: [PATCH 2/2] Adjust based on bot review --- src/exit_code.rs | 7 +++++-- src/run_environment/mod.rs | 5 ++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/exit_code.rs b/src/exit_code.rs index 4e56bdc7b..b877d5e0d 100644 --- a/src/exit_code.rs +++ b/src/exit_code.rs @@ -8,7 +8,10 @@ pub const FAILURE: i32 = 1; /// Invalid command-line usage (from Clap). pub const USAGE: i32 = 2; -/// Benchmark command exited with a non-zero status. +/// The benchmark process (the workload+tool) exited with a non-zero status. +/// Only simulation mode separates the workload from the tool, +/// because the wrapper script records the program's own status; +/// elsewhere the tool's status is the only one known. pub const BENCHMARK_FAILED: i32 = 3; /// Could not authenticate against CodSpeed. @@ -63,7 +66,7 @@ pub fn help_text() -> String { 0 Success\n \ {FAILURE} Failure\n \ {USAGE} Invalid command-line usage\n \ - {BENCHMARK_FAILED} The benchmark command itself failed\n \ + {BENCHMARK_FAILED} The benchmark process failed\n \ {AUTH_FAILED} Authentication or authorization failed\n \ {UPLOAD_FAILED} The benchmarks ran, but their results could not be uploaded" ) diff --git a/src/run_environment/mod.rs b/src/run_environment/mod.rs index 77ba7faa0..4ad658002 100644 --- a/src/run_environment/mod.rs +++ b/src/run_environment/mod.rs @@ -11,6 +11,7 @@ use provider::RunEnvironmentDetector; use crate::api_client::CodSpeedAPIClient; use crate::executor::config::OrchestratorConfig; +use crate::exit_code; use crate::prelude::*; pub use self::interfaces::*; @@ -49,7 +50,9 @@ pub async fn get_provider( } }; - provider.check_oidc_configuration(api_client)?; + provider + .check_oidc_configuration(api_client) + .map_err(exit_code::auth_failed)?; Ok(provider) }