feat(cli): report distinct exit codes for benchmark, auth and upload failures

src/cli/auth.rs

@@ -1,6 +1,7 @@
 use std::io::Read;
 use std::time::Duration;
 
+use super::status::{check_mark, cross_mark};
 use crate::api_client::{
     Authentication, CodSpeedAPIClient, RepositoryOverviewPayload,
     SessionAndRepositoryOverviewError, SessionAndRepositoryOverviewVars, SessionError,
@@ -10,14 +11,13 @@ use crate::cli::run::helpers::{
     ParsedRepository, find_repository_root, parse_repository_from_remote,
 };
 use crate::config::CodSpeedConfig;
+use crate::exit_code::auth_failed;
 use crate::prelude::*;
 use clap::{Args, Subcommand};
 use console::style;
 use git2::Repository;
 use tokio::time::{Instant, sleep};
 
-use super::status::{check_mark, cross_mark};
-
 #[derive(Debug, Args)]
 pub struct AuthArgs {
     #[command(subcommand)]
@@ -118,9 +118,9 @@ async fn login(
         .session()
         .await
         .map_err(|err| match err {
-            SessionError::Unauthenticated => {
-                anyhow!("Invalid token. The token is either malformed or has expired.")
-            }
+            SessionError::Unauthenticated => auth_failed(anyhow!(
+                "Invalid token. The token is either malformed or has expired."
+            )),
             SessionError::Other(err) => err,
         })?;
 

src/cli/exec/mod.rs

@@ -18,6 +18,7 @@ pub mod multi_targets;
 pub const DEFAULT_REPOSITORY_NAME: &str = "local-runs";
 
 #[derive(Args, Debug)]
+#[command(after_long_help = crate::exit_code::help_text())]
 pub struct ExecArgs {
     #[command(flatten)]
     pub shared: ExecAndRunSharedArgs,

src/cli/mod.rs

@@ -39,7 +39,12 @@ fn create_styles() -> Styles {
 }
 
 #[derive(Parser, Debug)]
-#[command(version, about = "The CodSpeed CLI tool", styles = create_styles())]
+#[command(
+    version,
+    about = "The CodSpeed CLI tool",
+    styles = create_styles(),
+    after_long_help = crate::exit_code::help_text(),
+)]
 pub struct Cli {
     /// The URL of the CodSpeed GraphQL API
     #[arg(long, env = "CODSPEED_API_URL", global = true, hide = true)]

src/cli/run/mod.rs

@@ -16,6 +16,7 @@ pub mod helpers;
 pub mod logger;
 
 #[derive(Args, Debug)]
+#[command(after_long_help = crate::exit_code::help_text())]
 pub struct RunArgs {
     #[command(flatten)]
     pub shared: ExecAndRunSharedArgs,

src/executor/memory/executor.rs

@@ -11,6 +11,7 @@ use crate::executor::helpers::run_with_sudo::is_root_user;
 use crate::executor::memory::tunables::MemoryTunables;
 use crate::executor::shared::fifo::RunnerFifo;
 use crate::executor::{ExecutionContext, Executor};
+use crate::exit_code::benchmark_failed;
 use crate::instruments::mongo_tracer::MongoTracer;
 use crate::prelude::*;
 use crate::runner_mode::RunnerMode;
@@ -188,7 +189,9 @@ impl Executor for MemoryExecutor {
         debug!("cmd exit status: {status:?}");
 
         if !status.success() {
-            bail!("failed to execute memory tracker process: {status}");
+            return Err(benchmark_failed(anyhow!(
+                "failed to execute memory tracker process: {status}"
+            )));
         }
 
         Ok(())

src/executor/orchestrator.rs

@@ -7,6 +7,7 @@ use crate::cli::run::logger::Logger;
 use crate::executor::config::BenchmarkTarget;
 use crate::executor::config::OrchestratorConfig;
 use crate::executor::helpers::profile_folder::create_profile_folder;
+use crate::exit_code::{auth_failed, upload_failed};
 use crate::prelude::*;
 use crate::run_environment::{self, RunEnvironment, RunEnvironmentProvider};
 use crate::runner_mode::RunnerMode;
@@ -223,7 +224,10 @@ impl Orchestrator {
 
         if !skip_upload {
             start_group!("Uploading results");
-            let last_upload_result = self.upload_all(&mut completed_runs, api_client).await?;
+            let last_upload_result = self
+                .upload_all(&mut completed_runs, api_client)
+                .await
+                .map_err(upload_failed)?;
             end_group!();
 
             if self.is_local() {
@@ -267,7 +271,10 @@ impl Orchestrator {
         let total_runs = completed_runs.len();
         for (run_part_index, (ctx, executor_name)) in completed_runs.iter_mut().enumerate() {
             // OIDC tokens can expire quickly, so refresh just before each upload
-            self.provider.set_oidc_token(api_client).await?;
+            self.provider
+                .set_oidc_token(api_client)
+                .await
+                .map_err(auth_failed)?;
 
             if run_part_index == 0 {
                 // After the mint, so this names the token the upload actually uses

src/executor/valgrind/measure.rs

@@ -6,6 +6,7 @@ use crate::executor::helpers::get_bench_command::get_bench_command;
 use crate::executor::helpers::run_command_with_log_pipe::run_command_with_log_pipe;
 use crate::executor::valgrind::helpers::ignored_objects_path::get_objects_path_to_ignore;
 use crate::executor::valgrind::helpers::python::is_free_threaded_python;
+use crate::exit_code::benchmark_failed;
 use crate::instruments::mongo_tracer::MongoTracer;
 use crate::prelude::*;
 use log::log_enabled;
@@ -220,7 +221,9 @@ pub async fn measure(
     };
     debug!("Program exit code = {cmd_status}");
     if cmd_status != 0 {
-        bail!("failed to execute the benchmark process, exit code: {cmd_status}");
+        return Err(benchmark_failed(anyhow!(
+            "failed to execute the benchmark process, exit code: {cmd_status}"
+        )));
     }
 
     Ok(())

src/executor/wall_time/executor.rs

@@ -18,6 +18,7 @@ use crate::executor::helpers::run_with_sudo::wrap_with_sudo;
 use crate::executor::shared::fifo::FifoBenchmarkData;
 use crate::executor::shared::fifo::RunnerFifo;
 use crate::executor::{ExecutionContext, ExecutorName, ExecutorSupport};
+use crate::exit_code::benchmark_failed;
 use crate::instruments::mongo_tracer::MongoTracer;
 use crate::prelude::*;
 use crate::runner_mode::RunnerMode;
@@ -177,7 +178,9 @@ impl Executor for WallTimeExecutor {
         debug!("cmd exit status: {status:?}");
 
         if !status.success() {
-            bail!("failed to execute the benchmark process: {status}");
+            return Err(benchmark_failed(anyhow!(
+                "failed to execute the benchmark process: {status}"
+            )));
         }
 
         Ok(())

src/exit_code.rs

@@ -0,0 +1,106 @@
+use std::fmt;
+
+use crate::prelude::*;
+
+/// Generic failure. Something not covered by other codes.
+pub const FAILURE: i32 = 1;
+
+/// Invalid command-line usage (from Clap).
+pub const USAGE: i32 = 2;
+
+/// The benchmark process (the workload+tool) exited with a non-zero status.
+/// Only simulation mode separates the workload from the tool,
+/// because the wrapper script records the program's own status;
+/// elsewhere the tool's status is the only one known.
+pub const BENCHMARK_FAILED: i32 = 3;
+
+/// Could not authenticate against CodSpeed.
+/// Retrying without fixing the credentials will not help.
+pub const AUTH_FAILED: i32 = 4;
+
+/// Benchmarks ran successfully, but their results could not be uploaded to CodSpeed
+/// (e.g. network error, server error, etc.).
+pub const UPLOAD_FAILED: i32 = 5;
+
+/// An error tagged with the exit code to report for it.
+#[derive(Debug)]
+pub struct Marked {
+    code: i32,
+    error: Error,
+}
+
+impl Marked {
+    fn wrap(code: i32, error: Error) -> Error {
+        let code = code_of(&error).unwrap_or(code);
+        Error::new(Marked { code, error })
+    }
+}
+
+impl fmt::Display for Marked {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt::Display::fmt(&self.error, f)
+    }
+}
+
+impl std::error::Error for Marked {
+    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+        self.error.source()
+    }
+}
+
+pub fn benchmark_failed(error: Error) -> Error {
+    Marked::wrap(BENCHMARK_FAILED, error)
+}
+
+pub fn auth_failed(error: Error) -> Error {
+    Marked::wrap(AUTH_FAILED, error)
+}
+
+pub fn upload_failed(error: Error) -> Error {
+    Marked::wrap(UPLOAD_FAILED, error)
+}
+
+pub fn help_text() -> String {
+    format!(
+        "Exit codes:\n  \
+         0  Success\n  \
+         {FAILURE}  Failure\n  \
+         {USAGE}  Invalid command-line usage\n  \
+         {BENCHMARK_FAILED}  The benchmark process failed\n  \
+         {AUTH_FAILED}  Authentication or authorization failed\n  \
+         {UPLOAD_FAILED}  The benchmarks ran, but their results could not be uploaded"
+    )
+}
+
+fn code_of(error: &Error) -> Option<i32> {
+    error
+        .chain()
+        .find_map(|cause| cause.downcast_ref::<Marked>())
+        .map(|marked| marked.code)
+}
+
+pub fn exit_code_for(error: &Error) -> i32 {
+    code_of(error).unwrap_or(FAILURE)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_exit_codes() {
+        assert_eq!(exit_code_for(&anyhow!("something went wrong")), FAILURE);
+        assert_eq!(
+            exit_code_for(&benchmark_failed(anyhow!("exit status: 101"))),
+            BENCHMARK_FAILED
+        );
+        assert_eq!(
+            exit_code_for(&auth_failed(anyhow!("Invalid token"))),
+            AUTH_FAILED
+        );
+        assert_eq!(
+            exit_code_for(&upload_failed(anyhow!("connection reset")).context("Uploading results")),
+            UPLOAD_FAILED
+        );
+    }
+}

src/lib.rs

@@ -6,6 +6,7 @@ mod binary_pins;
 pub mod cli;
 mod config;
 mod executor;
+pub mod exit_code;
 mod instruments;
 mod local_logger;
 pub mod logger;

src/main.rs

@@ -1,11 +1,12 @@
-use codspeed_runner::{clean_logger, cli};
+use codspeed_runner::{clean_logger, cli, exit_code};
 use console::style;
 use log::log_enabled;
 
 #[tokio::main(flavor = "current_thread")]
 async fn main() {
     let res = cli::run().await;
     if let Err(err) = res {
+        let code = exit_code::exit_code_for(&err);
         // Show the primary error
         let mut chain = err.chain();
         if let Some(primary) = chain.next() {
@@ -22,6 +23,6 @@ async fn main() {
             }
         }
         clean_logger();
-        std::process::exit(1);
+        std::process::exit(code);
     }
 }

src/run_environment/local/provider.rs

@@ -11,6 +11,7 @@ use crate::api_client::{
 use crate::cli::run::helpers::{find_repository_root, parse_repository_from_remote};
 use crate::executor::config::OrchestratorConfig;
 use crate::executor::config::RepositoryOverride;
+use crate::exit_code::auth_failed;
 use crate::local_logger::get_local_logger;
 use crate::prelude::*;
 use crate::run_environment::interfaces::{
@@ -271,9 +272,9 @@ impl LocalProvider {
             })
             .await
             .map_err(|err| match err {
-                SessionAndRepositoryOverviewError::Unauthenticated => {
-                    anyhow!("Invalid token. Run `codspeed auth login` to re-authenticate.")
-                }
+                SessionAndRepositoryOverviewError::Unauthenticated => auth_failed(anyhow!(
+                    "Invalid token. Run `codspeed auth login` to re-authenticate."
+                )),
                 SessionAndRepositoryOverviewError::Other(err) => err,
             })?;
 

src/run_environment/mod.rs

@@ -11,6 +11,7 @@ use provider::RunEnvironmentDetector;
 
 use crate::api_client::CodSpeedAPIClient;
 use crate::executor::config::OrchestratorConfig;
+use crate::exit_code;
 use crate::prelude::*;
 
 pub use self::interfaces::*;
@@ -49,7 +50,9 @@ pub async fn get_provider(
         }
     };
 
-    provider.check_oidc_configuration(api_client)?;
+    provider
+        .check_oidc_configuration(api_client)
+        .map_err(exit_code::auth_failed)?;
 
     Ok(provider)
 }

src/upload/uploader.rs

@@ -1,7 +1,10 @@
+use super::interfaces::{UploadData, UploadMetadata};
+use super::profile_archive::ProfileArchive;
 use crate::api_client::CodSpeedAPIClient;
 use crate::executor::ExecutionContext;
 use crate::executor::ExecutorName;
 use crate::executor::Orchestrator;
+use crate::exit_code::auth_failed;
 use crate::run_environment::RunEnvironment;
 use crate::upload::{UploadError, profile_archive::ProfileArchiveContent};
 use crate::{
@@ -21,9 +24,6 @@ use tokio::fs::File;
 use tokio::io::AsyncWriteExt;
 use tokio_tar::Builder;
 
-use super::interfaces::{UploadData, UploadMetadata};
-use super::profile_archive::ProfileArchive;
-
 fn bytes_to_mib(bytes: u64) -> u64 {
     bytes / (1024 * 1024)
 }
@@ -175,13 +175,18 @@ async fn retrieve_upload_data(
                     upload_metadata.run_environment_metadata.repository
                 );
 
-                bail!(
+                let error = anyhow!(
                     "Failed to retrieve upload data: {}\n  -> {} {}",
                     status,
                     style("Reason:").bold(),
                     // we have to manually apply the style to the error message, because nesting styles is not supported by the console crate: https://github.com/console-rs/console/issues/106
                     style(error_message).red()
                 );
+                return Err(if status == StatusCode::UNAUTHORIZED {
+                    auth_failed(error)
+                } else {
+                    error
+                });
             }
 
             Ok(response.json().await?)