diff --git a/crates/berth-core/src/provisioner.rs b/crates/berth-core/src/provisioner.rs index 49440db..f62e77e 100644 --- a/crates/berth-core/src/provisioner.rs +++ b/crates/berth-core/src/provisioner.rs @@ -34,6 +34,8 @@ pub const SECRETS: &str = "secrets"; pub const REMOVE: &str = "remove"; /// berth's own action that makes a provisioner registered by hand a node's. pub const ADOPT: &str = "adopt"; +/// berth's own action that gives a registered provisioner another name. +pub const RENAME: &str = "rename"; /// The start of berth's own action that makes a resource of a kind, such as /// `new-deployment`. pub const NEW: &str = "new-"; @@ -250,6 +252,113 @@ pub fn remove_entry(paths: &Paths, store: &Store, name: &str) -> Result<()> { Ok(()) } +/// Why the registered provisioner `old` cannot be called `new`, if it cannot. +pub fn rename_problem(paths: &Paths, old: &str, new: &str) -> Result> { + if entry(paths, old)?.is_none() { + return Err( + format!("No provisioner {old} is registered. See: berth provisioner list").into(), + ); + } + Ok(if !plugins::valid_name(new) { + Some(format!( + "{new} is not a name for a provisioner: lowercase letters, digits, - and _, starting with a letter" + )) + } else if new == old { + Some(format!("{old} has that name already")) + } else if entry(paths, new)?.is_some() { + Some(format!("There is a provisioner {new} already")) + } else { + None + }) +} + +/// Give the registered provisioner `old` the name `new`, in one change: its +/// entry with its settings and the node it is of, the secrets it keeps of its +/// own, the records of what it made and of the machines planned with it, and +/// its jobs. Each resource keeps its id, and the machines and what runs on +/// them are not touched. The caller holds the lock of `old`, as its job does; +/// the lock of `new` and those of what it made are taken here, so that no +/// work on one of them saves its record with the old name meanwhile. +pub fn rename(paths: &Paths, store: &Store, old: &str, new: &str) -> Result<()> { + if let Some(problem) = rename_problem(paths, old, new)? { + return Err(problem.into()); + } + let _new = crate::runner::lock( + paths, + &crate::runner::Subject::Provisioner(new.to_owned()), + "rename the provisioner", + )?; + let _held = crate::resource::list(paths)? + .iter() + .filter(|kept| kept.record.provisioner == old) + .map(|kept| kept.lock(paths)) + .collect::>>()?; + let shared = store::shared(paths)?; + let mut writes: Vec<(&str, String, String, Vec)> = vec![]; + let mut removals: Vec<(&str, String, String)> = vec![]; + let mut moved = |kind: &'static str, scope: &str, from: String, to: String, value: Vec| { + writes.push((kind, scope.to_owned(), to, value)); + removals.push((kind, scope.to_owned(), from)); + }; + for kind in [kind::PROVISIONER_ENTRY, kind::PROVISIONER] { + if let Some(value) = shared.read(kind, OWN, old)? { + moved(kind, OWN, old.to_owned(), new.to_owned(), value); + } + } + let (from, to) = (secret_name(old, ""), secret_name(new, "")); + let mut stale = vec![]; + for secret in store.all_secret_names()? { + if let Some(own) = secret.strip_prefix(&from) { + if let Some(value) = shared.read(kind::SECRET, OWN, &secret)? { + moved( + kind::SECRET, + OWN, + secret.clone(), + format!("{to}{own}"), + value, + ); + } + } else if secret.starts_with(&to) { + // Left by a provisioner of that name before; the renamed one has its own. + stale.push(secret); + } + } + for scope in [OWN, crate::profiles::FILE_SCOPE] { + for (id, value) in shared.entries(kind::RESOURCE, scope)? { + let mut record: crate::resource::Record = + serde_json::from_slice(&value).map_err(|error| { + format!("The record of the resource {id} in berth's store: {error}") + })?; + if record.provisioner == old { + record.provisioner = new.to_owned(); + writes.push(( + kind::RESOURCE, + scope.to_owned(), + id, + serde_json::to_vec(&record)?, + )); + } + } + } + let written: Vec<(&str, &str, &str, Vec)> = writes + .iter() + .map(|(kind, scope, name, value)| (*kind, scope.as_str(), name.as_str(), value.clone())) + .collect(); + let mut removed: Vec<(&str, &str, &str)> = removals + .iter() + .map(|(kind, scope, name)| (*kind, scope.as_str(), name.as_str())) + .collect(); + removed.extend( + stale + .iter() + .filter(|secret| !written.iter().any(|(_, _, name, _)| name == secret)) + .map(|secret| (kind::SECRET, OWN, secret.as_str())), + ); + let subject = |name: &str| crate::runner::Subject::Provisioner(name.to_owned()).key(); + shared.rename_subject(&subject(old), &subject(new), &written, &removed)?; + Ok(()) +} + /// The values of a provisioner's own arguments that berth keeps, by name. pub fn settings(paths: &Paths, name: &str) -> Result> { if let Some(entry) = entry(paths, name)? { @@ -608,6 +717,7 @@ pub fn actions(paths: &Paths, store: &Store, name: &str) -> Result> if entry(paths, name)?.is_some_and(|kept| kept.node.is_empty()) { actions.push(adopt_action(name)); } + actions.push(rename_action(name)); actions.push(remove_action(paths, name)); } Ok(actions) @@ -741,6 +851,27 @@ fn adopt_action(name: &str) -> Action { } } +/// berth's own action that gives a registered provisioner another name. +fn rename_action(name: &str) -> Action { + Action { + name: RENAME.into(), + label: "Rename".into(), + help: format!( + "Give {name} another name in berth. Its settings, the secrets it keeps of its own, the resources and machines it made or plans, and its jobs go with it. What it made is not touched, and a node's provisioner stays the node's." + ), + group: "Set-Up".into(), + arguments: vec![Argument { + name: "name".into(), + label: "New name".into(), + required: true, + help: "Lowercase letters, digits, - and _, starting with a letter".into(), + ..Argument::default() + }], + role: Role::Admin, + ..Action::default() + } +} + /// berth's own action that forgets a registered provisioner, which it does /// only when it holds nothing. fn remove_action(paths: &Paths, name: &str) -> Action { @@ -800,6 +931,7 @@ enum Doing { Secrets(BTreeMap, Store), Remove(Store), Adopt(String), + Rename(String, Store), Create(Box, Store), Program { dir: PathBuf, @@ -918,6 +1050,22 @@ pub fn prepare( work: Doing::Adopt(values.get("node").cloned().unwrap_or_default()), }); } + if action == RENAME { + let values = values_of(name, &found.arguments, given, &BTreeMap::new())?; + let new = values + .get("name") + .map(|new| new.trim().to_owned()) + .unwrap_or_default(); + if let Some(problem) = rename_problem(paths, name, &new)? { + return Err(problem.into()); + } + return Ok(Acting { + name: name.to_owned(), + hidden: vec![], + action: found, + work: Doing::Rename(new, store.clone()), + }); + } if let Some(kind) = action .strip_prefix(NEW) .filter(|kind| provisioner.kinds.iter().any(|known| known.id == *kind)) @@ -1092,6 +1240,14 @@ impl Acting { details: vec![], exit_code: None, }), + Doing::Rename(new, store) => { + rename(paths, &store, &self.name, &new)?; + Ok(Answered { + message: format!("{} is called {new} now", self.name), + details: vec![], + exit_code: None, + }) + } Doing::Create(creating, store) => { let id = creating.kept().record.id.clone(); let title = creating.kept().title(); diff --git a/crates/berth-core/src/store/mod.rs b/crates/berth-core/src/store/mod.rs index 79a7799..31ee510 100644 --- a/crates/berth-core/src/store/mod.rs +++ b/crates/berth-core/src/store/mod.rs @@ -145,6 +145,20 @@ pub struct Filter<'a> { pub limit: usize, } +/// Which jobs a list has, newest first. +#[derive(Debug, Clone, Default)] +pub struct JobFilter<'a> { + /// Only those of this subject, as [`crate::runner::Subject::key`] says it. + pub subject: Option<&'a str>, + /// Only those about this host, alone or among others. + pub host: Option<&'a str>, + /// Only those that have not ended, or only those that have. + pub running: Option, + /// Only those that started before the job of this id. + pub before: Option<&'a str>, + pub limit: usize, +} + #[derive(Debug, Clone, PartialEq)] pub struct Entry { pub key: String, @@ -393,7 +407,9 @@ pub trait Backend: Send { fn stop_job(&mut self, id: &str, by: &str) -> Result<()>; fn end_lost_jobs(&mut self, ended: &str) -> Result; fn job(&mut self, id: &str) -> Result>; - fn jobs(&mut self, limit: usize) -> Result>; + /// The jobs `filter` lets through, newest first. A `before` that names + /// no job lets none through. + fn jobs(&mut self, filter: &JobFilter<'_>) -> Result>; /// Drop messages from before `cutoff` and jobs that ended and started /// before it, in one transaction. Returns the dropped jobs' output files. @@ -428,6 +444,17 @@ pub trait Backend: Send { writes: &[StateRow], removals: &[(String, String, String)], ) -> Result<()>; + /// In one transaction: the change of [`Backend::change_states`], and the + /// jobs of the subject `old`, such as `provisioner:NAME`, given to `new`. + /// The tokens of jobs `new` had before go, so that asking again does not + /// find them. + fn rename_subject( + &mut self, + old: &str, + new: &str, + writes: &[StateRow], + removals: &[(String, String, String)], + ) -> Result<()>; /// In one transaction, while nobody else writes to the table: give each /// row the value and `sealed` of its new version, leaving `updated` as /// it is. False, with nothing changed, when the table no longer holds @@ -643,7 +670,20 @@ impl Store { /// Jobs, newest first. pub fn jobs(&self, limit: usize) -> Result> { - self.backend().jobs(limit.clamp(1, 1000)) + self.find_jobs(&JobFilter { + limit, + ..JobFilter::default() + }) + } + + /// The jobs `filter` lets through, newest first: at most its limit, + /// which is between 1 and 1,000. + pub fn find_jobs(&self, filter: &JobFilter<'_>) -> Result> { + let filter = JobFilter { + limit: filter.limit.clamp(1, 1000), + ..filter.clone() + }; + self.backend().jobs(&filter) } // ── Pruning ───────────────────────────────────────────────────────────── @@ -816,6 +856,30 @@ impl Store { self.backend().rename_host(old, new, &rows, &removals) } + /// A change as [`Store::change`] makes it, in the same transaction as + /// giving the jobs of the subject `old` to `new`, both as + /// [`crate::runner::Subject::key`] says them. + pub fn rename_subject( + &self, + old: &str, + new: &str, + writes: &[(&str, &str, &str, Vec)], + removals: &[(&str, &str, &str)], + ) -> Result<()> { + let updated = now(); + let rows = writes + .iter() + .map(|(kind, scope, name, value)| self.row(kind, scope, name, value, &updated)) + .collect::>>()?; + let removals: Vec<_> = removals + .iter() + .map(|(kind, scope, name)| { + ((*kind).to_owned(), (*scope).to_owned(), (*name).to_owned()) + }) + .collect(); + self.backend().rename_subject(old, new, &rows, &removals) + } + /// The ids of the keys the sealed rows of `state` were made with. pub fn key_ids(&self) -> Result> { let rows = self.backend().all_states()?; diff --git a/crates/berth-core/src/store/postgres.rs b/crates/berth-core/src/store/postgres.rs index da41209..fa74c9e 100644 --- a/crates/berth-core/src/store/postgres.rs +++ b/crates/berth-core/src/store/postgres.rs @@ -6,8 +6,8 @@ //! answer. use super::{ - Backend, CursorRow, Entry, Error, Filter, Job, KvRow, MessageRow, Result, Rows, StateRow, - Stored, Table, When, json_value, keep_from, keep_name, + Backend, CursorRow, Entry, Error, Filter, Job, JobFilter, KvRow, MessageRow, Result, Rows, + StateRow, Stored, Table, When, json_value, keep_from, keep_name, }; use berth_api::{From, Keep, Level, Message}; use postgres::config::SslMode; @@ -743,11 +743,22 @@ impl Backend for PostgresBackend { }) } - fn jobs(&mut self, limit: usize) -> Result> { + fn jobs(&mut self, filter: &JobFilter<'_>) -> Result> { + let subject = filter.subject.map(str::to_owned); + let host = filter.host.map(str::to_owned); + let (running, before) = (filter.running, filter.before.map(str::to_owned)); + let limit = filter.limit as i64; self.call(move |client| { let rows = client.query( - &format!("SELECT {JOB_COLUMNS} FROM jobs ORDER BY started DESC, id DESC LIMIT $1"), - &[&(limit as i64)], + &format!( + "SELECT {JOB_COLUMNS} FROM jobs + WHERE ($1::TEXT IS NULL OR subject = $1) + AND ($2::TEXT IS NULL OR strpos(', ' || host || ', ', ', ' || $2 || ', ') > 0) + AND ($3::BOOLEAN IS NULL OR (ended IS NULL) = $3) + AND ($4::TEXT IS NULL OR (started, id) < (SELECT started, id FROM jobs WHERE id = $4)) + ORDER BY started DESC, id DESC LIMIT $5" + ), + &[&subject, &host, &running, &before, &limit], )?; Ok(rows.iter().map(job).collect()) }) @@ -901,6 +912,35 @@ impl Backend for PostgresBackend { }) } + fn rename_subject( + &mut self, + old: &str, + new: &str, + writes: &[StateRow], + removals: &[(String, String, String)], + ) -> Result<()> { + let (writes, removals) = (writes.to_vec(), removals.to_vec()); + let (old, new) = (old.to_owned(), new.to_owned()); + self.call(move |client| { + let mut transaction = client.transaction()?; + for row in &writes { + put_state(&mut transaction, row)?; + } + for (kind, scope, name) in &removals { + transaction.execute( + "DELETE FROM state WHERE kind = $1 AND scope = $2 AND name = $3", + &[kind, scope, name], + )?; + } + transaction.execute("UPDATE jobs SET token = NULL WHERE subject = $1", &[&new])?; + transaction.execute( + "UPDATE jobs SET subject = $2 WHERE subject = $1", + &[&old, &new], + )?; + transaction.commit() + }) + } + fn swap_states(&mut self, rows: &[(StateRow, StateRow)]) -> Result { let rows = rows.to_vec(); self.call(move |client| { diff --git a/crates/berth-core/src/store/sqlite.rs b/crates/berth-core/src/store/sqlite.rs index c03d030..173801b 100644 --- a/crates/berth-core/src/store/sqlite.rs +++ b/crates/berth-core/src/store/sqlite.rs @@ -1,8 +1,8 @@ //! The store in a SQLite file. use super::{ - Backend, CursorRow, Entry, Error, Filter, Job, KvRow, MessageRow, Result, Rows, StateRow, - Stored, Table, When, json_value, keep_from, keep_name, + Backend, CursorRow, Entry, Error, Filter, Job, JobFilter, KvRow, MessageRow, Result, Rows, + StateRow, Stored, Table, When, json_value, keep_from, keep_name, }; use berth_api::{From, Level, Message}; use rusqlite::{Connection, OptionalExtension, params}; @@ -477,11 +477,25 @@ impl Backend for SqliteBackend { Ok(found) } - fn jobs(&mut self, limit: usize) -> Result> { + fn jobs(&mut self, filter: &JobFilter<'_>) -> Result> { let mut statement = self.db.prepare(&format!( - "SELECT {JOB_COLUMNS} FROM jobs ORDER BY started DESC, id DESC LIMIT ?1" + "SELECT {JOB_COLUMNS} FROM jobs + WHERE (?1 IS NULL OR subject = ?1) + AND (?2 IS NULL OR instr(', ' || host || ', ', ', ' || ?2 || ', ') > 0) + AND (?3 IS NULL OR (ended IS NULL) = ?3) + AND (?4 IS NULL OR (started, id) < (SELECT started, id FROM jobs WHERE id = ?4)) + ORDER BY started DESC, id DESC LIMIT ?5" ))?; - let rows = statement.query_map([limit as i64], job)?; + let rows = statement.query_map( + params![ + filter.subject, + filter.host, + filter.running, + filter.before, + filter.limit as i64 + ], + job, + )?; Ok(rows.collect::>>()?) } @@ -618,6 +632,34 @@ impl Backend for SqliteBackend { Ok(()) } + fn rename_subject( + &mut self, + old: &str, + new: &str, + writes: &[StateRow], + removals: &[(String, String, String)], + ) -> Result<()> { + let transaction = self + .db + .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; + for row in writes { + put_state(&transaction, row)?; + } + for (kind, scope, name) in removals { + transaction.execute( + "DELETE FROM state WHERE kind = ?1 AND scope = ?2 AND name = ?3", + [kind, scope, name], + )?; + } + transaction.execute("UPDATE jobs SET token = NULL WHERE subject = ?1", [new])?; + transaction.execute( + "UPDATE jobs SET subject = ?2 WHERE subject = ?1", + [old, new], + )?; + transaction.commit()?; + Ok(()) + } + fn swap_states(&mut self, rows: &[(StateRow, StateRow)]) -> Result { // Immediate: no other writer gets in between the count and the updates. let transaction = self diff --git a/crates/berth-core/src/store/suite.rs b/crates/berth-core/src/store/suite.rs index 4d359f7..7702c6c 100644 --- a/crates/berth-core/src/store/suite.rs +++ b/crates/berth-core/src/store/suite.rs @@ -87,11 +87,13 @@ on_each_backend!( old_messages_and_jobs_are_pruned_and_values_stay, a_repeated_token_finds_the_job_it_started, a_job_keeps_what_it_was_started_with, + jobs_are_listed_by_subject_host_and_state_a_page_at_a_time, a_store_is_up_to_date_and_says_what_it_is, state_rows_are_kept_listed_and_removed, replacing_states_leaves_other_scopes_and_kinds_alone, changes_are_all_or_nothing, a_renamed_host_takes_its_rows_messages_and_jobs, + a_renamed_subject_takes_its_rows_and_jobs, sealed_values_round_trip, a_sealed_store_without_its_key_says_so, plain_rows_are_read_with_a_key_loaded, @@ -513,6 +515,97 @@ fn a_job_keeps_what_it_was_started_with(place: &Place) { assert!(listed.iter().any(|job| job.arguments.len() == 3)); } +fn jobs_are_listed_by_subject_host_and_state_a_page_at_a_time(place: &Place) { + let store = place.open(); + let job = |id: &str, age: u64, host: &str, subject: &str, ended: bool| Job { + id: id.into(), + host: host.into(), + what: "apply".into(), + started: rfc3339(unix_now() - age), + ended: ended.then(|| rfc3339(unix_now())), + output_path: format!("/tmp/{id}.log"), + subject: subject.into(), + action: "apply".into(), + ..Job::default() + }; + store + .add_job(&job("j1", 50, "studio", "host:studio", true)) + .unwrap(); + store + .add_job(&job("j2", 40, "studio, lab", "installation", true)) + .unwrap(); + store + .add_job(&job("j3", 30, "", "provisioner:vm", false)) + .unwrap(); + store + .add_job(&job("j4", 20, "studio-2", "host:studio-2", true)) + .unwrap(); + store + .add_job(&job("j5", 10, "studio", "host:studio", false)) + .unwrap(); + let ids = |filter: JobFilter| -> Vec { + store + .find_jobs(&JobFilter { + limit: 10, + ..filter + }) + .unwrap() + .into_iter() + .map(|job| job.id) + .collect() + }; + assert_eq!(ids(JobFilter::default()), ["j5", "j4", "j3", "j2", "j1"]); + // A host is found alone and among others, and not as part of another's name. + let studio = JobFilter { + host: Some("studio"), + ..JobFilter::default() + }; + assert_eq!(ids(studio.clone()), ["j5", "j2", "j1"]); + assert_eq!( + ids(JobFilter { + subject: Some("host:studio"), + ..JobFilter::default() + }), + ["j5", "j1"] + ); + assert_eq!( + ids(JobFilter { + running: Some(true), + ..JobFilter::default() + }), + ["j5", "j3"] + ); + assert_eq!( + ids(JobFilter { + running: Some(false), + ..studio.clone() + }), + ["j2", "j1"] + ); + // A page goes on after the last job of the one before. + let first = store + .find_jobs(&JobFilter { + limit: 2, + ..JobFilter::default() + }) + .unwrap(); + assert_eq!(first.len(), 2); + assert_eq!( + ids(JobFilter { + before: Some(&first[1].id), + ..JobFilter::default() + }), + ["j3", "j2", "j1"] + ); + assert!( + ids(JobFilter { + before: Some("nothing"), + ..JobFilter::default() + }) + .is_empty() + ); +} + fn a_store_is_up_to_date_and_says_what_it_is(place: &Place) { let store = place.open(); let (name, steps) = match place { @@ -796,6 +889,63 @@ fn a_renamed_host_takes_its_rows_messages_and_jobs(place: &Place) { assert_eq!(store.job("j3").unwrap().unwrap().host, "test, lab"); } +fn a_renamed_subject_takes_its_rows_and_jobs(place: &Place) { + let store = place.open().with_keys(Some(keys()), None); + store.write("provisioner-entry", OWN, "vm", b"e").unwrap(); + let job = |id: &str, subject: &str, token: Option<&str>| Job { + id: id.into(), + what: "settings".into(), + started: rfc3339(unix_now()), + output_path: format!("/tmp/{id}.log"), + subject: subject.into(), + action: "settings".into(), + token: token.map(str::to_owned), + ..Job::default() + }; + store + .add_job(&job("j1", "provisioner:vm", Some("t"))) + .unwrap(); + // A provisioner of the new name that was removed left a job. + store + .add_job(&job("j2", "provisioner:lab", Some("t"))) + .unwrap(); + store + .add_job(&job("j3", "provisioner:other", None)) + .unwrap(); + store + .rename_subject( + "provisioner:vm", + "provisioner:lab", + &[("provisioner-entry", OWN, "lab", b"e".to_vec())], + &[("provisioner-entry", OWN, "vm")], + ) + .unwrap(); + assert_eq!( + all_names(&store), + owned(&[("provisioner-entry", "", "lab")]) + ); + assert_eq!( + store + .read("provisioner-entry", OWN, "lab") + .unwrap() + .unwrap(), + b"e" + ); + let subject = |id: &str| store.job(id).unwrap().unwrap().subject; + assert_eq!( + (subject("j1"), subject("j2"), subject("j3")), + ( + "provisioner:lab".to_owned(), + "provisioner:lab".to_owned(), + "provisioner:other".to_owned() + ) + ); + let found = store + .job_by_token("provisioner:lab", "settings", "t") + .unwrap(); + assert_eq!(found.map(|job| job.id), Some("j1".into())); +} + fn sealed_values_round_trip(place: &Place) { let keys = keys(); let store = place.open().with_keys(Some(copy(&keys)), None); diff --git a/crates/berth/src/cli/args.rs b/crates/berth/src/cli/args.rs index cc46a3f..422ff89 100644 --- a/crates/berth/src/cli/args.rs +++ b/crates/berth/src/cli/args.rs @@ -524,6 +524,8 @@ pub enum Provisioner { #[arg(value_name = "NAME=VALUE", required = true)] settings: Vec, }, + /// Give a provisioner another name in berth, with its settings, secrets and what it made. + Rename { name: String, new: String }, /// Forget a provisioner that holds nothing. #[command(visible_alias = "rm")] Remove { name: String }, @@ -652,6 +654,18 @@ pub enum Job { List { #[arg(long, default_value_t = 20)] limit: u32, + /// Only the jobs about this host. + #[arg(long)] + host: Option, + /// Only the jobs of this subject, such as provisioner:NAME or installation. + #[arg(long)] + subject: Option, + /// Only the jobs that are still running. + #[arg(long)] + running: bool, + /// Only the jobs that started before this one, to see the next page. + #[arg(long, value_name = "ID")] + before: Option, }, /// One job's state. Show { id: String }, @@ -854,7 +868,14 @@ pub enum Inbox { /// Done ones as well. #[arg(long)] all: bool, + /// Only those taken in before this one, by the number the list ends with. + #[arg(long, value_name = "SEQ")] + before: Option, }, + /// Mark a message done, so the list leaves it out. + Done { host: String, id: u64 }, + /// Mark a done message as open again. + Reopen { host: String, id: u64 }, } #[derive(Subcommand, Debug)] diff --git a/crates/berth/src/cli/data.rs b/crates/berth/src/cli/data.rs index 3ccac85..292001f 100644 --- a/crates/berth/src/cli/data.rs +++ b/crates/berth/src/cli/data.rs @@ -174,7 +174,24 @@ pub fn forward( } pub fn inbox(client: &Client, command: Inbox) -> Result<()> { - let Inbox::List { host, topic, all } = command; + let (host, topic, all, before) = match command { + Inbox::List { + host, + topic, + all, + before, + } => (host, topic, all, before), + Inbox::Done { host, id } => { + client.call("inbox.done", json!({"host": host, "id": id}), None)?; + out::done(format!("Message {id} of {host} is done")); + return Ok(()); + } + Inbox::Reopen { host, id } => { + client.call("inbox.reopen", json!({"host": host, "id": id}), None)?; + out::done(format!("Message {id} of {host} is open again")); + return Ok(()); + } + }; let mut args = json!({"limit": 200}); if let Some(host) = host { args["host"] = json!(host); @@ -185,6 +202,9 @@ pub fn inbox(client: &Client, command: Inbox) -> Result<()> { if !all { args["done"] = json!(false); } + if let Some(before) = before { + args["before"] = json!(before); + } let found = client.call("inbox.list", args, None)?; out::data(&found, || { let mut table = Table::new(&["", "time", "host", "id", "topic", "body"]); @@ -205,8 +225,10 @@ pub fn inbox(client: &Client, command: Inbox) -> Result<()> { ]); } table.print("No messages"); - if !found["next"].is_null() { - out::hint("Older messages are not shown"); + if let Some(next) = found["next"].as_i64() { + out::hint(format!( + "Older messages are not shown. The next ones: berth inbox list --before {next}" + )); } }); Ok(()) diff --git a/crates/berth/src/cli/jobs.rs b/crates/berth/src/cli/jobs.rs index 7406428..dd20884 100644 --- a/crates/berth/src/cli/jobs.rs +++ b/crates/berth/src/cli/jobs.rs @@ -24,8 +24,23 @@ fn state_of(job: &Value) -> String { pub fn command(client: &Client, command: Job) -> Result<()> { match command { - Job::List { limit } => { - let found = client.call("jobs.list", json!({"limit": limit}), None)?; + Job::List { + limit, + host, + subject, + running, + before, + } => { + let mut args = json!({"limit": limit}); + for (key, value) in [("host", host), ("subject", subject), ("before", before)] { + if let Some(value) = value { + args[key] = json!(value); + } + } + if running { + args["running"] = json!(true); + } + let found = client.call("jobs.list", args, None)?; out::data(&found, || { let mut table = Table::new(&["id", "action", "subject", "state", "started"]); for job in found.as_array().into_iter().flatten() { diff --git a/crates/berth/src/cli/things.rs b/crates/berth/src/cli/things.rs index 2f7aacb..be25371 100644 --- a/crates/berth/src/cli/things.rs +++ b/crates/berth/src/cli/things.rs @@ -657,6 +657,21 @@ pub fn provisioner(client: &Client, command: Provisioner) -> Result<()> { )?; Ok(()) } + Provisioner::Rename { name, new } => { + let act = Act { + arguments: BTreeMap::from([("name".to_owned(), new.clone())]), + ..Act::default() + }; + let acted = client.act(&Subject::Provisioner(name.clone()), "rename", &act)?; + follow::acted( + client, + acted, + &format!("Renaming {name} to {new}"), + false, + false, + )?; + Ok(()) + } Provisioner::Remove { name } => { ask::confirm(&format!( "This forgets the provisioner {name} with its settings." diff --git a/crates/berth/src/local/serve/api.rs b/crates/berth/src/local/serve/api.rs index b027344..dca46d5 100644 --- a/crates/berth/src/local/serve/api.rs +++ b/crates/berth/src/local/serve/api.rs @@ -693,15 +693,33 @@ async fn host(State(state): App, Path(name): Path) -> Answer, + /// Only the jobs about this host, alone or among others, as a roll-out is. + host: Option, + /// Only the jobs that have not ended, or with `false` only those that have. + running: Option, + /// Only the jobs that started before the job of this id: the next page after it. + before: Option, /// At most this many (default 100, at most 1,000). limit: Option, } -#[utoipa::path(get, path = "/v1/jobs", summary = "The jobs, newest first.", tag = "jobs", params(Limit), - responses((status = 200, description = "Jobs, newest first", body = Vec)))] -async fn jobs(State(state): App, Query(query): Query) -> Answer>> { - let jobs = state.store.jobs(query.limit.unwrap_or(100))?; +#[utoipa::path(get, path = "/v1/jobs", summary = "The jobs, newest first.", tag = "jobs", params(JobsQuery), + responses((status = 200, description = "Jobs, newest first", body = Vec), + (status = 404, description = "`before` names no job", body = ErrorBody)))] +async fn jobs(State(state): App, Query(query): Query) -> Answer>> { + if let Some(before) = &query.before { + find_job(&state, before)?; + } + let jobs = state.store.find_jobs(&store::JobFilter { + subject: query.subject.as_deref(), + host: query.host.as_deref(), + running: query.running, + before: query.before.as_deref(), + limit: query.limit.unwrap_or(100), + })?; Ok(Json( jobs.into_iter() .map(|job| job_info(&state.paths, job)) diff --git a/crates/berth/src/local/serve/tests.rs b/crates/berth/src/local/serve/tests.rs index 53e7893..d13eecb 100644 --- a/crates/berth/src/local/serve/tests.rs +++ b/crates/berth/src/local/serve/tests.rs @@ -2344,6 +2344,7 @@ async fn a_provisioner_has_a_page_and_actions_that_run_as_jobs() { "restart", "secrets", "adopt", + "rename", "remove" ] ); @@ -2612,6 +2613,184 @@ async fn a_provisioner_has_a_page_and_actions_that_run_as_jobs() { ); } +#[tokio::test(flavor = "multi_thread")] +async fn a_provisioner_is_renamed_with_everything_that_names_it() { + use berth_core::resource::{Kept, Record}; + let served = served_with(quick).await; + fake_provisioner(&served); + let mut client = served.client().await; + let client = &mut client; + register_fake(client).await; + served.state.paths.prepare().unwrap(); + let paths = &served.state.paths; + let registries = berth_core::registry::Store::own(served.state.store.clone()); + let message = |said: &Value| said["error"]["message"].as_str().unwrap_or("").to_owned(); + let (status, acted) = send( + client, + "POST", + "/v1/provisioners/vm/actions/settings", + json!({"arguments": {"server": "https://192.0.2.10:8443"}}), + ) + .await; + assert_eq!(status, 202, "{acted}"); + let before = acted["job"]["id"].as_str().unwrap().to_owned(); + job_ended(&served.socket, &before).await; + let own = berth_core::provisioner::secret_name("vm", "api_token"); + registries.set_secret(&own, Some("tok-own")).unwrap(); + // One left by a provisioner of the new name before is not taken over. + let stale = berth_core::provisioner::secret_name("lab", "other"); + registries.set_secret(&stale, Some("tok-stale")).unwrap(); + let record = |id: &str, kind: &str, provisioner: &str, host: &str, state: &str| Kept { + scope: String::new(), + record: Record { + id: id.into(), + provisioner: provisioner.into(), + kind: kind.into(), + name: id.into(), + host: host.into(), + state: state.into(), + ..Record::default() + }, + }; + record("r-wiki", "deployment", "vm", "", "running") + .save(paths) + .unwrap(); + record( + "r-plan", + "machine", + "vm", + "studio", + berth_core::resource::PLANNED, + ) + .save(paths) + .unwrap(); + record("r-other", "deployment", "old", "", "running") + .save(paths) + .unwrap(); + // A node's provisioner stays the node's. + let mut offered = berth_core::provisioner::entry(paths, "vm") + .unwrap() + .unwrap(); + offered.node = "dock".into(); + berth_core::provisioner::save_entry(paths, "vm", &offered).unwrap(); + + // A name that is taken or no name is refused before anything starts. + for (wanted, said) in [ + ("old", "There is a provisioner old already"), + ("Not A Name", "is not a name for a provisioner"), + ("vm", "vm has that name already"), + ] { + let (status, refused) = send( + client, + "POST", + "/v1/provisioners/vm/actions/rename", + json!({"arguments": {"name": wanted}}), + ) + .await; + assert_eq!(status, 400, "{refused}"); + assert!(message(&refused).contains(said), "{refused}"); + } + + let (status, acted) = send( + client, + "POST", + "/v1/provisioners/vm/actions/rename", + json!({"arguments": {"name": "lab"}}), + ) + .await; + assert_eq!(status, 202, "{acted}"); + let id = acted["job"]["id"].as_str().unwrap().to_owned(); + let (output, code) = job_ended(&served.socket, &id).await; + assert_eq!(code, 0, "{output}"); + assert!(output.contains("vm is called lab now"), "{output}"); + + assert_eq!( + send(client, "GET", "/v1/provisioners/vm", Value::Null) + .await + .0, + 404 + ); + let entry = berth_core::provisioner::entry(paths, "lab") + .unwrap() + .unwrap(); + assert_eq!( + (entry.kind.as_str(), entry.node.as_str()), + ("fake/vm", "dock") + ); + assert_eq!(entry.settings["server"], "https://192.0.2.10:8443"); + assert_eq!( + registries + .secret(&berth_core::provisioner::secret_name("lab", "api_token")) + .unwrap() + .as_deref(), + Some("tok-own") + ); + assert_eq!(registries.secret(&own).unwrap(), None); + assert_eq!(registries.secret(&stale).unwrap(), None); + let by: std::collections::BTreeMap = berth_core::resource::list(paths) + .unwrap() + .into_iter() + .map(|kept| (kept.record.id, kept.record.provisioner)) + .collect(); + assert_eq!( + by, + std::collections::BTreeMap::from([ + ("r-other".to_owned(), "old".to_owned()), + ("r-plan".to_owned(), "lab".to_owned()), + ("r-wiki".to_owned(), "lab".to_owned()), + ]) + ); + // Its jobs are the renamed one's, the rename among them. + for job in [&before, &id] { + let (_, found) = send(client, "GET", &format!("/v1/jobs/{job}"), Value::Null).await; + assert_eq!(found["subject"], "provisioner:lab", "{found}"); + } + // They are listed by their subject, a page at a time. + let listed = |found: &Value| -> Vec { + found + .as_array() + .unwrap() + .iter() + .map(|job| job["id"].as_str().unwrap().to_owned()) + .collect() + }; + let (_, found) = send( + client, + "GET", + "/v1/jobs?subject=provisioner:lab", + Value::Null, + ) + .await; + // Both started in the same second, so either may be first. + let both = listed(&found); + let mut sorted = both.clone(); + sorted.sort(); + let mut expected = vec![id.clone(), before.clone()]; + expected.sort(); + assert_eq!(sorted, expected); + let (_, found) = send( + client, + "GET", + "/v1/jobs?subject=provisioner:vm", + Value::Null, + ) + .await; + assert!(listed(&found).is_empty(), "{found}"); + let path = format!("/v1/jobs?subject=provisioner:lab&before={}", both[0]); + let (_, found) = send(client, "GET", &path, Value::Null).await; + assert_eq!(listed(&found), [both[1].clone()]); + let (_, found) = send(client, "GET", "/v1/jobs?running=true", Value::Null).await; + assert!(listed(&found).is_empty(), "{found}"); + let (status, _) = send(client, "GET", "/v1/jobs?before=nothing", Value::Null).await; + assert_eq!(status, 404); + let (status, page) = send(client, "GET", "/v1/provisioners/lab", Value::Null).await; + assert_eq!(status, 200, "{page}"); + assert_eq!( + page["provisioner"]["settings"]["server"], + "https://192.0.2.10:8443" + ); +} + #[tokio::test(flavor = "multi_thread")] async fn machines_are_created_and_deleted_through_the_api() { let served = served_with(quick).await; diff --git a/docs/content/docs/(guide)/deploy/provisioners.mdx b/docs/content/docs/(guide)/deploy/provisioners.mdx index 0f42fbe..3cd5d9b 100644 --- a/docs/content/docs/(guide)/deploy/provisioners.mdx +++ b/docs/content/docs/(guide)/deploy/provisioners.mdx @@ -63,6 +63,8 @@ The listing is shortened here. A secret marked *not set* is required and has no In the terminal UI, `a` on the **Provisioners** tab adds one. It asks for the name and then the type. +`berth provisioner rename NAME NEW` gives a provisioner another name. Its settings, its own secrets, the resources and machines it made or plans, and its jobs go with it, and what it made is not touched. The action `rename` on its page does the same. + `berth provisioner remove NAME` forgets a provisioner, with its settings and its own secrets. berth refuses while it holds anything, so remove what it made first. The action `remove` on its page does the same. The action `adopt` makes one you registered the provisioner of a [node](/docs/use/nodes#take-in-a-server-that-exists) that reaches the same server. ### From an earlier version @@ -98,6 +100,7 @@ Actions (berth provisioner run docker ACTION) Set-Up settings Change settings adopt Make it a node's + rename Rename remove Remove provisioner Engine engine Check the engine diff --git a/docs/content/docs/(guide)/extend/serve.mdx b/docs/content/docs/(guide)/extend/serve.mdx index f0ea353..397da5a 100644 --- a/docs/content/docs/(guide)/extend/serve.mdx +++ b/docs/content/docs/(guide)/extend/serve.mdx @@ -61,6 +61,7 @@ The inbox, jobs and key-value store are kept in berth's store for `retention_day ```bash berth inbox list studio --topic ask/ # what is open; --all includes what is done +berth inbox done studio 12 # marks message 12 of studio done berth kv set ui layout '{"columns": 2}' berth kv list ui ``` diff --git a/docs/content/docs/(guide)/reference/cli.mdx b/docs/content/docs/(guide)/reference/cli.mdx index 7dbf800..016b8ac 100644 --- a/docs/content/docs/(guide)/reference/cli.mdx +++ b/docs/content/docs/(guide)/reference/cli.mdx @@ -629,6 +629,19 @@ berth provisioner set NAME NAME=VALUE… | `NAME` | Its name. | | `NAME=VALUE…` | A setting and its new value. | +### berth provisioner rename + +Give a provisioner another name in berth, with its settings, secrets and what it made. + +```text +berth provisioner rename NAME NEW +``` + +| Argument | What it is | +| --- | --- | +| `NAME` | Its name. | +| `NEW` | Its new name. | + ### berth provisioner remove Forget a provisioner that holds nothing. @@ -818,12 +831,16 @@ The jobs, newest first. `berth job ls` is the same. ```text -berth job list [--limit LIMIT] +berth job list [--limit LIMIT] [--host HOST] [--subject SUBJECT] [--running] [--before ID] ``` | Argument | What it is | | --- | --- | | `--limit LIMIT` | By default `20`. | +| `--host HOST` | Only the jobs about this host. | +| `--subject SUBJECT` | Only the jobs of this subject, such as provisioner:NAME or installation. | +| `--running` | Only the jobs that are still running. | +| `--before ID` | Only the jobs that started before this one, to see the next page. | ### berth job show @@ -1265,7 +1282,7 @@ The messages, oldest first. `berth inbox ls` is the same. ```text -berth inbox list [HOST] [--topic PREFIX] [--all] +berth inbox list [HOST] [--topic PREFIX] [--all] [--before SEQ] ``` | Argument | What it is | @@ -1273,6 +1290,33 @@ berth inbox list [HOST] [--topic PREFIX] [--all] | `HOST` | A host's name. | | `--topic PREFIX` | Only topics that start with PREFIX. | | `--all` | Done ones as well. | +| `--before SEQ` | Only those taken in before this one, by the number the list ends with. | + +### berth inbox done + +Mark a message done, so the list leaves it out. + +```text +berth inbox done HOST ID +``` + +| Argument | What it is | +| --- | --- | +| `HOST` | A host's name. | +| `ID` | Its id. | + +### berth inbox reopen + +Mark a done message as open again. + +```text +berth inbox reopen HOST ID +``` + +| Argument | What it is | +| --- | --- | +| `HOST` | A host's name. | +| `ID` | Its id. | ## berth kv diff --git a/docs/content/docs/(guide)/use/cli.mdx b/docs/content/docs/(guide)/use/cli.mdx index 3b21aea..9a2b603 100644 --- a/docs/content/docs/(guide)/use/cli.mdx +++ b/docs/content/docs/(guide)/use/cli.mdx @@ -90,6 +90,8 @@ What changes a host runs as a job of the server, such as an apply or a roll-out. | --- | --- | | Start a job and leave it running | `berth apply studio --detach`, which needs a `berth serve` | | See the jobs | `berth job list` | +| See some of them | `berth job list --host studio`, `--subject provisioner:NAME` or `--running` | +| See the page after | `berth job list --before ID`, with the last job of the page before | | See one job | `berth job show ID` | | Read its log, and go on as it writes | `berth job logs ID -f` | | Wait for it, and take its status | `berth job wait ID` | @@ -174,6 +176,7 @@ See [Secrets](/docs/use/secrets) and [Registries](/docs/use/registries). | Call a service a plugin runs | `berth service call studio notes/store list` | | Read a host's channel as it comes | `berth channel listen studio` | | The messages the server took in | `berth inbox list` | +| Mark one done, or open again | `berth inbox done studio ID`, `berth inbox reopen studio ID` | | The server's key-value store | `berth kv list NAMESPACE`, `berth kv set NAMESPACE KEY JSON` | ## The server of this machine diff --git a/docs/content/docs/(guide)/use/nodes.mdx b/docs/content/docs/(guide)/use/nodes.mdx index e717631..4e8472d 100644 --- a/docs/content/docs/(guide)/use/nodes.mdx +++ b/docs/content/docs/(guide)/use/nodes.mdx @@ -65,7 +65,7 @@ At the end of an apply the role records what each feature offers in `/etc/berth/ | What | Where it comes from | | --- | --- | -| The name | `NODE-PLUGIN-TYPE`, such as `big-incus-vm` | +| The name | `NODE-PLUGIN-TYPE`, such as `big-incus-vm`. `berth provisioner rename` gives it another, and it stays the node's | | The node | The host it is on, on the provisioner's page (`node` over the API), and kept when the node is renamed | | Settings | Where berth reaches the node, such as `server=https://192.0.2.20:8444` | | Secrets | Public values only, such as the Incus server's certificate, kept as the provisioner's own `incus_server_cert` | diff --git a/docs/openapi/berth.openapi.json b/docs/openapi/berth.openapi.json index 2b2dfaf..7f9cbad 100644 --- a/docs/openapi/berth.openapi.json +++ b/docs/openapi/berth.openapi.json @@ -6094,6 +6094,42 @@ "get": { "operationId": "jobs", "parameters": [ + { + "description": "Only the jobs of this subject: `host:NAME`, `provisioner:NAME`, `resource:ID` or `installation`.", + "in": "query", + "name": "subject", + "required": false, + "schema": { + "type": "string" + } + }, + { + "description": "Only the jobs about this host, alone or among others, as a roll-out is.", + "in": "query", + "name": "host", + "required": false, + "schema": { + "type": "string" + } + }, + { + "description": "Only the jobs that have not ended, or with `false` only those that have.", + "in": "query", + "name": "running", + "required": false, + "schema": { + "type": "boolean" + } + }, + { + "description": "Only the jobs that started before the job of this id: the next page after it.", + "in": "query", + "name": "before", + "required": false, + "schema": { + "type": "string" + } + }, { "description": "At most this many (default 100, at most 1,000).", "in": "query", @@ -6118,6 +6154,16 @@ } }, "description": "Jobs, newest first" + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + }, + "description": "`before` names no job" } }, "summary": "The jobs, newest first.", diff --git a/docs/openapi/berth.operations.json b/docs/openapi/berth.operations.json index 0fefea4..2186374 100644 --- a/docs/openapi/berth.operations.json +++ b/docs/openapi/berth.operations.json @@ -4253,6 +4253,34 @@ }, { "args": [ + { + "description": "Only the jobs of this subject: `host:NAME`, `provisioner:NAME`, `resource:ID` or `installation`.", + "header": null, + "in": "query", + "name": "subject", + "required": false + }, + { + "description": "Only the jobs about this host, alone or among others, as a roll-out is.", + "header": null, + "in": "query", + "name": "host", + "required": false + }, + { + "description": "Only the jobs that have not ended, or with `false` only those that have.", + "header": null, + "in": "query", + "name": "running", + "required": false + }, + { + "description": "Only the jobs that started before the job of this id: the next page after it.", + "header": null, + "in": "query", + "name": "before", + "required": false + }, { "description": "At most this many (default 100, at most 1,000).", "header": null, @@ -4261,6 +4289,12 @@ "required": false } ], + "errors": [ + { + "description": "`before` names no job", + "status": "404" + } + ], "group": "jobs", "http": { "method": "GET", diff --git a/tests/e2e/e2e_provisioners.py b/tests/e2e/e2e_provisioners.py index 2f2e65a..27ec78c 100644 --- a/tests/e2e/e2e_provisioners.py +++ b/tests/e2e/e2e_provisioners.py @@ -194,7 +194,7 @@ def test_through_berth_serve_an_action_that_changes_something_is_a_job(self): 'unavailable': {'tidy': 'The bottom shelf is out of reach'}}) actions = serve.call('GET', '/v1/provisioners/box/actions') self.assertEqual([action['name'] for action in actions], - ['new-deployment', 'settings', 'look', 'sweep', 'tidy', 'adopt', 'remove']) + ['new-deployment', 'settings', 'look', 'sweep', 'tidy', 'adopt', 'rename', 'remove']) looked = serve.call('POST', '/v1/provisioners/box/actions/look') self.assertEqual(looked['result']['message'], 'Looked around attic') acted = serve.call('POST', '/v1/provisioners/box/actions/sweep', @@ -378,11 +378,20 @@ def test_5_the_provisioners_own_action_makes_one_and_it_is_kept_while_it_holds_o self.berth('provisioner', 'run', 'box', 'new-deployment', 'name=shed', 'spec=shed: 1\nbenches: 2\n', 'secrets=bench_token:TOKEN', '--yes') self.assertEqual((self.folder / 'spec.shed').read_text(), 'shed: 1\nbenches: 2\n') self.assertIn('shed', self.berth('resource', 'list').stdout) - refused = self.berth('provisioner', 'remove', 'box', '-y', check=False) + # Renamed, it keeps its settings and what it made. + taken = self.berth('provisioner', 'rename', 'box', 'old', check=False) + self.assertNotEqual(taken.returncode, 0) + self.assertIn('There is a provisioner old already', taken.stderr) + self.assertIn('box is called bench-lab now', self.berth('provisioner', 'rename', 'box', 'bench-lab').stdout) + self.assertRegex(self.berth('resource', 'list').stdout, r'(?m)^r-\S+\s+deployment\s+shed\s+bench-lab\s') + self.assertIn('The lab at attic answers', self.berth('provisioner', 'show', 'bench-lab').stdout) + self.assertIn('State running', self.berth('resource', 'show', 'shed').stdout) + self.assertNotEqual(self.berth('provisioner', 'show', 'box', check=False).returncode, 0) + refused = self.berth('provisioner', 'remove', 'bench-lab', '-y', check=False) self.assertNotEqual(refused.returncode, 0) - self.assertIn('box still holds', refused.stderr) + self.assertIn('bench-lab still holds', refused.stderr) self.berth('resource', 'remove', 'shed', '-y') - self.assertIn('Removed the provisioner box', self.berth('provisioner', 'remove', 'box', '-y').stdout) + self.assertIn('Removed the provisioner bench-lab', self.berth('provisioner', 'remove', 'bench-lab', '-y').stdout) self.assertNotIn('Lab box', self.berth('provisioner', 'list').stdout) diff --git a/tests/e2e/e2e_serve.py b/tests/e2e/e2e_serve.py index 6a8841f..ddf1307 100644 --- a/tests/e2e/e2e_serve.py +++ b/tests/e2e/e2e_serve.py @@ -80,6 +80,13 @@ def arrived(): self.assertTrue(arrived()[0]['done']) self.serve.call('DELETE', f'/v1/inbox/{HOST}/{number}/done') self.assertFalse(arrived()[0]['done']) + # The command line marks it the same way. + berth = self.serve.station.berth + self.assertIn(f'Message {number} of {HOST} is done', berth('inbox', 'done', HOST, str(number)).stdout) + self.assertTrue(arrived()[0]['done']) + self.assertNotIn(body['note'], berth('inbox', 'list', HOST, '--topic', 'e2e/inbox').stdout) + berth('inbox', 'reopen', HOST, str(number)) + self.assertFalse(arrived()[0]['done']) @needs_e2e diff --git a/tests/e2e/harness.py b/tests/e2e/harness.py index afd56f1..d3f5e00 100644 --- a/tests/e2e/harness.py +++ b/tests/e2e/harness.py @@ -144,7 +144,7 @@ def follows_a_job(args): ('host', 'rename'), ('host', 'start'), ('host', 'stop'), ('key', 'trust'), ('key', 'rotate'), ('resource', 'create'), ('resource', 'set'), ('resource', 'remove'), ('resource', 'run'), ('provisioner', 'run'), - ('provisioner', 'set')}) + ('provisioner', 'set'), ('provisioner', 'rename')}) class Workstation: