From 0d539429e749149041fea7af310a7284979cff08 Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Wed, 11 Nov 2026 14:59:58 +0000 Subject: [PATCH 01/11] Extract shared solver utilities into crate::optimisation Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/graph/investment.rs | 2 +- src/lib.rs | 1 + src/optimisation.rs | 175 ++++++++++++++++++ .../investment/competitive/constraints.rs | 2 +- .../investment/competitive/optimisation.rs | 8 +- .../investment/portfolio/optimisation.rs | 4 +- src/simulation/optimisation.rs | 81 +------- src/simulation/optimisation/constraints.rs | 87 +-------- 8 files changed, 188 insertions(+), 172 deletions(-) create mode 100644 src/optimisation.rs diff --git a/src/graph/investment.rs b/src/graph/investment.rs index ea9617a11..70bbf8d22 100644 --- a/src/graph/investment.rs +++ b/src/graph/investment.rs @@ -1,10 +1,10 @@ //! Module for solving the investment order of commodities use super::{CommoditiesGraph, GraphEdge, GraphNode}; use crate::commodity::{CommodityMap, CommodityType}; +use crate::optimisation::{ModelError, solve_optimal}; use crate::process::ProcessID; use crate::region::RegionID; use crate::simulation::market::MarketSet; -use crate::simulation::optimisation::{ModelError, solve_optimal}; use anyhow::{Context, Result}; use highs::{Col, RowProblem, Sense}; use indexmap::{IndexMap, IndexSet}; diff --git a/src/lib.rs b/src/lib.rs index 5072bf3eb..657c6007a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -32,6 +32,7 @@ pub mod id; pub mod input; pub mod log; pub mod model; +mod optimisation; pub mod output; pub mod patch; pub mod process; diff --git a/src/optimisation.rs b/src/optimisation.rs new file mode 100644 index 000000000..658b5cad1 --- /dev/null +++ b/src/optimisation.rs @@ -0,0 +1,175 @@ +//! Shared helpers for building and solving HiGHS optimisation models. +use crate::asset::AssetRef; +use anyhow::{Context, Result, anyhow}; +use highs::HighsModelStatus; +use std::collections::HashMap; +use std::error::Error; + +/// Defines the possible errors that can occur when running the solver +#[derive(Debug, derive_more::Display, derive_more::From)] +pub enum ModelError { + /// An optimal solution could not be found + #[display("Could not find optimal result: {_0:?}")] + NonOptimal(HighsModelStatus), + /// Another error occurred + #[display("{_0}")] + Other(anyhow::Error), +} + +impl ModelError { + /// Convert this error into an [`anyhow::Error`] + pub fn into_anyhow(self) -> anyhow::Error { + match self { + ModelError::NonOptimal(status) => anyhow!("Could not find optimal result: {status:?}"), + ModelError::Other(error) => error, + } + } +} + +impl Error for ModelError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + ModelError::NonOptimal(_) => None, + ModelError::Other(error) => Some(error.as_ref()), + } + } +} + +/// Apply the specified HiGHS options from a [`toml::Table`] +pub fn apply_highs_options_from_toml( + model: &mut highs::Model, + options: &toml::Table, +) -> Result<()> { + // Attempt to set an option, returning an error if it fails + macro_rules! try_set_opt { + ($option:expr, $value:expr) => {{ + model + .try_set_option($option.as_str(), $value) + .map_err(|_| anyhow!("Invalid option name or value"))?; + + Ok(()) + }}; + } + + // Iterate through options, applying each in turn to the HiGHS model + for (option, value) in options { + match value { + toml::Value::String(value) => try_set_opt!(option, value.as_str()), + toml::Value::Integer(value) => match i32::try_from(*value) { + Ok(value) => try_set_opt!(option, value), + Err(_) => Err(anyhow!("Value out of range")), + }, + toml::Value::Float(value) => try_set_opt!(option, *value), + toml::Value::Boolean(value) => try_set_opt!(option, *value), + _ => Err(anyhow!("HiGHS options cannot have this type")), + } + .with_context(|| format!("Failed to set option \"{option}\" to value \"{value}\""))?; + } + + Ok(()) +} + +/// Try to solve the model, returning an error if the model is incoherent or result is non-optimal +pub fn solve_optimal(model: highs::Model) -> Result { + let solved = model + .try_solve() + .map_err(|status| anyhow!("Incoherent model: {status:?}"))?; + + match solved.status() { + HighsModelStatus::Optimal => Ok(solved), + status => Err(status.into()), + } +} + +/// Groups assets that have equivalent dispatch properties. +/// +/// Assets are first bucketed by `dispatch_equivalence_hash()` to avoid unnecessary pairwise +/// comparisons. Within each hash bucket, assets are compared using `is_dispatch_equivalent()`, +/// which is the authoritative check for equivalence. This also handles hash collisions correctly. +/// +/// The caller must ensure that `assets` contains only assets eligible for equal-utilisation +/// constraints. +pub fn group_dispatch_equivalent_assets<'a, I>(assets: I) -> Vec> +where + I: Iterator, +{ + // Group assets by comparing each one with the first asset in each group. The hash index + // avoids comparing an asset with groups that cannot contain an equivalent asset, while the + // exact comparison preserves correctness in the event of hash collisions. + let mut asset_groups: Vec> = Vec::new(); + let mut group_indices: HashMap> = HashMap::new(); + for asset in assets { + let hash = asset.dispatch_equivalence_hash(); + + // Only groups with the same hash can possibly match. + let candidate_groups = group_indices.get(&hash); + + // Find a group whose representative is actually equivalent. + let matching_group = candidate_groups + .into_iter() + .flatten() + .find(|&&group_index| asset_groups[group_index][0].is_dispatch_equivalent(asset)); + + if let Some(group_index) = matching_group { + asset_groups[*group_index].push(asset); + } else { + let group_index = asset_groups.len(); + asset_groups.push(vec![asset]); + group_indices.entry(hash).or_default().push(group_index); + } + } + + asset_groups +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::asset::{Asset, AssetCapacity}; + use crate::fixture::{asset, process}; + use crate::process::Process; + use crate::units::{Capacity, MoneyPerActivity}; + use rstest::rstest; + use std::sync::Arc; + + #[test] + fn groups_no_assets() { + assert_eq!( + group_dispatch_equivalent_assets(std::iter::empty()), + [] as [std::vec::Vec<&AssetRef>; 0] + ); + } + + #[rstest] + fn groups_equivalent_assets(asset: Asset) { + let mut equivalent = asset.clone(); + equivalent.set_capacity(AssetCapacity::single(Capacity(3.0))); + let assets = [AssetRef::from(asset), AssetRef::from(equivalent)]; + + let groups = group_dispatch_equivalent_assets(assets.iter()); + + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].len(), 2); + } + + #[rstest] + fn groups_non_equivalent_assets_are_separate(asset: Asset, mut process: Process) { + Arc::make_mut(process.parameters.get_mut(&("GBR".into(), 2015)).unwrap()) + .variable_operating_cost = MoneyPerActivity(1.0); + let different = Asset::new_ready( + "agent1".into(), + Arc::new(process), + "GBR".into(), + AssetCapacity::single(Capacity(2.0)), + 2015, + ) + .unwrap(); + let assets = [AssetRef::from(asset), AssetRef::from(different)]; + + let groups = group_dispatch_equivalent_assets(assets.iter()); + + assert_eq!(groups.len(), 2); + assert_eq!(groups[0].len(), 1); + assert_eq!(groups[1].len(), 1); + } +} diff --git a/src/simulation/investment/competitive/constraints.rs b/src/simulation/investment/competitive/constraints.rs index 5f293cf01..0d9c700ec 100644 --- a/src/simulation/investment/competitive/constraints.rs +++ b/src/simulation/investment/competitive/constraints.rs @@ -3,8 +3,8 @@ use super::optimisation::Variable; use crate::asset::AssetRef; use crate::commodity::Commodity; use crate::model::Model; +use crate::optimisation::group_dispatch_equivalent_assets; use crate::simulation::investment::DemandMap; -use crate::simulation::optimisation::group_dispatch_equivalent_assets; use crate::time_slice::{Season, TimeSliceID, TimeSliceInfo, TimeSliceSelection}; use crate::units::{MoneyPerCapacityPerYear, Year}; use highs::RowProblem as Problem; diff --git a/src/simulation/investment/competitive/optimisation.rs b/src/simulation/investment/competitive/optimisation.rs index d74378211..e60c5d487 100644 --- a/src/simulation/investment/competitive/optimisation.rs +++ b/src/simulation/investment/competitive/optimisation.rs @@ -8,12 +8,12 @@ use super::record_asset_selection; use crate::asset::AssetRef; use crate::commodity::Commodity; use crate::model::Model; +use crate::optimisation::ModelError; +use crate::optimisation::apply_highs_options_from_toml; +use crate::optimisation::group_dispatch_equivalent_assets; +use crate::optimisation::solve_optimal; use crate::simulation::activity::ActivityMap; use crate::simulation::investment::DemandMap; -use crate::simulation::optimisation::ModelError; -use crate::simulation::optimisation::apply_highs_options_from_toml; -use crate::simulation::optimisation::group_dispatch_equivalent_assets; -use crate::simulation::optimisation::solve_optimal; use crate::time_slice::{TimeSliceID, TimeSliceInfo}; use crate::units::{Activity, Flow}; use anyhow::{Context, Result}; diff --git a/src/simulation/investment/portfolio/optimisation.rs b/src/simulation/investment/portfolio/optimisation.rs index f307e9993..7c3a3113d 100644 --- a/src/simulation/investment/portfolio/optimisation.rs +++ b/src/simulation/investment/portfolio/optimisation.rs @@ -2,11 +2,11 @@ use crate::asset::AssetRef; use crate::commodity::Commodity; use crate::model::Model; +use crate::optimisation::ModelError; +use crate::optimisation::{apply_highs_options_from_toml, solve_optimal}; use crate::process::ProcessID; use crate::simulation::activity::ActivityMap; use crate::simulation::investment::DemandMap; -use crate::simulation::optimisation::ModelError; -use crate::simulation::optimisation::{apply_highs_options_from_toml, solve_optimal}; use crate::simulation::prices::Prices; use crate::time_slice::{TimeSliceID, TimeSliceInfo}; use crate::units::{Activity, Capacity}; diff --git a/src/simulation/optimisation.rs b/src/simulation/optimisation.rs index 38d3d56db..b3b8bd61d 100644 --- a/src/simulation/optimisation.rs +++ b/src/simulation/optimisation.rs @@ -5,6 +5,7 @@ use crate::asset::{Asset, AssetRef}; use crate::commodity::CommodityID; use crate::input::format_items_with_cap; use crate::model::Model; +use crate::optimisation::{ModelError, apply_highs_options_from_toml, solve_optimal}; use crate::output::DataWriter; use crate::region::RegionID; use crate::simulation::PriceMap; @@ -12,16 +13,14 @@ use crate::simulation::activity::{ActivityMap, FlowMap, create_flow_map}; use crate::simulation::investment::AllDemandMap; use crate::time_slice::{TimeSliceID, TimeSliceInfo, TimeSliceSelection}; use crate::units::{Activity, Flow, Money, MoneyPerActivity, MoneyPerFlow}; -use anyhow::{Context, Result, anyhow, bail}; +use anyhow::{Context, Result, bail}; use highs::{HighsModelStatus, RowProblem as Problem, Sense}; use indexmap::{IndexMap, IndexSet}; use itertools::{chain, iproduct}; use log::warn; -use std::error::Error; use std::ops::Range; mod constraints; -pub(crate) use constraints::group_dispatch_equivalent_assets; use constraints::{ConstraintKeys, add_model_constraints}; /// A decision variable in the optimisation @@ -280,82 +279,6 @@ impl Solution<'_> { } } -/// Defines the possible errors that can occur when running the solver -#[derive(Debug, derive_more::Display, derive_more::From)] -pub enum ModelError { - /// An optimal solution could not be found - #[display("Could not find optimal result: {_0:?}")] - NonOptimal(HighsModelStatus), - /// Another error occurred - #[display("{_0}")] - Other(anyhow::Error), -} - -impl ModelError { - /// Convert this error into an [`anyhow::Error`] - pub fn into_anyhow(self) -> anyhow::Error { - match self { - ModelError::NonOptimal(status) => anyhow!("Could not find optimal result: {status:?}"), - ModelError::Other(error) => error, - } - } -} - -impl Error for ModelError { - fn source(&self) -> Option<&(dyn Error + 'static)> { - match self { - ModelError::NonOptimal(_) => None, - ModelError::Other(error) => Some(error.as_ref()), - } - } -} - -/// Apply the specified HiGHS options from a [`toml::Table`] -pub fn apply_highs_options_from_toml( - model: &mut highs::Model, - options: &toml::Table, -) -> Result<()> { - // Attempt to set an option, returning an error if it fails - macro_rules! try_set_opt { - ($option:expr, $value:expr) => {{ - model - .try_set_option($option.as_str(), $value) - .map_err(|_| anyhow!("Invalid option name or value"))?; - - Ok(()) - }}; - } - - // Iterate through options, applying each in turn to the HiGHS model - for (option, value) in options { - match value { - toml::Value::String(value) => try_set_opt!(option, value.as_str()), - toml::Value::Integer(value) => match i32::try_from(*value) { - Ok(value) => try_set_opt!(option, value), - Err(_) => Err(anyhow!("Value out of range")), - }, - toml::Value::Float(value) => try_set_opt!(option, *value), - toml::Value::Boolean(value) => try_set_opt!(option, *value), - _ => Err(anyhow!("HiGHS options cannot have this type")), - } - .with_context(|| format!("Failed to set option \"{option}\" to value \"{value}\""))?; - } - - Ok(()) -} - -/// Try to solve the model, returning an error if the model is incoherent or result is non-optimal -pub fn solve_optimal(model: highs::Model) -> Result { - let solved = model - .try_solve() - .map_err(|status| anyhow!("Incoherent model: {status:?}"))?; - - match solved.status() { - HighsModelStatus::Optimal => Ok(solved), - status => Err(status.into()), - } -} - /// Filter prices data to only include prices for markets not being balanced /// /// Markets being balanced (i.e. with commodity balance constraints) will have prices calculated diff --git a/src/simulation/optimisation/constraints.rs b/src/simulation/optimisation/constraints.rs index 912f54af3..632892c4a 100644 --- a/src/simulation/optimisation/constraints.rs +++ b/src/simulation/optimisation/constraints.rs @@ -3,6 +3,7 @@ use super::VariableMap; use crate::asset::{AssetIterator, AssetRef}; use crate::commodity::{BalanceType, CommodityID, CommodityType}; use crate::model::Model; +use crate::optimisation::group_dispatch_equivalent_assets; use crate::process::FlowDirection; use crate::region::RegionID; use crate::simulation::investment::AllDemandMap; @@ -10,7 +11,6 @@ use crate::time_slice::{Season, TimeSliceInfo, TimeSliceSelection}; use crate::units::{Flow, MoneyPerCapacityPerYear, UnitType, Year}; use highs::RowProblem as Problem; use indexmap::IndexMap; -use std::collections::HashMap; /// Corresponding variables for a constraint along with the row offset in the solution pub struct KeysWithOffset { @@ -503,47 +503,6 @@ where ActivityKeys { offset, keys } } -/// Groups assets that have equivalent dispatch properties. -/// -/// Assets are first bucketed by `dispatch_equivalence_hash()` to avoid unnecessary pairwise -/// comparisons. Within each hash bucket, assets are compared using `is_dispatch_equivalent()`, -/// which is the authoritative check for equivalence. This also handles hash collisions correctly. -/// -/// The caller must ensure that `assets` contains only assets eligible for equal-utilisation -/// constraints. -pub fn group_dispatch_equivalent_assets<'a, I>(assets: I) -> Vec> -where - I: Iterator, -{ - // Group assets by comparing each one with the first asset in each group. The hash index - // avoids comparing an asset with groups that cannot contain an equivalent asset, while the - // exact comparison preserves correctness in the event of hash collisions. - let mut asset_groups: Vec> = Vec::new(); - let mut group_indices: HashMap> = HashMap::new(); - for asset in assets { - let hash = asset.dispatch_equivalence_hash(); - - // Only groups with the same hash can possibly match. - let candidate_groups = group_indices.get(&hash); - - // Find a group whose representative is actually equivalent. - let matching_group = candidate_groups - .into_iter() - .flatten() - .find(|&&group_index| asset_groups[group_index][0].is_dispatch_equivalent(asset)); - - if let Some(group_index) = matching_group { - asset_groups[*group_index].push(asset); - } else { - let group_index = asset_groups.len(); - asset_groups.push(vec![asset]); - group_indices.entry(hash).or_default().push(group_index); - } - } - - asset_groups -} - /// Add constraints requiring dispatch-equivalent assets to have equal utilisation in each time /// slice. /// @@ -592,13 +551,12 @@ fn add_equal_utilisation_constraints<'a, I>( #[cfg(test)] mod tests { use super::*; - use crate::asset::{Asset, AssetCapacity}; use crate::commodity::Commodity; use crate::fixture::{asset, process, process_flows_map, svd_commodity}; use crate::process::Process; use crate::process::{FlowType, ProcessFlow}; use crate::time_slice::TimeSliceSelection; - use crate::units::{Capacity, FlowPerActivity, MoneyPerActivity, MoneyPerFlow}; + use crate::units::{FlowPerActivity, MoneyPerFlow}; use indexmap::indexmap; use rstest::rstest; use std::sync::Arc; @@ -638,45 +596,4 @@ mod tests { ); assert_eq!(result, Flow(expected)); } - - #[test] - fn groups_no_assets() { - assert_eq!( - group_dispatch_equivalent_assets(std::iter::empty()), - [] as [std::vec::Vec<&AssetRef>; 0] - ); - } - - #[rstest] - fn groups_equivalent_assets(asset: Asset) { - let mut equivalent = asset.clone(); - equivalent.set_capacity(AssetCapacity::single(Capacity(3.0))); - let assets = [AssetRef::from(asset), AssetRef::from(equivalent)]; - - let groups = group_dispatch_equivalent_assets(assets.iter()); - - assert_eq!(groups.len(), 1); - assert_eq!(groups[0].len(), 2); - } - - #[rstest] - fn groups_non_equivalent_assets_are_separate(asset: Asset, mut process: Process) { - Arc::make_mut(process.parameters.get_mut(&("GBR".into(), 2015)).unwrap()) - .variable_operating_cost = MoneyPerActivity(1.0); - let different = Asset::new_ready( - "agent1".into(), - Arc::new(process), - "GBR".into(), - AssetCapacity::single(Capacity(2.0)), - 2015, - ) - .unwrap(); - let assets = [AssetRef::from(asset), AssetRef::from(different)]; - - let groups = group_dispatch_equivalent_assets(assets.iter()); - - assert_eq!(groups.len(), 2); - assert_eq!(groups[0].len(), 1); - assert_eq!(groups[1].len(), 1); - } } From ccfa311a9e574fbcb2004ff834d7035b9dfde4dd Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Wed, 11 Nov 2026 15:07:41 +0000 Subject: [PATCH 02/11] Introduce FlowEstimator seam for dispatch in the agent loop Move the partial dispatch runs out of the shared investment and market code into a PartialDispatch implementation, and remove shadow prices from Prices. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- benches/assets.rs | 1 + src/simulation.rs | 113 ++++++++++++++++++++++++++++++++--- src/simulation/investment.rs | 99 ++++++++++++++++++++---------- src/simulation/market.rs | 74 +++++++++++------------ src/simulation/prices.rs | 20 ++++--- 5 files changed, 220 insertions(+), 87 deletions(-) diff --git a/benches/assets.rs b/benches/assets.rs index 417b93eeb..7b64ef823 100644 --- a/benches/assets.rs +++ b/benches/assets.rs @@ -115,6 +115,7 @@ fn calculate_seed_prices( BASE_YEAR, ) .expect("Failed to calculate prices") + .0 } /// Build `n` synthetic candidate processes, based on cycling through `templates`, so that we can diff --git a/src/simulation.rs b/src/simulation.rs index 3900c75a7..9785ec127 100644 --- a/src/simulation.rs +++ b/src/simulation.rs @@ -1,15 +1,21 @@ //! Functionality for running the MUSE2 simulation across milestone years. use crate::asset::{Asset, AssetPool, AssetRef}; +use crate::commodity::CommodityID; use crate::model::Model; use crate::output::DataWriter; use crate::process::ProcessMap; +use crate::region::RegionID; +use crate::simulation::investment::Selection; +use crate::simulation::market::MarketSet; use crate::simulation::prices::{Prices, calculate_prices_from_solutions}; use crate::timeit::DispatchTimer; use crate::units::Capacity; use anyhow::{Context, Result}; use context_manager; -use log::info; +use itertools::Itertools; +use log::{debug, info}; use std::path::Path; +use std::slice; use std::sync::Arc; pub mod activity; @@ -17,7 +23,9 @@ use activity::FlowMap; pub mod optimisation; use optimisation::DispatchRun; pub mod investment; -use investment::{AgentRunMode, collect_preset_demands_for_year, run_agents_for_year}; +use investment::{ + AgentRunMode, AllDemandMap, FlowEstimator, collect_preset_demands_for_year, run_agents_for_year, +}; pub mod market; pub mod prices; pub use prices::PriceMap; @@ -59,7 +67,7 @@ pub fn run(model: &Model, output_path: &Path, debug_model: bool) -> Result<()> { // Run dispatch optimisation info!("Running dispatch optimisation..."); // There are no previous prices in the base year, so cyclic markets are seeded with zero - let (mut prices, flow_map) = run_dispatch_for_year( + let (mut prices, mut shadow_prices, flow_map) = run_dispatch_for_year( model, &asset_pool, &candidates, @@ -93,12 +101,14 @@ pub fn run(model: &Model, output_path: &Path, debug_model: bool) -> Result<()> { // Perform agent investment info!("Running agent investment..."); // NB: activities from asset selection are not currently used + let mut flow_estimator = PartialDispatch::new(model, year, &shadow_prices); let selected_assets = run_agents_for_year( model, year, &existing_assets, AgentRunMode::Invest, &prices, + &mut flow_estimator, &mut writer, ) .context("Agent investment failed")? @@ -106,7 +116,7 @@ pub fn run(model: &Model, output_path: &Path, debug_model: bool) -> Result<()> { // Run dispatch optimisation to get updated prices for the next iteration info!("Running dispatch optimisation..."); - let (new_prices, ..) = run_dispatch_for_year( + let (new_prices, new_shadow_prices, ..) = run_dispatch_for_year( model, &selected_assets, &candidates, @@ -124,6 +134,7 @@ pub fn run(model: &Model, output_path: &Path, debug_model: bool) -> Result<()> { // Update prices for the next iteration prices = new_prices; + shadow_prices = new_shadow_prices; // Clear writer context writer.clear_debug_context(); @@ -167,7 +178,7 @@ pub fn run(model: &Model, output_path: &Path, debug_model: bool) -> Result<()> { // Run dispatch optimisation info!("Running final dispatch optimisation for year {year}..."); - let (new_prices, flow_map) = + let (new_prices, new_shadow_prices, flow_map) = run_dispatch_for_year(model, &asset_pool, &candidates, &prices, year, &mut writer)?; // Write results of dispatch optimisation to file @@ -176,6 +187,7 @@ pub fn run(model: &Model, output_path: &Path, debug_model: bool) -> Result<()> { // Prices for the next year prices = new_prices; + shadow_prices = new_shadow_prices; } writer.flush()?; @@ -189,6 +201,9 @@ pub fn run(model: &Model, output_path: &Path, debug_model: bool) -> Result<()> { /// from a run with both existing and candidate assets. If either flows or prices cannot be /// calculated, empty maps are returned. /// +/// Returns the prices, the shadow prices from the run with candidates, and the flows of the +/// existing assets. +/// /// `previous_prices` (from the previous year or ironing-out iteration) are used to seed price /// calculations for cyclic markets. /// @@ -201,7 +216,7 @@ fn run_dispatch_for_year( previous_prices: &Prices, year: u32, writer: &mut DataWriter, -) -> Result<(Prices, FlowMap)> { +) -> Result<(Prices, PriceMap, FlowMap)> { // Sanity checks debug_assert!(assets.iter().all(|asset| !asset.is_candidate())); debug_assert!(candidates.iter().all(|asset| asset.is_candidate())); @@ -234,11 +249,11 @@ fn run_dispatch_for_year( // Calculate prices from the appropriate solution(s). If there were candidates, use the solution // that includes them; otherwise use the existing solution alone. If there were no assets at // all, return empty prices. - let prices = match ( + let (prices, shadow_prices) = match ( solution_existing.as_ref(), solution_with_candidates.as_ref(), ) { - (None, None) => Prices::default(), + (None, None) => (Prices::default(), PriceMap::default()), (Some(existing), None) => calculate_prices_from_solutions( model, existing, @@ -265,7 +280,87 @@ fn run_dispatch_for_year( )?, }; - Ok((prices, flow_map)) + Ok((prices, shadow_prices, flow_map)) +} + +/// Estimates the flows of assets selected by agents using partial dispatch runs. +/// +/// Each run dispatches only the assets selected so far and balances only the markets which agents +/// have already settled. +struct PartialDispatch<'a> { + /// The model + model: &'a Model, + /// The milestone year + year: u32, + /// Preset demands for the year + preset_demands: AllDemandMap, + /// Shadow prices used for markets without a balance constraint + shadow_prices: &'a PriceMap, +} + +impl<'a> PartialDispatch<'a> { + fn new(model: &'a Model, year: u32, shadow_prices: &'a PriceMap) -> Self { + Self { + model, + year, + preset_demands: collect_preset_demands_for_year(&model.commodities, year), + shadow_prices, + } + } +} + +impl FlowEstimator for PartialDispatch<'_> { + fn flows_after_market_set( + &mut self, + market_set: &MarketSet, + settled_markets: &[(CommodityID, RegionID)], + all_selected: &Selection, + _new_selection: &Selection, + writer: &mut DataWriter, + ) -> Result { + debug!("Running post-investment dispatch for '{market_set}'"); + + // As upstream markets by definition will not yet have producers, we explicitly set + // their prices using external values so that they don't appear free + let solution = DispatchRun::new( + self.model, + &all_selected.assets, + self.year, + &self.preset_demands, + ) + .without_commodity_constraints() + .with_market_balance_subset(settled_markets) + .with_input_prices(self.shadow_prices) + .run(&format!("post {market_set} investment"), writer)?; + + Ok(solution.create_flow_map()) + } + + fn flows_after_cycle_market( + &mut self, + cycle: &[(CommodityID, RegionID)], + market: &(CommodityID, RegionID), + pass: u32, + net_demand: &AllDemandMap, + selection: &Selection, + writer: &mut DataWriter, + ) -> Result { + let markets_str = cycle.iter().map(|(c, r)| format!("{c}|{r}")).join(", "); + let (commodity_id, region_id) = market; + let description = + format!("cycle ({markets_str}) post {commodity_id}|{region_id} investment pass {pass}"); + + debug!("Running {description}"); + let solution = DispatchRun::new(self.model, &selection.assets, self.year, net_demand) + .without_commodity_constraints() + .with_market_balance_subset(slice::from_ref(market)) + .with_input_prices(self.shadow_prices) + .run(&description, writer) + .with_context(|| format!("Dispatch failed for cycle ({markets_str})"))?; + debug!("Completed {description}"); + + Ok(solution.create_flow_map()) + } } /// Create candidate assets for all potential processes in a specified year diff --git a/src/simulation/investment.rs b/src/simulation/investment.rs index 5c2e9706f..914e4d7b1 100644 --- a/src/simulation/investment.rs +++ b/src/simulation/investment.rs @@ -1,12 +1,12 @@ //! Code for performing agent investment. use super::activity::{ActivityMap, FlowMap, create_flow_map}; -use super::optimisation::DispatchRun; use crate::asset::{Asset, AssetRef}; use crate::commodity::{Commodity, CommodityID, CommodityMap}; use crate::model::Model; use crate::output::DataWriter; use crate::process::Process; use crate::region::RegionID; +use crate::simulation::market::MarketSet; use crate::simulation::prices::Prices; use crate::time_slice::{TimeSliceInfo, TimeSliceLevel, TimeSliceSelection}; use crate::timeit::InvestmentTimer; @@ -111,6 +111,41 @@ pub enum AgentRunMode { Invest, } +/// Determines the commodity flows of assets selected by agents. +/// +/// These flows are used to update the net demand seen by markets later in the investment order. +/// How they are obtained depends on the simulation protocol. +pub trait FlowEstimator { + /// Called after agents have been run for a top-level market set (only if something was + /// selected). + /// + /// `settled_markets` contains all markets processed so far, including those in `market_set`. + /// `all_selected` is the cumulative selection including `new_selection`. + fn flows_after_market_set( + &mut self, + market_set: &MarketSet, + settled_markets: &[(CommodityID, RegionID)], + all_selected: &Selection, + new_selection: &Selection, + writer: &mut DataWriter, + ) -> Result; + + /// Called after agents have been run for one market inside a cycle (only if something was + /// selected). + /// + /// `pass` is the selection pass within the cycle (1 or 2) and `net_demand` is the net demand + /// for that pass. + fn flows_after_cycle_market( + &mut self, + cycle: &[(CommodityID, RegionID)], + market: &(CommodityID, RegionID), + pass: u32, + net_demand: &AllDemandMap, + selection: &Selection, + writer: &mut DataWriter, + ) -> Result; +} + /// The assets and processes available to an agent to meet its share of demand for one market #[derive(Default)] pub struct AssetOptions { @@ -133,7 +168,8 @@ pub type AllDemandMap = IndexMap<(CommodityID, RegionID, TimeSliceSelection), Fl /// * `year` - Milestone year being processed /// * `assets` - The asset pool (commissioned and otherwise) /// * `mode` - Whether agents dispatch committed assets or make investment decisions -/// * `prices` - Commodity prices calculated in the previous full system dispatch +/// * `prices` - Commodity prices calculated in the previous dispatch +/// * `flow_estimator` - Determines the flows of selected assets, to propagate demand upstream /// * `writer` - Data writer /// /// # Returns @@ -147,11 +183,11 @@ pub fn run_agents_for_year( assets: &[AssetRef], mode: AgentRunMode, prices: &Prices, + flow_estimator: &mut dyn FlowEstimator, writer: &mut DataWriter, ) -> Result { // Initialise net demand map - let preset_demands = collect_preset_demands_for_year(&model.commodities, year); - let mut net_demand = preset_demands.clone(); + let mut net_demand = collect_preset_demands_for_year(&model.commodities, year); // Keep track of all the assets selected // This includes Commissioned assets that are selected for retention, and new Ready assets @@ -163,23 +199,30 @@ pub fn run_agents_for_year( investment_order.iter().join(" -> ") ); - // Keep track of the markets that have been seen so far. This will be used to apply - // balance constraints in the dispatch optimisation - we only apply balance constraints for - // markets that have been seen so far. - let mut seen_markets = Vec::new(); + // Keep track of the markets that have been settled so far, i.e. those for which agents have + // already been run. Flow estimators may use this to restrict their calculations. + let mut settled_markets = Vec::new(); // Iterate over market sets in the investment order for this year for market_set in investment_order { // Select assets for this market set - let selection = - market_set.run_agents(model, year, mode, &net_demand, assets, prices, writer)?; - - // Update our list of seen markets + let selection = market_set.run_agents( + model, + year, + mode, + &net_demand, + assets, + prices, + flow_estimator, + writer, + )?; + + // Update our list of settled markets for market in market_set.iter_markets() { - seen_markets.push(market.clone()); + settled_markets.push(market.clone()); } - // If no assets have been selected, skip dispatch optimisation + // If no assets have been selected, skip flow estimation // **TODO**: this probably means there's no demand for the market, which we could // presumably preempt if selection.is_empty() { @@ -188,28 +231,22 @@ pub fn run_agents_for_year( } // Add the selected assets to the list of all selected assets - let selected_assets = selection.assets.clone(); - all_selected.extend(selection); + let new_selection = selection; + all_selected.extend(new_selection.clone()); - // Perform dispatch optimisation with assets that have been selected so far + // Determine the flows of the assets selected so far // **TODO**: presumably we only need to do this for selected_assets, as assets added in // previous iterations should not change - debug!("Running post-investment dispatch for '{market_set}'"); - - // As upstream markets by definition will not yet have producers, we explicitly set - // their prices using external values so that they don't appear free - let solution = DispatchRun::new(model, &all_selected.assets, year, &preset_demands) - .without_commodity_constraints() - .with_market_balance_subset(&seen_markets) - .with_input_prices(&prices.shadow) - .run(&format!("post {market_set} investment"), writer)?; + let flows = flow_estimator.flows_after_market_set( + market_set, + &settled_markets, + &all_selected, + &new_selection, + writer, + )?; // Update demand map with flows from newly added assets - update_net_demand_map( - &mut net_demand, - &solution.create_flow_map(), - &selected_assets, - ); + update_net_demand_map(&mut net_demand, &flows, &new_selection.assets); } Ok(all_selected) diff --git a/src/simulation/market.rs b/src/simulation/market.rs index 8a5e6fb17..74f5fc6aa 100644 --- a/src/simulation/market.rs +++ b/src/simulation/market.rs @@ -1,7 +1,6 @@ //! Code for creating sets of markets. use super::activity::FlowMap; use super::investment::Selection; -use super::optimisation::DispatchRun; use crate::agent::{Agent, AgentType}; use crate::asset::{AssetIterator, AssetRef}; use crate::commodity::{Commodity, CommodityID}; @@ -10,7 +9,7 @@ use crate::output::DataWriter; use crate::process::{Process, ProcessID}; use crate::region::RegionID; use crate::simulation::investment::{ - AgentRunMode, AllDemandMap, AssetOptions, DemandMap, update_net_demand_map, + AgentRunMode, AllDemandMap, AssetOptions, DemandMap, FlowEstimator, update_net_demand_map, }; use crate::simulation::investment::{competitive, portfolio}; use crate::simulation::prices::Prices; @@ -63,6 +62,7 @@ impl MarketSet { /// * `demand` – Net demand profiles available to all markets before selection. /// * `assets` – Assets available to agents in this market. /// * `prices` – Commodity price assumptions to use when valuing investments. + /// * `flow_estimator` – Determines the flows of selected assets, to propagate demand upstream. /// * `writer` – Data sink used to log optimisation artefacts. /// /// # Returns @@ -77,6 +77,7 @@ impl MarketSet { demand: &AllDemandMap, assets: &[AssetRef], prices: &Prices, + flow_estimator: &mut dyn FlowEstimator, writer: &mut DataWriter, ) -> Result { match self { @@ -112,6 +113,7 @@ impl MarketSet { demand, assets, prices, + flow_estimator, writer, ) .with_context(|| { @@ -127,8 +129,16 @@ impl MarketSet { debug!("Starting agent runs for layer '{self}'"); let mut layer_selection = Selection::default(); for investment_set in investment_sets { - let selection = investment_set - .run_agents(model, year, mode, demand, assets, prices, writer)?; + let selection = investment_set.run_agents( + model, + year, + mode, + demand, + assets, + prices, + flow_estimator, + writer, + )?; layer_selection.extend(selection); } debug!("Completed agent runs for layer '{self}'"); @@ -284,8 +294,8 @@ pub fn run_agents_for_single_market( /// market earlier in the investment order. In `Invest` mode, assets for these pairs are selected /// in the first pass and retained, then the remaining assets are selected in a second pass that /// excludes those processes. In `Dispatch` mode, there is no reselection, so all assets from the -/// first pass are returned. A dispatch optimisation after each market updates demand for subsequent -/// markets. +/// first pass are returned. After each market, the flows of the selected assets are estimated using +/// `flow_estimator` and used to update demand for subsequent markets. #[allow(clippy::too_many_arguments, clippy::too_many_lines)] pub fn run_agents_for_cycle( model: &Model, @@ -296,10 +306,9 @@ pub fn run_agents_for_cycle( demand: &AllDemandMap, assets: &[AssetRef], prices: &Prices, + flow_estimator: &mut dyn FlowEstimator, writer: &mut DataWriter, ) -> Result { - // Precompute a joined string for logging - let markets_str = markets.iter().map(|(c, r)| format!("{c}|{r}")).join(", "); let is_feedback_asset = |asset: &AssetRef| { feedback_processes.contains(&(asset.process_id().clone(), asset.region_id().clone())) }; @@ -341,21 +350,17 @@ pub fn run_agents_for_cycle( retained.retain(|asset| mode == AgentRunMode::Dispatch || is_feedback_asset(asset)); retained_first_pass.extend(retained); - // Run dispatch - debug!("Running cycle ({markets_str}) post {commodity_id}|{region_id} investment pass 1"); - let solution = DispatchRun::new(model, &selection.assets, year, &net_demand) - .without_commodity_constraints() - .with_market_balance_subset(std::slice::from_ref(market)) - .with_input_prices(&prices.shadow) - .run( - &format!("cycle ({markets_str}) post {commodity_id}|{region_id} investment pass 1"), - writer, - ) - .with_context(|| format!("Dispatch failed for cycle ({markets_str})"))?; - debug!("Completed cycle ({markets_str}) post {commodity_id}|{region_id} investment pass 1"); + // Determine the flows of the newly selected assets + let flows = flow_estimator.flows_after_cycle_market( + markets, + market, + 1, + &net_demand, + &selection, + writer, + )?; // Update demand map with flows from newly selected assets - let flows = solution.create_flow_map(); for ((asset, commodity_id, time_slice), flow) in &flows { if mode == AgentRunMode::Invest && is_feedback_asset(asset) { excluded_flows @@ -405,25 +410,18 @@ pub fn run_agents_for_cycle( continue; } - // Run dispatch - debug!("Running cycle ({markets_str}) post {commodity_id}|{region_id} investment pass 2"); - let solution = DispatchRun::new(model, &selection.assets, year, &net_demand) - .without_commodity_constraints() - .with_market_balance_subset(std::slice::from_ref(market)) - .with_input_prices(&prices.shadow) - .run( - &format!("cycle ({markets_str}) post {commodity_id}|{region_id} investment pass 2"), - writer, - ) - .with_context(|| format!("Dispatch failed for cycle ({markets_str})"))?; - debug!("Completed cycle ({markets_str}) post {commodity_id}|{region_id} investment pass 2"); + // Determine the flows of the newly selected assets + let flows = flow_estimator.flows_after_cycle_market( + markets, + market, + 2, + &net_demand, + &selection, + writer, + )?; // Update demand map with flows from newly selected assets - update_net_demand_map( - &mut net_demand, - &solution.create_flow_map(), - &selection.assets, - ); + update_net_demand_map(&mut net_demand, &flows, &selection.assets); second_pass.extend(selection); } diff --git a/src/simulation/prices.rs b/src/simulation/prices.rs index 0bed7fbde..f48e024cd 100644 --- a/src/simulation/prices.rs +++ b/src/simulation/prices.rs @@ -53,13 +53,11 @@ impl WeightedAverageAccumulator { } } -/// Sets of prices calculated from dispatch results +/// Sets of prices calculated from the results of a simulation protocol #[derive(Default)] pub struct Prices { /// Commodity market prices calculated according to user-specified strategies pub market: PriceMap, - /// Commodity shadow prices - pub shadow: PriceMap, /// Commodity fallback prices calculated according to the model's `fallback_pricing_strategy` pub fallback: PriceMap, } @@ -69,6 +67,10 @@ pub struct Prices { /// This extracts the activities of existing assets from `solution_without_candidates` and the /// shadow prices from `solution_with_candidates`, then calls [`calculate_prices`]. /// +/// # Returns +/// +/// The calculated prices, along with the shadow prices from `solution_with_candidates`. +/// /// # Arguments /// /// * `model` - The model @@ -85,8 +87,8 @@ pub fn calculate_prices_from_solutions( candidates: &[AssetRef], previous_prices: &Prices, year: u32, -) -> Result { - let mut prices = calculate_prices( +) -> Result<(Prices, PriceMap)> { + let prices = calculate_prices( model, &solution_without_candidates.activity_map_for_existing(), candidates, @@ -96,8 +98,9 @@ pub fn calculate_prices_from_solutions( // Shadow prices come from the solution _with_ candidates, so that we can get them for // commodities not produced/consumed by any existing assets. - prices.shadow = PriceMap::from_iter(solution_with_candidates.iter_commodity_balance_duals()); - Ok(prices) + let shadow_prices = + PriceMap::from_iter(solution_with_candidates.iter_commodity_balance_duals()); + Ok((prices, shadow_prices)) } /// Calculate commodity prices. @@ -116,7 +119,7 @@ pub fn calculate_prices_from_solutions( /// /// # Returns /// -/// Market prices and fallback prices. The `shadow` prices are left empty. +/// Market prices and fallback prices. pub fn calculate_prices( model: &Model, activity: &ActivityMap, @@ -163,7 +166,6 @@ pub fn calculate_prices( Ok(Prices { market: market_prices, - shadow: PriceMap::default(), fallback: fallback_prices, }) } From 2ba9046b4390a1d286925de80e8875b4a74d3f7b Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Wed, 11 Nov 2026 15:12:31 +0000 Subject: [PATCH 03/11] Create simulation::system_dispatch submodule Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- benches/assets.rs | 7 +- src/output.rs | 2 +- src/simulation.rs | 345 +-------------- src/simulation/prices.rs | 42 -- src/simulation/system_dispatch.rs | 395 ++++++++++++++++++ .../{ => system_dispatch}/optimisation.rs | 0 .../optimisation/constraints.rs | 0 7 files changed, 405 insertions(+), 386 deletions(-) create mode 100644 src/simulation/system_dispatch.rs rename src/simulation/{ => system_dispatch}/optimisation.rs (100%) rename src/simulation/{ => system_dispatch}/optimisation/constraints.rs (100%) diff --git a/benches/assets.rs b/benches/assets.rs index 7b64ef823..b12a277ba 100644 --- a/benches/assets.rs +++ b/benches/assets.rs @@ -14,8 +14,9 @@ use muse2::simulation::investment::competitive::{get_candidate_assets, select_be use muse2::simulation::market::{ collect_agent_limits, get_demand_portion_for_market, get_responsible_agents, }; -use muse2::simulation::optimisation::DispatchRun; -use muse2::simulation::prices::{Prices, calculate_prices_from_solutions}; +use muse2::simulation::prices::Prices; +use muse2::simulation::system_dispatch::calculate_prices_from_solutions; +use muse2::simulation::system_dispatch::optimisation::DispatchRun; use rayon::ThreadPoolBuilder; use std::hint::black_box; use std::sync::Arc; @@ -88,7 +89,7 @@ fn build_assets_for_investment_year( } /// Dispatch the base year (with and without the investment-year candidates) to obtain realistic -/// seed prices for investment appraisal, as per `simulation::run_dispatch_for_year` (this is the +/// seed prices for investment appraisal, as per `simulation::system_dispatch::run_dispatch_for_year` (this is the /// dispatch performed before any agent investment has taken place). fn calculate_seed_prices( model: &Model, diff --git a/src/output.rs b/src/output.rs index 3d3bc9aa5..cb07fe4cf 100644 --- a/src/output.rs +++ b/src/output.rs @@ -5,8 +5,8 @@ use crate::commodity::CommodityID; use crate::process::ProcessID; use crate::region::RegionID; use crate::simulation::activity::FlowMap; -use crate::simulation::optimisation::Solution; use crate::simulation::prices::PriceMap; +use crate::simulation::system_dispatch::optimisation::Solution; use crate::time_slice::TimeSliceID; use crate::units::{Activity, Capacity, Flow, Money, MoneyPerActivity, MoneyPerFlow}; use anyhow::{Context, Result, ensure}; diff --git a/src/simulation.rs b/src/simulation.rs index 9785ec127..2390fadfe 100644 --- a/src/simulation.rs +++ b/src/simulation.rs @@ -1,34 +1,19 @@ //! Functionality for running the MUSE2 simulation across milestone years. -use crate::asset::{Asset, AssetPool, AssetRef}; -use crate::commodity::CommodityID; +use crate::asset::{Asset, AssetRef}; use crate::model::Model; use crate::output::DataWriter; use crate::process::ProcessMap; -use crate::region::RegionID; -use crate::simulation::investment::Selection; -use crate::simulation::market::MarketSet; -use crate::simulation::prices::{Prices, calculate_prices_from_solutions}; -use crate::timeit::DispatchTimer; use crate::units::Capacity; -use anyhow::{Context, Result}; -use context_manager; -use itertools::Itertools; -use log::{debug, info}; +use anyhow::Result; use std::path::Path; -use std::slice; use std::sync::Arc; pub mod activity; -use activity::FlowMap; -pub mod optimisation; -use optimisation::DispatchRun; pub mod investment; -use investment::{ - AgentRunMode, AllDemandMap, FlowEstimator, collect_preset_demands_for_year, run_agents_for_year, -}; pub mod market; pub mod prices; pub use prices::PriceMap; +pub mod system_dispatch; /// Run the simulation. /// @@ -37,330 +22,10 @@ pub use prices::PriceMap; /// * `model` - The model to run /// * `output_path` - The folder to which output files will be written /// * `debug_model` - Whether to write additional information (e.g. duals) to output files -#[allow(clippy::too_many_lines)] pub fn run(model: &Model, output_path: &Path, debug_model: bool) -> Result<()> { let mut writer = DataWriter::create(output_path, &model.model_path, debug_model)?; - let mut user_assets = model.user_assets.clone(); - let mut asset_pool = AssetPool::new(); // active assets - - // Iterate over milestone years - let mut year_iter = model.iter_years().peekable(); - let year = year_iter.next().unwrap(); // Unwrap is safe: model must contain at least one milestone year - - info!("Milestone year: {year}"); - - // Commission assets for base year - let new_assets = asset_pool.commission_new(year, &mut user_assets); - - // Write assets to file - writer.write_assets(new_assets)?; - writer.write_asset_capacities(year, &asset_pool)?; - - // Gather candidates for the next year, if any - let next_year = year_iter.peek().copied(); - let mut candidates = candidate_assets_for_next_year( - &model.processes, - next_year, - model.parameters.candidate_asset_capacity, - ); - - // Run dispatch optimisation - info!("Running dispatch optimisation..."); - // There are no previous prices in the base year, so cyclic markets are seeded with zero - let (mut prices, mut shadow_prices, flow_map) = run_dispatch_for_year( - model, - &asset_pool, - &candidates, - &Prices::default(), - year, - &mut writer, - )?; - - // Write results of dispatch optimisation to file - writer.write_flows(year, &flow_map)?; - writer.write_prices(year, &prices.market)?; - - while let Some(year) = year_iter.next() { - info!("Milestone year: {year}"); - - // Decommission assets whose lifetime has passed - asset_pool.decommission_old(year); - - // Commission user-defined assets for this year - let mut new_assets = asset_pool.commission_new(year, &mut user_assets).to_vec(); - - // Take all the active assets as a list of existing assets - let existing_assets = asset_pool.take(); - - // Iterative loop to "iron out" prices via repeated investment and dispatch - let mut ironing_out_iter = 0; - let selected_assets: Vec = loop { - // Add context to the writer - writer.set_debug_context(format!("ironing out iteration {ironing_out_iter}")); - - // Perform agent investment - info!("Running agent investment..."); - // NB: activities from asset selection are not currently used - let mut flow_estimator = PartialDispatch::new(model, year, &shadow_prices); - let selected_assets = run_agents_for_year( - model, - year, - &existing_assets, - AgentRunMode::Invest, - &prices, - &mut flow_estimator, - &mut writer, - ) - .context("Agent investment failed")? - .assets; - - // Run dispatch optimisation to get updated prices for the next iteration - info!("Running dispatch optimisation..."); - let (new_prices, new_shadow_prices, ..) = run_dispatch_for_year( - model, - &selected_assets, - &candidates, - &prices, - year, - &mut writer, - )?; - - // Check if prices have converged using time slice-weighted averages - let prices_stable = prices.market.within_tolerance_weighted( - &new_prices.market, - model.parameters.price_tolerance, - &model.time_slice_info, - ); - - // Update prices for the next iteration - prices = new_prices; - shadow_prices = new_shadow_prices; - - // Clear writer context - writer.clear_debug_context(); - - // Break early if prices have converged - if prices_stable { - info!("Prices converged after {} iterations", ironing_out_iter + 1); - break selected_assets; - } - - // Break if max iterations reached - ironing_out_iter += 1; - if ironing_out_iter == model.parameters.max_ironing_out_iterations { - info!( - "Max ironing out iterations ({}) reached", - model.parameters.max_ironing_out_iterations - ); - break selected_assets; - } - }; - - // Add selected_assets to the active pool, receiving the newly commissioned ones - let newly_selected = asset_pool.extend(selected_assets); - new_assets.extend_from_slice(newly_selected); - - // Decommission unused assets - asset_pool.mothball_unretained(existing_assets, year); - asset_pool.decommission_mothballed(year, model.parameters.mothball_years); - - // Write newly commissioned assets - writer.write_assets(&new_assets)?; - writer.write_asset_capacities(year, &asset_pool)?; - - // Gather candidates for the next year, if any - let next_year = year_iter.peek().copied(); - candidates = candidate_assets_for_next_year( - &model.processes, - next_year, - model.parameters.candidate_asset_capacity, - ); - - // Run dispatch optimisation - info!("Running final dispatch optimisation for year {year}..."); - let (new_prices, new_shadow_prices, flow_map) = - run_dispatch_for_year(model, &asset_pool, &candidates, &prices, year, &mut writer)?; - - // Write results of dispatch optimisation to file - writer.write_flows(year, &flow_map)?; - writer.write_prices(year, &new_prices.market)?; - - // Prices for the next year - prices = new_prices; - shadow_prices = new_shadow_prices; - } - - writer.flush()?; - - Ok(()) -} - -/// Run dispatch to get flows and prices for a milestone year. -/// -/// Tries to get commodity flows from a dispatch run which just includes existing assets and prices -/// from a run with both existing and candidate assets. If either flows or prices cannot be -/// calculated, empty maps are returned. -/// -/// Returns the prices, the shadow prices from the run with candidates, and the flows of the -/// existing assets. -/// -/// `previous_prices` (from the previous year or ironing-out iteration) are used to seed price -/// calculations for cyclic markets. -/// -/// Fully mothballed assets or mothballed tranches thereof are excluded. -#[context_manager::wrap(DispatchTimer)] -fn run_dispatch_for_year( - model: &Model, - assets: &[AssetRef], - candidates: &[AssetRef], - previous_prices: &Prices, - year: u32, - writer: &mut DataWriter, -) -> Result<(Prices, PriceMap, FlowMap)> { - // Sanity checks - debug_assert!(assets.iter().all(|asset| !asset.is_candidate())); - debug_assert!(candidates.iter().all(|asset| asset.is_candidate())); - - let market_demands = collect_preset_demands_for_year(&model.commodities, year); - - // Run dispatch optimisation with existing assets only, if there are any. If not, then assume no - // flows (i.e. all are zero) - let (solution_existing, flow_map) = if assets.is_empty() { - (None, FlowMap::default()) - } else { - let solution = DispatchRun::new(model, assets, year, &market_demands) - .run("final without candidates", writer)?; - let flow_map = solution.create_flow_map(); - (Some(solution), flow_map) - }; - - // Perform a separate dispatch run with both existing assets and candidates, if there are any, - // to get shadow prices. If not, use the existing solution alone. - let solution_with_candidates = if candidates.is_empty() { - None - } else { - Some( - DispatchRun::new(model, assets, year, &market_demands) - .with_candidates(candidates) - .run("final with candidates", writer)?, - ) - }; - - // Calculate prices from the appropriate solution(s). If there were candidates, use the solution - // that includes them; otherwise use the existing solution alone. If there were no assets at - // all, return empty prices. - let (prices, shadow_prices) = match ( - solution_existing.as_ref(), - solution_with_candidates.as_ref(), - ) { - (None, None) => (Prices::default(), PriceMap::default()), - (Some(existing), None) => calculate_prices_from_solutions( - model, - existing, - existing, - candidates, - previous_prices, - year, - )?, - (Some(existing), Some(with_candidates)) => calculate_prices_from_solutions( - model, - existing, - with_candidates, - candidates, - previous_prices, - year, - )?, - (None, Some(with_candidates)) => calculate_prices_from_solutions( - model, - with_candidates, - with_candidates, - candidates, - previous_prices, - year, - )?, - }; - - Ok((prices, shadow_prices, flow_map)) -} - -/// Estimates the flows of assets selected by agents using partial dispatch runs. -/// -/// Each run dispatches only the assets selected so far and balances only the markets which agents -/// have already settled. -struct PartialDispatch<'a> { - /// The model - model: &'a Model, - /// The milestone year - year: u32, - /// Preset demands for the year - preset_demands: AllDemandMap, - /// Shadow prices used for markets without a balance constraint - shadow_prices: &'a PriceMap, -} - -impl<'a> PartialDispatch<'a> { - fn new(model: &'a Model, year: u32, shadow_prices: &'a PriceMap) -> Self { - Self { - model, - year, - preset_demands: collect_preset_demands_for_year(&model.commodities, year), - shadow_prices, - } - } -} - -impl FlowEstimator for PartialDispatch<'_> { - fn flows_after_market_set( - &mut self, - market_set: &MarketSet, - settled_markets: &[(CommodityID, RegionID)], - all_selected: &Selection, - _new_selection: &Selection, - writer: &mut DataWriter, - ) -> Result { - debug!("Running post-investment dispatch for '{market_set}'"); - - // As upstream markets by definition will not yet have producers, we explicitly set - // their prices using external values so that they don't appear free - let solution = DispatchRun::new( - self.model, - &all_selected.assets, - self.year, - &self.preset_demands, - ) - .without_commodity_constraints() - .with_market_balance_subset(settled_markets) - .with_input_prices(self.shadow_prices) - .run(&format!("post {market_set} investment"), writer)?; - - Ok(solution.create_flow_map()) - } - - fn flows_after_cycle_market( - &mut self, - cycle: &[(CommodityID, RegionID)], - market: &(CommodityID, RegionID), - pass: u32, - net_demand: &AllDemandMap, - selection: &Selection, - writer: &mut DataWriter, - ) -> Result { - let markets_str = cycle.iter().map(|(c, r)| format!("{c}|{r}")).join(", "); - let (commodity_id, region_id) = market; - let description = - format!("cycle ({markets_str}) post {commodity_id}|{region_id} investment pass {pass}"); - - debug!("Running {description}"); - let solution = DispatchRun::new(self.model, &selection.assets, self.year, net_demand) - .without_commodity_constraints() - .with_market_balance_subset(slice::from_ref(market)) - .with_input_prices(self.shadow_prices) - .run(&description, writer) - .with_context(|| format!("Dispatch failed for cycle ({markets_str})"))?; - debug!("Completed {description}"); - - Ok(solution.create_flow_map()) - } + system_dispatch::run(model, &mut writer)?; + writer.flush() } /// Create candidate assets for all potential processes in a specified year diff --git a/src/simulation/prices.rs b/src/simulation/prices.rs index f48e024cd..6a32460b5 100644 --- a/src/simulation/prices.rs +++ b/src/simulation/prices.rs @@ -7,7 +7,6 @@ use crate::model::Model; use crate::region::RegionID; use crate::simulation::activity::ActivityMap; use crate::simulation::market::MarketSet; -use crate::simulation::optimisation::Solution; use crate::time_slice::{TimeSliceID, TimeSliceInfo, TimeSliceSelection}; use crate::units::{Activity, Dimensionless, Flow, MoneyPerFlow, UnitType, Year}; use anyhow::Result; @@ -62,47 +61,6 @@ pub struct Prices { pub fallback: PriceMap, } -/// Calculate commodity prices from the solutions to dispatch optimisations. -/// -/// This extracts the activities of existing assets from `solution_without_candidates` and the -/// shadow prices from `solution_with_candidates`, then calls [`calculate_prices`]. -/// -/// # Returns -/// -/// The calculated prices, along with the shadow prices from `solution_with_candidates`. -/// -/// # Arguments -/// -/// * `model` - The model -/// * `solution_without_candidates` - Solution to dispatch optimisation without candidates -/// * `solution_with_candidates` - Solution to dispatch optimisation with candidates -/// * `candidates` - The candidate assets included in `solution_with_candidates`, if any -/// * `previous_prices` - Prices from the previous year or ironing-out iteration, used to seed -/// price calculations for cyclic markets -/// * `year` - The year for which prices are being calculated -pub fn calculate_prices_from_solutions( - model: &Model, - solution_without_candidates: &Solution, - solution_with_candidates: &Solution, - candidates: &[AssetRef], - previous_prices: &Prices, - year: u32, -) -> Result<(Prices, PriceMap)> { - let prices = calculate_prices( - model, - &solution_without_candidates.activity_map_for_existing(), - candidates, - previous_prices, - year, - )?; - - // Shadow prices come from the solution _with_ candidates, so that we can get them for - // commodities not produced/consumed by any existing assets. - let shadow_prices = - PriceMap::from_iter(solution_with_candidates.iter_commodity_balance_duals()); - Ok((prices, shadow_prices)) -} - /// Calculate commodity prices. /// /// Calculate prices for each commodity/region/time-slice according to the commodity's configured diff --git a/src/simulation/system_dispatch.rs b/src/simulation/system_dispatch.rs new file mode 100644 index 000000000..06c5a5773 --- /dev/null +++ b/src/simulation/system_dispatch.rs @@ -0,0 +1,395 @@ +//! The `system_dispatch` simulation protocol. +//! +//! Agents choose capacity for each milestone year; commodity flows and prices are then obtained +//! from a system-wide, cost-minimising dispatch optimisation. +//! +//! In the base year, the assets are dispatched with and without candidate assets to get flows and +//! prices. In subsequent years, agents invest, with partial dispatch runs used to propagate demand +//! between markets. This is repeated with updated prices until prices converge ("ironing out"), +//! before a final dispatch of the resulting asset pool gives the flows and prices for the year. +use crate::asset::{AssetPool, AssetRef}; +use crate::commodity::CommodityID; +use crate::model::Model; +use crate::output::DataWriter; +use crate::region::RegionID; +use crate::simulation::activity::FlowMap; +use crate::simulation::candidate_assets_for_next_year; +use crate::simulation::investment::{ + AgentRunMode, AllDemandMap, FlowEstimator, Selection, collect_preset_demands_for_year, + run_agents_for_year, +}; +use crate::simulation::market::MarketSet; +use crate::simulation::prices::{PriceMap, Prices, calculate_prices}; +use crate::timeit::DispatchTimer; +use anyhow::{Context, Result}; +use context_manager; +use itertools::Itertools; +use log::{debug, info}; +use std::slice; + +pub mod optimisation; +use optimisation::{DispatchRun, Solution}; + +/// Run the simulation using the `system_dispatch` protocol. +/// +/// # Arguments: +/// +/// * `model` - The model to run +/// * `writer` - The data writer to which results are written +#[allow(clippy::too_many_lines)] +pub fn run(model: &Model, writer: &mut DataWriter) -> Result<()> { + let mut user_assets = model.user_assets.clone(); + let mut asset_pool = AssetPool::new(); // active assets + + // Iterate over milestone years + let mut year_iter = model.iter_years().peekable(); + let year = year_iter.next().unwrap(); // Unwrap is safe: model must contain at least one milestone year + + info!("Milestone year: {year}"); + + // Commission assets for base year + let new_assets = asset_pool.commission_new(year, &mut user_assets); + + // Write assets to file + writer.write_assets(new_assets)?; + writer.write_asset_capacities(year, &asset_pool)?; + + // Gather candidates for the next year, if any + let next_year = year_iter.peek().copied(); + let mut candidates = candidate_assets_for_next_year( + &model.processes, + next_year, + model.parameters.candidate_asset_capacity, + ); + + // Run dispatch optimisation + info!("Running dispatch optimisation..."); + // There are no previous prices in the base year, so cyclic markets are seeded with zero + let (mut prices, mut shadow_prices, flow_map) = run_dispatch_for_year( + model, + &asset_pool, + &candidates, + &Prices::default(), + year, + writer, + )?; + + // Write results of dispatch optimisation to file + writer.write_flows(year, &flow_map)?; + writer.write_prices(year, &prices.market)?; + + while let Some(year) = year_iter.next() { + info!("Milestone year: {year}"); + + // Decommission assets whose lifetime has passed + asset_pool.decommission_old(year); + + // Commission user-defined assets for this year + let mut new_assets = asset_pool.commission_new(year, &mut user_assets).to_vec(); + + // Take all the active assets as a list of existing assets + let existing_assets = asset_pool.take(); + + // Iterative loop to "iron out" prices via repeated investment and dispatch + let mut ironing_out_iter = 0; + let selected_assets: Vec = loop { + // Add context to the writer + writer.set_debug_context(format!("ironing out iteration {ironing_out_iter}")); + + // Perform agent investment + info!("Running agent investment..."); + // NB: activities from asset selection are not currently used + let mut flow_estimator = PartialDispatch::new(model, year, &shadow_prices); + let selected_assets = run_agents_for_year( + model, + year, + &existing_assets, + AgentRunMode::Invest, + &prices, + &mut flow_estimator, + writer, + ) + .context("Agent investment failed")? + .assets; + + // Run dispatch optimisation to get updated prices for the next iteration + info!("Running dispatch optimisation..."); + let (new_prices, new_shadow_prices, ..) = + run_dispatch_for_year(model, &selected_assets, &candidates, &prices, year, writer)?; + + // Check if prices have converged using time slice-weighted averages + let prices_stable = prices.market.within_tolerance_weighted( + &new_prices.market, + model.parameters.price_tolerance, + &model.time_slice_info, + ); + + // Update prices for the next iteration + prices = new_prices; + shadow_prices = new_shadow_prices; + + // Clear writer context + writer.clear_debug_context(); + + // Break early if prices have converged + if prices_stable { + info!("Prices converged after {} iterations", ironing_out_iter + 1); + break selected_assets; + } + + // Break if max iterations reached + ironing_out_iter += 1; + if ironing_out_iter == model.parameters.max_ironing_out_iterations { + info!( + "Max ironing out iterations ({}) reached", + model.parameters.max_ironing_out_iterations + ); + break selected_assets; + } + }; + + // Add selected_assets to the active pool, receiving the newly commissioned ones + let newly_selected = asset_pool.extend(selected_assets); + new_assets.extend_from_slice(newly_selected); + + // Decommission unused assets + asset_pool.mothball_unretained(existing_assets, year); + asset_pool.decommission_mothballed(year, model.parameters.mothball_years); + + // Write newly commissioned assets + writer.write_assets(&new_assets)?; + writer.write_asset_capacities(year, &asset_pool)?; + + // Gather candidates for the next year, if any + let next_year = year_iter.peek().copied(); + candidates = candidate_assets_for_next_year( + &model.processes, + next_year, + model.parameters.candidate_asset_capacity, + ); + + // Run dispatch optimisation + info!("Running final dispatch optimisation for year {year}..."); + let (new_prices, new_shadow_prices, flow_map) = + run_dispatch_for_year(model, &asset_pool, &candidates, &prices, year, writer)?; + + // Write results of dispatch optimisation to file + writer.write_flows(year, &flow_map)?; + writer.write_prices(year, &new_prices.market)?; + + // Prices for the next year + prices = new_prices; + shadow_prices = new_shadow_prices; + } + + Ok(()) +} + +/// Run dispatch to get flows and prices for a milestone year. +/// +/// Tries to get commodity flows from a dispatch run which just includes existing assets and prices +/// from a run with both existing and candidate assets. If either flows or prices cannot be +/// calculated, empty maps are returned. +/// +/// Returns the prices, the shadow prices from the run with candidates, and the flows of the +/// existing assets. +/// +/// `previous_prices` (from the previous year or ironing-out iteration) are used to seed price +/// calculations for cyclic markets. +/// +/// Fully mothballed assets or mothballed tranches thereof are excluded. +#[context_manager::wrap(DispatchTimer)] +fn run_dispatch_for_year( + model: &Model, + assets: &[AssetRef], + candidates: &[AssetRef], + previous_prices: &Prices, + year: u32, + writer: &mut DataWriter, +) -> Result<(Prices, PriceMap, FlowMap)> { + // Sanity checks + debug_assert!(assets.iter().all(|asset| !asset.is_candidate())); + debug_assert!(candidates.iter().all(|asset| asset.is_candidate())); + + let market_demands = collect_preset_demands_for_year(&model.commodities, year); + + // Run dispatch optimisation with existing assets only, if there are any. If not, then assume no + // flows (i.e. all are zero) + let (solution_existing, flow_map) = if assets.is_empty() { + (None, FlowMap::default()) + } else { + let solution = DispatchRun::new(model, assets, year, &market_demands) + .run("final without candidates", writer)?; + let flow_map = solution.create_flow_map(); + (Some(solution), flow_map) + }; + + // Perform a separate dispatch run with both existing assets and candidates, if there are any, + // to get shadow prices. If not, use the existing solution alone. + let solution_with_candidates = if candidates.is_empty() { + None + } else { + Some( + DispatchRun::new(model, assets, year, &market_demands) + .with_candidates(candidates) + .run("final with candidates", writer)?, + ) + }; + + // Calculate prices from the appropriate solution(s). If there were candidates, use the solution + // that includes them; otherwise use the existing solution alone. If there were no assets at + // all, return empty prices. + let (prices, shadow_prices) = match ( + solution_existing.as_ref(), + solution_with_candidates.as_ref(), + ) { + (None, None) => (Prices::default(), PriceMap::default()), + (Some(existing), None) => calculate_prices_from_solutions( + model, + existing, + existing, + candidates, + previous_prices, + year, + )?, + (Some(existing), Some(with_candidates)) => calculate_prices_from_solutions( + model, + existing, + with_candidates, + candidates, + previous_prices, + year, + )?, + (None, Some(with_candidates)) => calculate_prices_from_solutions( + model, + with_candidates, + with_candidates, + candidates, + previous_prices, + year, + )?, + }; + + Ok((prices, shadow_prices, flow_map)) +} + +/// Estimates the flows of assets selected by agents using partial dispatch runs. +/// +/// Each run dispatches only the assets selected so far and balances only the markets which agents +/// have already settled. +struct PartialDispatch<'a> { + /// The model + model: &'a Model, + /// The milestone year + year: u32, + /// Preset demands for the year + preset_demands: AllDemandMap, + /// Shadow prices used for markets without a balance constraint + shadow_prices: &'a PriceMap, +} + +impl<'a> PartialDispatch<'a> { + fn new(model: &'a Model, year: u32, shadow_prices: &'a PriceMap) -> Self { + Self { + model, + year, + preset_demands: collect_preset_demands_for_year(&model.commodities, year), + shadow_prices, + } + } +} + +impl FlowEstimator for PartialDispatch<'_> { + fn flows_after_market_set( + &mut self, + market_set: &MarketSet, + settled_markets: &[(CommodityID, RegionID)], + all_selected: &Selection, + _new_selection: &Selection, + writer: &mut DataWriter, + ) -> Result { + debug!("Running post-investment dispatch for '{market_set}'"); + + // As upstream markets by definition will not yet have producers, we explicitly set + // their prices using external values so that they don't appear free + let solution = DispatchRun::new( + self.model, + &all_selected.assets, + self.year, + &self.preset_demands, + ) + .without_commodity_constraints() + .with_market_balance_subset(settled_markets) + .with_input_prices(self.shadow_prices) + .run(&format!("post {market_set} investment"), writer)?; + + Ok(solution.create_flow_map()) + } + + fn flows_after_cycle_market( + &mut self, + cycle: &[(CommodityID, RegionID)], + market: &(CommodityID, RegionID), + pass: u32, + net_demand: &AllDemandMap, + selection: &Selection, + writer: &mut DataWriter, + ) -> Result { + let markets_str = cycle.iter().map(|(c, r)| format!("{c}|{r}")).join(", "); + let (commodity_id, region_id) = market; + let description = + format!("cycle ({markets_str}) post {commodity_id}|{region_id} investment pass {pass}"); + + debug!("Running {description}"); + let solution = DispatchRun::new(self.model, &selection.assets, self.year, net_demand) + .without_commodity_constraints() + .with_market_balance_subset(slice::from_ref(market)) + .with_input_prices(self.shadow_prices) + .run(&description, writer) + .with_context(|| format!("Dispatch failed for cycle ({markets_str})"))?; + debug!("Completed {description}"); + + Ok(solution.create_flow_map()) + } +} + +/// Calculate commodity prices from the solutions to dispatch optimisations. +/// +/// This extracts the activities of existing assets from `solution_without_candidates` and the +/// shadow prices from `solution_with_candidates`, then calls [`calculate_prices`]. +/// +/// # Returns +/// +/// The calculated prices, along with the shadow prices from `solution_with_candidates`. +/// +/// # Arguments +/// +/// * `model` - The model +/// * `solution_without_candidates` - Solution to dispatch optimisation without candidates +/// * `solution_with_candidates` - Solution to dispatch optimisation with candidates +/// * `candidates` - The candidate assets included in `solution_with_candidates`, if any +/// * `previous_prices` - Prices from the previous year or ironing-out iteration, used to seed +/// price calculations for cyclic markets +/// * `year` - The year for which prices are being calculated +pub fn calculate_prices_from_solutions( + model: &Model, + solution_without_candidates: &Solution, + solution_with_candidates: &Solution, + candidates: &[AssetRef], + previous_prices: &Prices, + year: u32, +) -> Result<(Prices, PriceMap)> { + let prices = calculate_prices( + model, + &solution_without_candidates.activity_map_for_existing(), + candidates, + previous_prices, + year, + )?; + + // Shadow prices come from the solution _with_ candidates, so that we can get them for + // commodities not produced/consumed by any existing assets. + let shadow_prices = + PriceMap::from_iter(solution_with_candidates.iter_commodity_balance_duals()); + Ok((prices, shadow_prices)) +} diff --git a/src/simulation/optimisation.rs b/src/simulation/system_dispatch/optimisation.rs similarity index 100% rename from src/simulation/optimisation.rs rename to src/simulation/system_dispatch/optimisation.rs diff --git a/src/simulation/optimisation/constraints.rs b/src/simulation/system_dispatch/optimisation/constraints.rs similarity index 100% rename from src/simulation/optimisation/constraints.rs rename to src/simulation/system_dispatch/optimisation/constraints.rs From f61b70496a56b6b2f2a7df8eaa2d48c48037034a Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Wed, 11 Nov 2026 15:19:04 +0000 Subject: [PATCH 04/11] Move dispatch debug output into system_dispatch output.rs no longer depends on dispatch internals. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/output.rs | 476 +---------------- src/simulation/system_dispatch.rs | 6 + .../system_dispatch/optimisation.rs | 14 +- src/simulation/system_dispatch/output.rs | 485 ++++++++++++++++++ 4 files changed, 508 insertions(+), 473 deletions(-) create mode 100644 src/simulation/system_dispatch/output.rs diff --git a/src/output.rs b/src/output.rs index cb07fe4cf..4f0057330 100644 --- a/src/output.rs +++ b/src/output.rs @@ -6,12 +6,10 @@ use crate::process::ProcessID; use crate::region::RegionID; use crate::simulation::activity::FlowMap; use crate::simulation::prices::PriceMap; -use crate::simulation::system_dispatch::optimisation::Solution; use crate::time_slice::TimeSliceID; -use crate::units::{Activity, Capacity, Flow, Money, MoneyPerActivity, MoneyPerFlow}; +use crate::units::{Capacity, Flow, MoneyPerFlow}; use anyhow::{Context, Result, ensure}; use csv; -use indexmap::IndexMap; use serde::{Deserialize, Serialize}; use std::borrow::Cow; use std::collections::HashMap; @@ -34,18 +32,6 @@ const ASSETS_FILE_NAME: &str = "assets.csv"; /// The output file name for asset capacities const ASSET_CAPACITIES_FILE_NAME: &str = "asset_capacities.csv"; -/// Debug output file for asset dispatch -const ACTIVITY_ASSET_DISPATCH: &str = "debug_dispatch_assets.csv"; - -/// The output file name for commodity balance duals -const COMMODITY_BALANCE_DUALS_FILE_NAME: &str = "debug_commodity_balance_duals.csv"; - -/// The output file name for unmet demand values -const UNMET_DEMAND_FILE_NAME: &str = "debug_unmet_demand.csv"; - -/// The output file name for extra solver output values -const SOLVER_VALUES_FILE_NAME: &str = "debug_solver.csv"; - /// Get the default output directory for the model pub fn get_output_dir(model_dir: &Path, results_root: PathBuf) -> Result { // Get the model name from the dir path. This ends up being convoluted because we need to check @@ -188,85 +174,26 @@ struct CommodityPriceRow { price: MoneyPerFlow, } -/// Represents the activity in a row of the dispatch CSV file -#[derive(Serialize, Deserialize, Debug, PartialEq)] -struct DispatchRow { - milestone_year: u32, - run_description: String, - asset_id: Option, - process_id: ProcessID, - region_id: RegionID, - time_slice: TimeSliceID, - activity: Option, - activity_dual: Option, - column_dual: Option, -} - -/// Represents the commodity balance duals data in a row of the commodity balance duals CSV file -#[derive(Serialize, Deserialize, Debug, PartialEq)] -struct CommodityBalanceDualsRow { - milestone_year: u32, - run_description: String, - commodity_id: CommodityID, - region_id: RegionID, - time_slice: TimeSliceID, - value: MoneyPerFlow, -} - -/// Represents the unmet demand data in a row of the unmet demand CSV file -#[derive(Serialize, Deserialize, Debug, PartialEq)] -struct UnmetDemandRow { - milestone_year: u32, - run_description: String, - commodity_id: CommodityID, - region_id: RegionID, - time_slice: TimeSliceID, - value: Flow, -} - -/// Represents solver output values -#[derive(Serialize, Deserialize, Debug, PartialEq)] -struct SolverValuesRow { - milestone_year: u32, - run_description: String, - objective_value: Money, -} - /// For writing extra debug information about the model pub struct DebugDataWriter { context: Option, output_path: PathBuf, - unmet_demand_file_path: PathBuf, - commodity_balance_duals_writer: csv::Writer, - unmet_demand_writer: Option>, - solver_values_writer: csv::Writer, - dispatch_asset_writer: csv::Writer, /// Writers for debug files owned by other modules, created on first use custom_writers: HashMap<&'static str, csv::Writer>, } impl DebugDataWriter { - /// Open CSV files to write debug info to + /// Create a writer for debug info /// /// # Arguments /// /// * `output_path` - Folder where files will be saved - fn create(output_path: &Path) -> Result { - let new_writer = |file_name| { - let file_path = output_path.join(file_name); - csv::Writer::from_path(file_path) - }; - - Ok(Self { + fn create(output_path: &Path) -> Self { + Self { context: None, output_path: output_path.to_path_buf(), - unmet_demand_file_path: output_path.join(UNMET_DEMAND_FILE_NAME), - commodity_balance_duals_writer: new_writer(COMMODITY_BALANCE_DUALS_FILE_NAME)?, - unmet_demand_writer: None, - solver_values_writer: new_writer(SOLVER_VALUES_FILE_NAME)?, - dispatch_asset_writer: new_writer(ACTIVITY_ASSET_DISPATCH)?, custom_writers: HashMap::new(), - }) + } } /// Prepend the current context to the run description @@ -283,171 +210,6 @@ impl DebugDataWriter { self.with_context(run_description) } - /// Write debug info about the dispatch optimisation - fn write_dispatch_debug_info( - &mut self, - milestone_year: u32, - run_description: &str, - solution: &Solution, - ) -> Result<()> { - self.write_dispatch( - milestone_year, - run_description, - solution.iter_activity(), - solution.iter_activity_duals(), - solution.iter_column_duals(), - )?; - self.write_commodity_balance_duals( - milestone_year, - run_description, - solution.iter_commodity_balance_duals(), - )?; - self.write_unmet_demand( - milestone_year, - run_description, - solution.iter_unmet_demand(), - )?; - self.write_solver_values(milestone_year, run_description, solution.objective_value)?; - Ok(()) - } - - // Write activity to file - fn write_dispatch<'a, I, J, K>( - &mut self, - milestone_year: u32, - run_description: &str, - iter_activity: I, - iter_activity_duals: J, - iter_column_duals: K, - ) -> Result<()> - where - I: Iterator, - J: Iterator, - K: Iterator, - { - // To account for different order of entries or missing ones, we first compile data in hash map - type CompiledActivityData = ( - Option, - Option, - Option, - ); - let mut map: IndexMap<(&AssetRef, &TimeSliceID), CompiledActivityData> = IndexMap::new(); - - // For the activities - for (asset, time_slice, activity) in iter_activity { - map.entry((asset, time_slice)).or_default().0 = Some(activity); - } - // The activity duals - for (asset, time_slice, activity_dual) in iter_activity_duals { - map.entry((asset, time_slice)).or_default().1 = Some(activity_dual); - } - // And the column duals - for (asset, time_slice, column_dual) in iter_column_duals { - map.entry((asset, time_slice)).or_default().2 = Some(column_dual); - } - - for ((asset, time_slice), (activity, activity_dual, column_dual)) in map { - let row = DispatchRow { - milestone_year, - run_description: self.with_context(run_description).into_owned(), - asset_id: asset.id(), - process_id: asset.process_id().clone(), - region_id: asset.region_id().clone(), - time_slice: time_slice.clone(), - activity, - activity_dual, - column_dual, - }; - self.dispatch_asset_writer.serialize(row)?; - } - - Ok(()) - } - - /// Write commodity balance duals to file - fn write_commodity_balance_duals<'a, I>( - &mut self, - milestone_year: u32, - run_description: &str, - iter: I, - ) -> Result<()> - where - I: Iterator, - { - for (commodity_id, region_id, time_slice, value) in iter { - let row = CommodityBalanceDualsRow { - milestone_year, - run_description: self.with_context(run_description).into_owned(), - commodity_id: commodity_id.clone(), - region_id: region_id.clone(), - time_slice: time_slice.clone(), - value, - }; - self.commodity_balance_duals_writer.serialize(row)?; - } - - Ok(()) - } - - /// Write unmet demand values to file - fn write_unmet_demand<'a, I>( - &mut self, - milestone_year: u32, - run_description: &str, - iter: I, - ) -> Result<()> - where - I: Iterator, - { - let mut rows = iter.peekable(); - - if rows.peek().is_none() { - return Ok(()); - } - - // If the unmet demand writer already exist, we panic, as it should not happen - assert!( - self.unmet_demand_writer.is_none(), - "Unmet demand file already exists!" - ); - - let run_description = self.with_context(run_description).into_owned(); - let writer = self - .unmet_demand_writer - .insert(csv::Writer::from_path(&self.unmet_demand_file_path)?); - for (commodity_id, region_id, time_slice, value) in rows { - let row = UnmetDemandRow { - milestone_year, - run_description: run_description.clone(), - commodity_id: commodity_id.clone(), - region_id: region_id.clone(), - time_slice: time_slice.clone(), - value, - }; - writer.serialize(row)?; - } - - Ok(()) - } - - /// Write additional solver output values to file - fn write_solver_values( - &mut self, - milestone_year: u32, - run_description: &str, - objective_value: Money, - ) -> Result<()> { - let row = SolverValuesRow { - milestone_year, - run_description: self.with_context(run_description).into_owned(), - objective_value, - }; - self.solver_values_writer.serialize(row)?; - self.solver_values_writer.flush()?; - - Ok(()) - } - /// Get a writer for a custom debug CSV file, creating the file on first use. /// /// File names must start with `debug_` and end with `.csv`. @@ -474,12 +236,6 @@ impl DebugDataWriter { /// Flush the underlying streams fn flush(&mut self) -> Result<()> { - if let Some(wrt) = &mut self.unmet_demand_writer { - wrt.flush()?; - } - self.commodity_balance_duals_writer.flush()?; - self.solver_values_writer.flush()?; - self.dispatch_asset_writer.flush()?; for wrt in self.custom_writers.values_mut() { wrt.flush()?; } @@ -513,12 +269,7 @@ impl DataWriter { csv::Writer::from_path(file_path) }; - let debug_writer = if save_debug_info { - // Create debug CSV files - Some(DebugDataWriter::create(output_path)?) - } else { - None - }; + let debug_writer = save_debug_info.then(|| DebugDataWriter::create(output_path)); Ok(Self { assets: new_writer(ASSETS_FILE_NAME)?, @@ -529,20 +280,6 @@ impl DataWriter { }) } - /// Write debug info about the dispatch optimisation - pub fn write_dispatch_debug_info( - &mut self, - milestone_year: u32, - run_description: &str, - solution: &Solution, - ) -> Result<()> { - if let Some(wtr) = &mut self.debug { - wtr.write_dispatch_debug_info(milestone_year, run_description, solution)?; - } - - Ok(()) - } - /// Get the debug writer, if debug output is enabled. pub fn debug_writer(&mut self) -> Option<&mut DebugDataWriter> { self.debug.as_mut() @@ -809,207 +546,6 @@ mod tests { assert_equal(records, iter::once(expected)); } - #[rstest] - fn write_commodity_balance_duals( - commodity_id: CommodityID, - region_id: RegionID, - time_slice: TimeSliceID, - ) { - let milestone_year = 2020; - let run_description = "test_run".to_string(); - let value = MoneyPerFlow(0.5); - let dir = tempdir().unwrap(); - - // Write commodity balance dual - { - let mut writer = DebugDataWriter::create(dir.path()).unwrap(); - writer - .write_commodity_balance_duals( - milestone_year, - &run_description, - iter::once((&commodity_id, ®ion_id, &time_slice, value)), - ) - .unwrap(); - writer.flush().unwrap(); - } - - // Read back and compare - let expected = CommodityBalanceDualsRow { - milestone_year, - run_description, - commodity_id, - region_id, - time_slice, - value, - }; - let records: Vec = - csv::Reader::from_path(dir.path().join(COMMODITY_BALANCE_DUALS_FILE_NAME)) - .unwrap() - .into_deserialize() - .try_collect() - .unwrap(); - assert_equal(records, iter::once(expected)); - } - - #[rstest] - fn write_unmet_demand(commodity_id: CommodityID, region_id: RegionID, time_slice: TimeSliceID) { - let milestone_year = 2020; - let run_description = "test_run".to_string(); - let value = Flow(0.5); - let dir = tempdir().unwrap(); - - // Write unmet demand - { - let mut writer = DebugDataWriter::create(dir.path()).unwrap(); - writer - .write_unmet_demand( - milestone_year, - &run_description, - iter::once((&commodity_id, ®ion_id, &time_slice, value)), - ) - .unwrap(); - writer.flush().unwrap(); - } - - // Read back and compare - let expected = UnmetDemandRow { - milestone_year, - run_description, - commodity_id, - region_id, - time_slice, - value, - }; - let records: Vec = - csv::Reader::from_path(dir.path().join(UNMET_DEMAND_FILE_NAME)) - .unwrap() - .into_deserialize() - .try_collect() - .unwrap(); - assert_equal(records, iter::once(expected)); - } - - #[rstest] - fn write_dispatch(assets: AssetPool, time_slice: TimeSliceID) { - let milestone_year = 2020; - let run_description = "test_run".to_string(); - let activity = Activity(100.5); - let activity_dual = MoneyPerActivity(-1.5); - let column_dual = MoneyPerActivity(5.0); - let dir = tempdir().unwrap(); - let asset = assets.iter().next().unwrap(); - - // Write activity - { - let mut writer = DebugDataWriter::create(dir.path()).unwrap(); - writer - .write_dispatch( - milestone_year, - &run_description, - iter::once((asset, &time_slice, activity)), - iter::once((asset, &time_slice, activity_dual)), - iter::once((asset, &time_slice, column_dual)), - ) - .unwrap(); - writer.flush().unwrap(); - } - - // Read back and compare - let expected = DispatchRow { - milestone_year, - run_description, - asset_id: asset.id(), - process_id: asset.process_id().clone(), - region_id: asset.region_id().clone(), - time_slice, - activity: Some(activity), - activity_dual: Some(activity_dual), - column_dual: Some(column_dual), - }; - let records: Vec = - csv::Reader::from_path(dir.path().join(ACTIVITY_ASSET_DISPATCH)) - .unwrap() - .into_deserialize() - .try_collect() - .unwrap(); - assert_equal(records, iter::once(expected)); - } - - #[rstest] - fn write_dispatch_with_missing_keys(assets: AssetPool, time_slice: TimeSliceID) { - let milestone_year = 2020; - let run_description = "test_run".to_string(); - let activity = Activity(100.5); - let dir = tempdir().unwrap(); - let asset = assets.iter().next().unwrap(); - - // Write activity - { - let mut writer = DebugDataWriter::create(dir.path()).unwrap(); - writer - .write_dispatch( - milestone_year, - &run_description, - iter::once((asset, &time_slice, activity)), - iter::empty::<(&AssetRef, &TimeSliceID, MoneyPerActivity)>(), - iter::empty::<(&AssetRef, &TimeSliceID, MoneyPerActivity)>(), - ) - .unwrap(); - writer.flush().unwrap(); - } - - // Read back and compare - let expected = DispatchRow { - milestone_year, - run_description, - asset_id: asset.id(), - process_id: asset.process_id().clone(), - region_id: asset.region_id().clone(), - time_slice, - activity: Some(activity), - activity_dual: None, - column_dual: None, - }; - let records: Vec = - csv::Reader::from_path(dir.path().join(ACTIVITY_ASSET_DISPATCH)) - .unwrap() - .into_deserialize() - .try_collect() - .unwrap(); - assert_equal(records, iter::once(expected)); - } - - #[rstest] - fn write_solver_values() { - let milestone_year = 2020; - let run_description = "test_run".to_string(); - let objective_value = Money(1234.56); - let dir = tempdir().unwrap(); - - // Write solver values - { - let mut writer = DebugDataWriter::create(dir.path()).unwrap(); - writer - .write_solver_values(milestone_year, &run_description, objective_value) - .unwrap(); - writer.flush().unwrap(); - } - - // Read back and compare - let expected = SolverValuesRow { - milestone_year, - run_description, - objective_value, - }; - let records: Vec = - csv::Reader::from_path(dir.path().join(SOLVER_VALUES_FILE_NAME)) - .unwrap() - .into_deserialize() - .try_collect() - .unwrap(); - assert_equal(records, iter::once(expected)); - } - #[rstest] #[case(true)] #[case(false)] diff --git a/src/simulation/system_dispatch.rs b/src/simulation/system_dispatch.rs index 06c5a5773..f64e43873 100644 --- a/src/simulation/system_dispatch.rs +++ b/src/simulation/system_dispatch.rs @@ -27,7 +27,9 @@ use itertools::Itertools; use log::{debug, info}; use std::slice; +mod output; pub mod optimisation; +use output::create_debug_files; use optimisation::{DispatchRun, Solution}; /// Run the simulation using the `system_dispatch` protocol. @@ -38,6 +40,10 @@ use optimisation::{DispatchRun, Solution}; /// * `writer` - The data writer to which results are written #[allow(clippy::too_many_lines)] pub fn run(model: &Model, writer: &mut DataWriter) -> Result<()> { + if let Some(debug_writer) = writer.debug_writer() { + create_debug_files(debug_writer)?; + } + let mut user_assets = model.user_assets.clone(); let mut asset_pool = AssetPool::new(); // active assets diff --git a/src/simulation/system_dispatch/optimisation.rs b/src/simulation/system_dispatch/optimisation.rs index b3b8bd61d..773eb91e0 100644 --- a/src/simulation/system_dispatch/optimisation.rs +++ b/src/simulation/system_dispatch/optimisation.rs @@ -1,6 +1,7 @@ //! Code for performing dispatch optimisation. //! //! This is used to calculate commodity flows and prices. +use super::output::write_dispatch_debug_info; use crate::asset::{Asset, AssetRef}; use crate::commodity::CommodityID; use crate::input::format_items_with_cap; @@ -407,7 +408,12 @@ impl<'model, 'run> DispatchRun<'model, 'run> { match self.run_without_unmet_demand_variables(markets_to_balance, input_prices) { // If the run is successful, we write debug info and return the solution Ok(solution) => { - writer.write_dispatch_debug_info(self.year, run_description, &solution)?; + write_dispatch_debug_info( + writer.debug_writer(), + self.year, + run_description, + &solution, + )?; Ok(solution) } @@ -484,7 +490,8 @@ impl<'model, 'run> DispatchRun<'model, 'run> { Ok(solution) => { let diagnostic_run_description = format!("{run_description} COMMODITY_CONSTRAINTS_DIAGNOSTIC"); - writer.write_dispatch_debug_info( + write_dispatch_debug_info( + writer.debug_writer(), self.year, &diagnostic_run_description, &solution, @@ -528,7 +535,8 @@ impl<'model, 'run> DispatchRun<'model, 'run> { // never returned as the result of the original dispatch run. let diagnostic_run_description = format!("{run_description} UNMET_DEMAND_DIAGNOSTIC"); - writer.write_dispatch_debug_info( + write_dispatch_debug_info( + writer.debug_writer(), self.year, &diagnostic_run_description, &solution, diff --git a/src/simulation/system_dispatch/output.rs b/src/simulation/system_dispatch/output.rs new file mode 100644 index 000000000..371007c1d --- /dev/null +++ b/src/simulation/system_dispatch/output.rs @@ -0,0 +1,485 @@ +//! Debug output files for the dispatch optimisation. +use super::optimisation::Solution; +use crate::asset::{AssetID, AssetRef}; +use crate::commodity::CommodityID; +use crate::output::DebugDataWriter; +use crate::process::ProcessID; +use crate::region::RegionID; +use crate::time_slice::TimeSliceID; +use crate::units::{Activity, Flow, Money, MoneyPerActivity, MoneyPerFlow}; +use anyhow::Result; +use indexmap::IndexMap; +use serde::{Deserialize, Serialize}; + +/// Debug output file for asset dispatch +const ACTIVITY_ASSET_DISPATCH: &str = "debug_dispatch_assets.csv"; + +/// The output file name for commodity balance duals +const COMMODITY_BALANCE_DUALS_FILE_NAME: &str = "debug_commodity_balance_duals.csv"; + +/// The output file name for unmet demand values +const UNMET_DEMAND_FILE_NAME: &str = "debug_unmet_demand.csv"; + +/// The output file name for extra solver output values +const SOLVER_VALUES_FILE_NAME: &str = "debug_solver.csv"; + +/// Represents the activity in a row of the dispatch CSV file +#[derive(Serialize, Deserialize, Debug, PartialEq)] +struct DispatchRow { + milestone_year: u32, + run_description: String, + asset_id: Option, + process_id: ProcessID, + region_id: RegionID, + time_slice: TimeSliceID, + activity: Option, + activity_dual: Option, + column_dual: Option, +} + +/// Represents the commodity balance duals data in a row of the commodity balance duals CSV file +#[derive(Serialize, Deserialize, Debug, PartialEq)] +struct CommodityBalanceDualsRow { + milestone_year: u32, + run_description: String, + commodity_id: CommodityID, + region_id: RegionID, + time_slice: TimeSliceID, + value: MoneyPerFlow, +} + +/// Represents the unmet demand data in a row of the unmet demand CSV file +#[derive(Serialize, Deserialize, Debug, PartialEq)] +struct UnmetDemandRow { + milestone_year: u32, + run_description: String, + commodity_id: CommodityID, + region_id: RegionID, + time_slice: TimeSliceID, + value: Flow, +} + +/// Represents solver output values +#[derive(Serialize, Deserialize, Debug, PartialEq)] +struct SolverValuesRow { + milestone_year: u32, + run_description: String, + objective_value: Money, +} + +/// Create the debug files which are always present when writing debug output +/// +/// The unmet demand file is not included, as it is only created if there is unmet demand. +pub fn create_debug_files(writer: &mut DebugDataWriter) -> Result<()> { + writer.get_custom_writer(ACTIVITY_ASSET_DISPATCH)?; + writer.get_custom_writer(COMMODITY_BALANCE_DUALS_FILE_NAME)?; + writer.get_custom_writer(SOLVER_VALUES_FILE_NAME)?; + Ok(()) +} + +/// Write debug info about the dispatch optimisation +pub fn write_dispatch_debug_info( + writer: Option<&mut DebugDataWriter>, + milestone_year: u32, + run_description: &str, + solution: &Solution, +) -> Result<()> { + let Some(writer) = writer else { + return Ok(()); + }; + + let run_description = writer.debug_run_description(run_description); + write_dispatch( + writer, + milestone_year, + &run_description, + solution.iter_activity(), + solution.iter_activity_duals(), + solution.iter_column_duals(), + )?; + write_commodity_balance_duals( + writer, + milestone_year, + &run_description, + solution.iter_commodity_balance_duals(), + )?; + write_unmet_demand( + writer, + milestone_year, + &run_description, + solution.iter_unmet_demand(), + )?; + write_solver_values( + writer, + milestone_year, + &run_description, + solution.objective_value, + ) +} + +/// Write activity to file +/// +/// `run_description` should already include any debug context. +fn write_dispatch<'a, I, J, K>( + writer: &mut DebugDataWriter, + milestone_year: u32, + run_description: &str, + iter_activity: I, + iter_activity_duals: J, + iter_column_duals: K, +) -> Result<()> +where + I: Iterator, + J: Iterator, + K: Iterator, +{ + // To account for different order of entries or missing ones, we first compile data in hash map + type CompiledActivityData = ( + Option, + Option, + Option, + ); + let mut map: IndexMap<(&AssetRef, &TimeSliceID), CompiledActivityData> = IndexMap::new(); + + // For the activities + for (asset, time_slice, activity) in iter_activity { + map.entry((asset, time_slice)).or_default().0 = Some(activity); + } + // The activity duals + for (asset, time_slice, activity_dual) in iter_activity_duals { + map.entry((asset, time_slice)).or_default().1 = Some(activity_dual); + } + // And the column duals + for (asset, time_slice, column_dual) in iter_column_duals { + map.entry((asset, time_slice)).or_default().2 = Some(column_dual); + } + + let rows = map.into_iter().map( + |((asset, time_slice), (activity, activity_dual, column_dual))| DispatchRow { + milestone_year, + run_description: run_description.to_string(), + asset_id: asset.id(), + process_id: asset.process_id().clone(), + region_id: asset.region_id().clone(), + time_slice: time_slice.clone(), + activity, + activity_dual, + column_dual, + }, + ); + let output_writer = writer.get_custom_writer(ACTIVITY_ASSET_DISPATCH)?; + for row in rows { + output_writer.serialize(row)?; + } + Ok(()) +} + +/// Write commodity balance duals to file +/// +/// `run_description` should already include any debug context. +fn write_commodity_balance_duals<'a, I>( + writer: &mut DebugDataWriter, + milestone_year: u32, + run_description: &str, + iter: I, +) -> Result<()> +where + I: Iterator, +{ + let rows = iter.map( + |(commodity_id, region_id, time_slice, value)| CommodityBalanceDualsRow { + milestone_year, + run_description: run_description.to_string(), + commodity_id: commodity_id.clone(), + region_id: region_id.clone(), + time_slice: time_slice.clone(), + value, + }, + ); + let output_writer = writer.get_custom_writer(COMMODITY_BALANCE_DUALS_FILE_NAME)?; + for row in rows { + output_writer.serialize(row)?; + } + Ok(()) +} + +/// Write unmet demand values to file +/// +/// The file is only created if there is unmet demand. `run_description` should already include any +/// debug context. +fn write_unmet_demand<'a, I>( + writer: &mut DebugDataWriter, + milestone_year: u32, + run_description: &str, + iter: I, +) -> Result<()> +where + I: Iterator, +{ + let mut iter = iter.peekable(); + if iter.peek().is_none() { + return Ok(()); + } + + let rows = iter.map( + |(commodity_id, region_id, time_slice, value)| UnmetDemandRow { + milestone_year, + run_description: run_description.to_string(), + commodity_id: commodity_id.clone(), + region_id: region_id.clone(), + time_slice: time_slice.clone(), + value, + }, + ); + let output_writer = writer.get_custom_writer(UNMET_DEMAND_FILE_NAME)?; + for row in rows { + output_writer.serialize(row)?; + } + Ok(()) +} + +/// Write additional solver output values to file +/// +/// `run_description` should already include any debug context. +fn write_solver_values( + writer: &mut DebugDataWriter, + milestone_year: u32, + run_description: &str, + objective_value: Money, +) -> Result<()> { + let row = SolverValuesRow { + milestone_year, + run_description: run_description.to_string(), + objective_value, + }; + let output_writer = writer.get_custom_writer(SOLVER_VALUES_FILE_NAME)?; + output_writer.serialize(row)?; + output_writer.flush()?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::asset::AssetPool; + use crate::fixture::{assets, commodity_id, region_id, time_slice}; + use crate::output::DataWriter; + use itertools::{Itertools, assert_equal}; + use rstest::rstest; + use serde::de::DeserializeOwned; + use std::fmt::Debug; + use std::iter; + use std::path::Path; + use tempfile::tempdir; + + /// Read back the rows of a CSV file + fn read_rows(dir: &Path, file_name: &str) -> Vec { + csv::Reader::from_path(dir.join(file_name)) + .unwrap() + .into_deserialize() + .try_collect() + .unwrap() + } + + /// Check that a single row is written to the given file + fn assert_single_row( + dir: &Path, + file_name: &str, + expected: T, + ) { + assert_equal(read_rows::(dir, file_name), iter::once(expected)); + } + + #[rstest] + fn write_commodity_balance_duals_works( + commodity_id: CommodityID, + region_id: RegionID, + time_slice: TimeSliceID, + ) { + let milestone_year = 2020; + let run_description = "test_run".to_string(); + let value = MoneyPerFlow(0.5); + let dir = tempdir().unwrap(); + + { + let mut writer = DataWriter::create(dir.path(), dir.path(), true).unwrap(); + write_commodity_balance_duals( + writer.debug_writer().unwrap(), + milestone_year, + &run_description, + iter::once((&commodity_id, ®ion_id, &time_slice, value)), + ) + .unwrap(); + writer.flush().unwrap(); + } + + let expected = CommodityBalanceDualsRow { + milestone_year, + run_description, + commodity_id, + region_id, + time_slice, + value, + }; + assert_single_row(dir.path(), COMMODITY_BALANCE_DUALS_FILE_NAME, expected); + } + + #[rstest] + fn write_unmet_demand_works( + commodity_id: CommodityID, + region_id: RegionID, + time_slice: TimeSliceID, + ) { + let milestone_year = 2020; + let run_description = "test_run".to_string(); + let value = Flow(0.5); + let dir = tempdir().unwrap(); + + { + let mut writer = DataWriter::create(dir.path(), dir.path(), true).unwrap(); + write_unmet_demand( + writer.debug_writer().unwrap(), + milestone_year, + &run_description, + iter::once((&commodity_id, ®ion_id, &time_slice, value)), + ) + .unwrap(); + writer.flush().unwrap(); + } + + let expected = UnmetDemandRow { + milestone_year, + run_description, + commodity_id, + region_id, + time_slice, + value, + }; + assert_single_row(dir.path(), UNMET_DEMAND_FILE_NAME, expected); + } + + #[test] + fn write_unmet_demand_skips_file_if_empty() { + let dir = tempdir().unwrap(); + + { + let mut writer = DataWriter::create(dir.path(), dir.path(), true).unwrap(); + write_unmet_demand( + writer.debug_writer().unwrap(), + 2020, + "test_run", + iter::empty(), + ) + .unwrap(); + writer.flush().unwrap(); + } + + assert!(!dir.path().join(UNMET_DEMAND_FILE_NAME).exists()); + } + + #[rstest] + #[case(Some(MoneyPerActivity(-1.5)), Some(MoneyPerActivity(5.0)))] + #[case(None, None)] + fn write_dispatch_works( + assets: AssetPool, + time_slice: TimeSliceID, + #[case] activity_dual: Option, + #[case] column_dual: Option, + ) { + let milestone_year = 2020; + let run_description = "test_run".to_string(); + let activity = Activity(100.5); + let dir = tempdir().unwrap(); + let asset = assets.iter().next().unwrap(); + + // Missing duals should result in empty fields + { + let mut writer = DataWriter::create(dir.path(), dir.path(), true).unwrap(); + write_dispatch( + writer.debug_writer().unwrap(), + milestone_year, + &run_description, + iter::once((asset, &time_slice, activity)), + activity_dual + .map(|dual| (asset, &time_slice, dual)) + .into_iter(), + column_dual + .map(|dual| (asset, &time_slice, dual)) + .into_iter(), + ) + .unwrap(); + writer.flush().unwrap(); + } + + let expected = DispatchRow { + milestone_year, + run_description, + asset_id: asset.id(), + process_id: asset.process_id().clone(), + region_id: asset.region_id().clone(), + time_slice, + activity: Some(activity), + activity_dual, + column_dual, + }; + assert_single_row(dir.path(), ACTIVITY_ASSET_DISPATCH, expected); + } + + #[test] + fn write_solver_values_works() { + let milestone_year = 2020; + let run_description = "test_run".to_string(); + let objective_value = Money(1234.56); + let dir = tempdir().unwrap(); + + { + let mut writer = DataWriter::create(dir.path(), dir.path(), true).unwrap(); + write_solver_values( + writer.debug_writer().unwrap(), + milestone_year, + &run_description, + objective_value, + ) + .unwrap(); + writer.flush().unwrap(); + } + + let expected = SolverValuesRow { + milestone_year, + run_description, + objective_value, + }; + assert_single_row(dir.path(), SOLVER_VALUES_FILE_NAME, expected); + } + + #[test] + fn create_debug_files_creates_expected_files() { + let dir = tempdir().unwrap(); + + { + let mut writer = DataWriter::create(dir.path(), dir.path(), true).unwrap(); + create_debug_files(writer.debug_writer().unwrap()).unwrap(); + writer.flush().unwrap(); + } + + for file_name in [ + ACTIVITY_ASSET_DISPATCH, + COMMODITY_BALANCE_DUALS_FILE_NAME, + SOLVER_VALUES_FILE_NAME, + ] { + assert!(dir.path().join(file_name).exists()); + } + assert!(!dir.path().join(UNMET_DEMAND_FILE_NAME).exists()); + } + + #[test] + fn create_debug_files_does_not_overwrite_existing_file() { + let dir = tempdir().unwrap(); + let path = dir.path().join(ACTIVITY_ASSET_DISPATCH); + std::fs::write(&path, "existing data").unwrap(); + let mut writer = DataWriter::create(dir.path(), dir.path(), true).unwrap(); + + assert!(create_debug_files(writer.debug_writer().unwrap()).is_err()); + assert_eq!(std::fs::read_to_string(path).unwrap(), "existing data"); + } +} From 076fffe939b1c011681e36b99f40e2e007b8afc0 Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Wed, 11 Nov 2026 15:26:04 +0000 Subject: [PATCH 05/11] Add simulation_protocol model setting Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- schemas/input/model.yaml | 18 +++++++++++++++++- src/model/parameters.rs | 32 ++++++++++++++++++++++++++++++-- src/simulation.rs | 19 ++++++++++++++++++- 3 files changed, 65 insertions(+), 4 deletions(-) diff --git a/schemas/input/model.yaml b/schemas/input/model.yaml index bf3d1b726..1f750122e 100644 --- a/schemas/input/model.yaml +++ b/schemas/input/model.yaml @@ -18,6 +18,18 @@ properties: US dollars in 2020 terms), or take any other form chosen by the modeller. The label is metadata only: MUSE2 does not interpret it, perform currency conversion, or adjust monetary values for inflation. Monetary inputs should use a consistent currency and reference year. + simulation_protocol: + type: string + description: The protocol used to simulate each milestone year + default: system_dispatch + enum: + - system_dispatch + notes: | + A simulation protocol defines how agents' investment decisions are combined with dispatch of + the energy system to produce commodity flows and prices for each milestone year. Currently the + only option is `system_dispatch`, in which agents choose capacity and flows and prices come + from a system-wide, cost-minimising dispatch optimisation. See + [the model documentation](../model/README.html#simulation-protocols) for details. candidate_asset_capacity: type: number description: Capacity to give to candidate assets in dispatch optimisation @@ -30,6 +42,8 @@ properties: default: 1e-6 notes: | The default value should work. Do not change this value unless you know what you're doing! + + Only used by the `system_dispatch` simulation protocol. capacity_tranche_fraction: type: number description: Scales the size of inferred investment tranches for candidate assets @@ -64,6 +78,8 @@ properties: default: 1e9 notes: | Currently this only applies to the LCOX appraisal. + + Only used by the `system_dispatch` simulation protocol. seasonal_utilisation_penalty: type: number description: Penalty per unit of capacity used within a season @@ -124,7 +140,7 @@ properties: properties: {} dispatch_options: type: object - description: HiGHS options applied only to dispatch optimisations + description: HiGHS options applied only to dispatch optimisations (only used by the `system_dispatch` simulation protocol) properties: {} appraisal_options: type: object diff --git a/src/model/parameters.rs b/src/model/parameters.rs index d4c36085d..b1b0745fe 100644 --- a/src/model/parameters.rs +++ b/src/model/parameters.rs @@ -8,6 +8,7 @@ use crate::input::{ deserialise_finite_non_negative, deserialise_proportion_nonzero, input_err_msg, is_sorted_and_unique, read_toml, }; +use crate::simulation::SimulationProtocol; use crate::units::{Capacity, Dimensionless, Flow, MoneyPerCapacityPerYear, MoneyPerFlow}; use anyhow::{Context, Result, ensure}; use itertools::Itertools; @@ -78,12 +79,16 @@ pub struct ModelParameters { /// Allow potentially dangerous options to be enabled. #[serde(rename = "please_give_me_broken_results")] // Can't use constant here :-( pub allow_dangerous_options: bool, + /// The protocol used to simulate each milestone year. + pub simulation_protocol: SimulationProtocol, /// The (small) value of capacity given to candidate assets. /// /// Don't change unless you know what you're doing. pub candidate_asset_capacity: Capacity, /// The epsilon added to commodity balance lower bounds to force candidate dispatch. /// + /// Only used by the `system_dispatch` simulation protocol. + /// /// Don't change unless you know what you're doing. #[serde(deserialize_with = "deserialise_finite_non_negative")] pub commodity_balance_epsilon: Flow, @@ -101,7 +106,8 @@ pub struct ModelParameters { pub fallback_pricing_strategy: PricingStrategy, /// The cost applied to unmet demand. /// - /// Currently this only applies to the LCOX appraisal. + /// Currently this only applies to the LCOX appraisal. Only used by the `system_dispatch` + /// simulation protocol. pub value_of_lost_load: MoneyPerFlow, /// Additive penalty per unit of capacity used within a season. #[serde(deserialize_with = "deserialise_finite_non_negative")] @@ -123,7 +129,8 @@ pub struct ModelParameters { pub remaining_demand_absolute_tolerance: Flow, /// Options for the HiGHS solver. /// - /// For a full list of options, see [the HiGHS documentation]. + /// For a full list of options, see [the HiGHS documentation]. The `dispatch_options` are only + /// used by the `system_dispatch` simulation protocol. /// /// [the HiGHS documentation]: https://ergo-code.github.io/HiGHS/stable/options/definitions/ pub highs: HighsOptions, @@ -139,6 +146,7 @@ impl Default for ModelParameters { // Default values for optional parameters currency: None, allow_dangerous_options: false, + simulation_protocol: SimulationProtocol::default(), candidate_asset_capacity: Capacity(1e-4), commodity_balance_epsilon: Flow(1e-6), capacity_tranche_fraction: Dimensionless(0.05), @@ -461,6 +469,26 @@ mod tests { assert_eq!(model_params.price_cycle_iterations, 3); } + #[rstest] + #[case("", Some(SimulationProtocol::SystemDispatch))] + #[case( + "simulation_protocol = \"system_dispatch\"", + Some(SimulationProtocol::SystemDispatch) + )] + #[case("simulation_protocol = \"unknown_protocol\"", None)] + fn model_params_simulation_protocol( + #[case] setting: &str, + #[case] expected: Option, + ) { + let result: Result = + toml::from_str(&format!("milestone_years = [2020, 2100]\n{setting}")); + + assert_eq!( + result.ok().map(|params| params.simulation_protocol), + expected + ); + } + #[test] fn model_params_currency_is_optional_metadata() { let with_currency: ModelParameters = toml::from_str( diff --git a/src/simulation.rs b/src/simulation.rs index 2390fadfe..973df66ec 100644 --- a/src/simulation.rs +++ b/src/simulation.rs @@ -5,6 +5,8 @@ use crate::output::DataWriter; use crate::process::ProcessMap; use crate::units::Capacity; use anyhow::Result; +use log::info; +use serde::Deserialize; use std::path::Path; use std::sync::Arc; @@ -15,6 +17,17 @@ pub mod prices; pub use prices::PriceMap; pub mod system_dispatch; +/// The protocol used to simulate each milestone year, selected with `simulation_protocol` in +/// `model.toml` +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, strum::Display)] +#[serde(rename_all = "snake_case")] +#[strum(serialize_all = "snake_case")] +pub enum SimulationProtocol { + /// Agents choose capacity; flows and prices come from a system-wide cost-minimising dispatch + #[default] + SystemDispatch, +} + /// Run the simulation. /// /// # Arguments: @@ -24,7 +37,11 @@ pub mod system_dispatch; /// * `debug_model` - Whether to write additional information (e.g. duals) to output files pub fn run(model: &Model, output_path: &Path, debug_model: bool) -> Result<()> { let mut writer = DataWriter::create(output_path, &model.model_path, debug_model)?; - system_dispatch::run(model, &mut writer)?; + let protocol = model.parameters.simulation_protocol; + info!("Simulation protocol: {protocol}"); + match protocol { + SimulationProtocol::SystemDispatch => system_dispatch::run(model, &mut writer)?, + } writer.flush() } From f6f3f1c01cc4ecc874c180c6a009b3aa5c30a233 Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Wed, 11 Nov 2026 15:34:22 +0000 Subject: [PATCH 06/11] Document simulation protocols Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/developer_guide/custom_highs_options.md | 5 +- docs/model/README.md | 94 ++++++++++++-------- docs/model/dispatch_optimisation.md | 2 + docs/release_notes/upcoming.md | 3 + schemas/input/model.yaml | 2 +- src/simulation.rs | 20 +++++ 6 files changed, 85 insertions(+), 41 deletions(-) diff --git a/docs/developer_guide/custom_highs_options.md b/docs/developer_guide/custom_highs_options.md index 186d55c2b..ffec31b3c 100644 --- a/docs/developer_guide/custom_highs_options.md +++ b/docs/developer_guide/custom_highs_options.md @@ -8,7 +8,9 @@ set `please_give_me_broken_results = true` in your [`model.toml` file][model.tom You can change any of the options exposed by the HiGHS solver; for more information, see [the HiGHS documentation][highs-opts]. -You can set options to be applied to all optimisations, just dispatch or just appraisal. +You can set options to be applied to all optimisations, just dispatch or just appraisal. Note that +dispatch options are only used by the `system_dispatch` [simulation protocol][protocols], which is +currently the only one available. Here is an example: @@ -33,4 +35,5 @@ optimality_tolerance = 10e-6 ``` [model.toml]: ../file_formats/input_files.md#model-parameters-modeltoml +[protocols]: ../model/README.md#simulation-protocols [highs-opts]: https://ergo-code.github.io/HiGHS/stable/options/definitions/ diff --git a/docs/model/README.md b/docs/model/README.md index 2819d1dd0..8117f15a4 100644 --- a/docs/model/README.md +++ b/docs/model/README.md @@ -53,45 +53,61 @@ At a high level, the user defines: ## Framework Overview The model operates sequentially across a series of milestone years (MSYs). For the base year, -existing assets are commissioned and a [dispatch optimisation][dispatch-optimisation] is run to -establish [commodity prices][prices] for the first investment year. For each subsequent MSY, the -model performs the following steps: - -### 1. Decommission end-of-life assets - -Assets whose scheduled decommissioning year has been reached are removed from the active pool. - -### 2. Agent investment - -Agents select assets to meet commodity demand for the current MSY. Investment decisions follow a -pre-computed **investment order** derived from the topology of the [commodity graph][commodity-graphs]: -markets are processed deepest-first (most downstream to most upstream), grouped into **layers** of -independent markets at the same depth. For each market, each responsible agent selects assets using -the algorithm for its [agent type][agents]. See [Investment][investment] for full details. - -After all markets in each layer are settled, a **partial system dispatch** is run over all assets -selected so far. This propagates demand upstream: input commodity flows consumed by newly committed -assets become demand targets for the markets not yet invested in. - -### 3. Mothballing and decommissioning - -Capacity of previously commissioned assets that was not selected for retention is mothballed. This -may be the whole of an asset or only some of its capacity. Any capacity that has remained mothballed -for `mothball_years` (defined in [`model.toml`][model-toml]) is permanently decommissioned. - -### 4. Ironing-out loop - -Investment and dispatch are repeated iteratively to resolve any price instability introduced by new -capacity commitments. Each iteration runs the full agent investment pass followed by a full system -dispatch. The loop terminates when time-slice-weighted market prices have converged within -`price_tolerance`, or when `max_ironing_out_iterations` is reached (both defined in -[`model.toml`][model-toml]). - -### 5. Final dispatch - -A full [dispatch optimisation][dispatch-optimisation] is run over the settled asset pool. The -resulting flows and commodity prices are written to the output files and carried forward as the -price basis for the next MSY's investment. +existing assets are commissioned and the model establishes the [commodity prices][prices] used as +the starting point for the first investment year. For each subsequent MSY, the model performs the +following steps: + +1. **Decommission end-of-life assets.** Assets whose scheduled decommissioning year has been + reached are removed from the active pool. + +2. **Agent investment.** Agents select assets to meet commodity demand for the current MSY. + Investment decisions follow a pre-computed **investment order** derived from the topology of the + [commodity graph][commodity-graphs]: markets are processed deepest-first (most downstream to most + upstream), grouped into **layers** of independent markets at the same depth. For each market, + each responsible agent selects assets using the algorithm for its [agent type][agents]. As markets + are settled, the commodity flows of the newly selected assets are used to propagate demand + upstream: input commodity flows consumed by newly committed assets become demand targets for the + markets not yet invested in. See [Investment][investment] for full details. + +3. **Ironing out.** Investment is repeated iteratively with updated prices to resolve any price + instability introduced by new capacity commitments. The loop terminates when time-slice-weighted + market prices have converged within `price_tolerance`, or when `max_ironing_out_iterations` is + reached (both defined in [`model.toml`][model-toml]). + +4. **Mothballing and decommissioning.** Capacity of previously commissioned assets that was not + selected for retention is mothballed. This may be the whole of an asset or only some of its + capacity. Any capacity that has remained mothballed for `mothball_years` (defined in + [`model.toml`][model-toml]) is permanently decommissioned. + +5. **Final flows and prices.** The commodity flows and prices for the MSY are calculated for the + settled asset pool, written to the output files, and carried forward as the price basis for the + next MSY's investment. + +How commodity flows and prices are obtained in each of these steps is determined by the +[simulation protocol](#simulation-protocols). + +## Simulation protocols + +The simulation protocol is selected with the `simulation_protocol` setting in +[`model.toml`][model-toml]. If it is not specified, `system_dispatch` is used. + +### `system_dispatch` + +In this protocol, agents choose capacity and the flows and prices for each year are obtained from a +system-wide [dispatch optimisation][dispatch-optimisation]. It works as follows: + +- **Base year:** Dispatch optimisations are run over the existing assets, with and without candidate + assets, to establish commodity flows and prices. +- **Agent investment:** After all markets in each layer are settled, a **partial system dispatch** is + run over all assets selected so far. For markets in a cycle, a partial dispatch is run after each + market is settled. This provides the flows used to propagate demand upstream. +- **Ironing out:** Each iteration runs the full agent investment pass followed by a full system + dispatch of the selected assets, from which updated prices are calculated. +- **Final dispatch:** A full dispatch optimisation is run over the settled asset pool. The + resulting flows and commodity prices are written to the output files. + +The `value_of_lost_load`, `commodity_balance_epsilon` and `highs.dispatch_options` settings in +[`model.toml`][model-toml] are only used by this protocol. [investment]: ./investment.md [agents]: ./agents.md diff --git a/docs/model/dispatch_optimisation.md b/docs/model/dispatch_optimisation.md index e50ea7ce5..8881722bd 100644 --- a/docs/model/dispatch_optimisation.md +++ b/docs/model/dispatch_optimisation.md @@ -5,6 +5,8 @@ milestone year (MSY), the dispatch model calculates the least-cost operation of energy system assets to satisfy commodity demands. It is solved as a Linear Programme (LP) using the HiGHS solver. +This formulation is used by the `system_dispatch` [simulation protocol](README.md#simulation-protocols). + ## Sets and Indices The energy system is defined using the following sets: diff --git a/docs/release_notes/upcoming.md b/docs/release_notes/upcoming.md index c507a3172..9b1c6b1b2 100644 --- a/docs/release_notes/upcoming.md +++ b/docs/release_notes/upcoming.md @@ -9,6 +9,9 @@ - Agents now have an agent type, set with the new optional `agent_type` column in `agents.csv`. The new `portfolio` agent type selects assets for each market by solving a single optimisation, rather than appraising options one at a time ([#1574]) +- A new optional `simulation_protocol` setting in `model.toml` selects how each milestone year is + simulated. The only value currently available is `system_dispatch`, which is the default and + corresponds to the existing behaviour ## Breaking changes diff --git a/schemas/input/model.yaml b/schemas/input/model.yaml index 1f750122e..8b3b10e41 100644 --- a/schemas/input/model.yaml +++ b/schemas/input/model.yaml @@ -29,7 +29,7 @@ properties: the energy system to produce commodity flows and prices for each milestone year. Currently the only option is `system_dispatch`, in which agents choose capacity and flows and prices come from a system-wide, cost-minimising dispatch optimisation. See - [the model documentation](../model/README.html#simulation-protocols) for details. + [the model documentation](../model/index.html#simulation-protocols) for details. candidate_asset_capacity: type: number description: Capacity to give to candidate assets in dispatch optimisation diff --git a/src/simulation.rs b/src/simulation.rs index 973df66ec..4201b4b9d 100644 --- a/src/simulation.rs +++ b/src/simulation.rs @@ -1,4 +1,24 @@ //! Functionality for running the MUSE2 simulation across milestone years. +//! +//! How each milestone year is simulated is determined by the [`SimulationProtocol`], selected with +//! the `simulation_protocol` setting in `model.toml`. Each protocol lives in its own submodule +//! (e.g. [`system_dispatch`]) and provides a `run` function which loops over the milestone years. +//! Protocols share the agent investment code in [`investment`] and [`market`], and the price +//! calculation code in [`prices`]. The only protocol-specific part of the agent investment loop is +//! how the flows of newly selected assets are determined, which protocols supply by implementing +//! [`investment::FlowEstimator`]. +//! +//! # Adding a new protocol +//! +//! 1. Add a variant to [`SimulationProtocol`], and a match arm in [`run`] +//! 2. Create a submodule with a `pub fn run(model: &Model, writer: &mut DataWriter) -> Result<()>` +//! function, and implement [`investment::FlowEstimator`] if it uses [`investment::run_agents_for_year`] +//! 3. If the protocol has its own debug output files, write them with +//! [`DataWriter::write_debug_rows`] +//! 4. Add the new value to the `simulation_protocol` entry in `schemas/input/model.yaml`, noting any +//! settings which are only used by the new protocol +//! 5. Document the protocol in `docs/model/README.md`, and add a release note +//! 6. Add regression tests for the protocol use crate::asset::{Asset, AssetRef}; use crate::model::Model; use crate::output::DataWriter; From 05283c5d0b51b50a3246c5fb2bb9b68f295c0925 Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Wed, 11 Nov 2026 15:38:39 +0000 Subject: [PATCH 07/11] Make shared price doc comments protocol-neutral Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/simulation/investment/competitive.rs | 2 +- src/simulation/investment/portfolio.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/simulation/investment/competitive.rs b/src/simulation/investment/competitive.rs index d5afd04c7..0236d77de 100644 --- a/src/simulation/investment/competitive.rs +++ b/src/simulation/investment/competitive.rs @@ -53,7 +53,7 @@ use std::sync::Arc; /// * `options` - Committed assets, existing assets and processes available to the agent /// * `agent_addition_limits` - The agent's share of addition limits for candidate processes /// * `agent_total_limits` - The agent's share of total capacity limits for processes -/// * `prices` - Commodity prices from the previous full system dispatch +/// * `prices` - Commodity prices calculated in the previous dispatch /// * `year` - Current milestone year /// * `debug_writer` - Optional debug writer /// diff --git a/src/simulation/investment/portfolio.rs b/src/simulation/investment/portfolio.rs index f118d4a4c..5bc92427d 100644 --- a/src/simulation/investment/portfolio.rs +++ b/src/simulation/investment/portfolio.rs @@ -41,7 +41,7 @@ use output::write_debug_info; /// * `options` - Committed assets, existing assets and processes available to the agent /// * `agent_addition_limits` - The agent's share of addition limits for candidate processes /// * `agent_total_limits` - The agent's share of total capacity limits for processes -/// * `prices` - Commodity prices from the previous full system dispatch +/// * `prices` - Commodity prices calculated in the previous dispatch /// * `year` - Current milestone year /// * `debug_writer` - Optional debug writer /// From e1f571f3f2153e06208b50e9731529ff5d423df4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci-lite[bot]" <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Date: Mon, 16 Nov 2026 09:00:31 +0000 Subject: [PATCH 08/11] [pre-commit.ci lite] apply automatic fixes --- src/simulation/system_dispatch.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/simulation/system_dispatch.rs b/src/simulation/system_dispatch.rs index f64e43873..c5fe449ef 100644 --- a/src/simulation/system_dispatch.rs +++ b/src/simulation/system_dispatch.rs @@ -27,10 +27,10 @@ use itertools::Itertools; use log::{debug, info}; use std::slice; -mod output; pub mod optimisation; -use output::create_debug_files; +mod output; use optimisation::{DispatchRun, Solution}; +use output::create_debug_files; /// Run the simulation using the `system_dispatch` protocol. /// From ecf145dacf35192019fc82af6ded4e1d08f9919e Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Mon, 16 Nov 2026 11:07:57 +0000 Subject: [PATCH 09/11] Fix docs --- src/simulation.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/simulation.rs b/src/simulation.rs index 4201b4b9d..2bf8cfc19 100644 --- a/src/simulation.rs +++ b/src/simulation.rs @@ -13,8 +13,9 @@ //! 1. Add a variant to [`SimulationProtocol`], and a match arm in [`run`] //! 2. Create a submodule with a `pub fn run(model: &Model, writer: &mut DataWriter) -> Result<()>` //! function, and implement [`investment::FlowEstimator`] if it uses [`investment::run_agents_for_year`] -//! 3. If the protocol has its own debug output files, write them with -//! [`DataWriter::write_debug_rows`] +//! 3. If the protocol has its own debug output files, write them using +//! [`crate::output::DebugDataWriter::get_custom_writer`], available through +//! [`DataWriter::debug_writer`] //! 4. Add the new value to the `simulation_protocol` entry in `schemas/input/model.yaml`, noting any //! settings which are only used by the new protocol //! 5. Document the protocol in `docs/model/README.md`, and add a release note From 23cee76e671ea88430582198b0f1071c2177e5d7 Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Mon, 16 Nov 2026 15:32:08 +0000 Subject: [PATCH 10/11] Shotren comment --- src/simulation.rs | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/simulation.rs b/src/simulation.rs index 2bf8cfc19..3db339b2a 100644 --- a/src/simulation.rs +++ b/src/simulation.rs @@ -7,19 +7,6 @@ //! calculation code in [`prices`]. The only protocol-specific part of the agent investment loop is //! how the flows of newly selected assets are determined, which protocols supply by implementing //! [`investment::FlowEstimator`]. -//! -//! # Adding a new protocol -//! -//! 1. Add a variant to [`SimulationProtocol`], and a match arm in [`run`] -//! 2. Create a submodule with a `pub fn run(model: &Model, writer: &mut DataWriter) -> Result<()>` -//! function, and implement [`investment::FlowEstimator`] if it uses [`investment::run_agents_for_year`] -//! 3. If the protocol has its own debug output files, write them using -//! [`crate::output::DebugDataWriter::get_custom_writer`], available through -//! [`DataWriter::debug_writer`] -//! 4. Add the new value to the `simulation_protocol` entry in `schemas/input/model.yaml`, noting any -//! settings which are only used by the new protocol -//! 5. Document the protocol in `docs/model/README.md`, and add a release note -//! 6. Add regression tests for the protocol use crate::asset::{Asset, AssetRef}; use crate::model::Model; use crate::output::DataWriter; From 88a331177f986a098c08716806330a9919eb4d65 Mon Sep 17 00:00:00 2001 From: Tom Bland Date: Tue, 17 Nov 2026 14:31:23 +0000 Subject: [PATCH 11/11] Fix links --- docs/developer_guide/custom_highs_options.md | 2 +- docs/model/dispatch_optimisation.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/developer_guide/custom_highs_options.md b/docs/developer_guide/custom_highs_options.md index ffec31b3c..83f0c9bd9 100644 --- a/docs/developer_guide/custom_highs_options.md +++ b/docs/developer_guide/custom_highs_options.md @@ -35,5 +35,5 @@ optimality_tolerance = 10e-6 ``` [model.toml]: ../file_formats/input_files.md#model-parameters-modeltoml -[protocols]: ../model/README.md#simulation-protocols +[protocols]: ../model/index.html#simulation-protocols [highs-opts]: https://ergo-code.github.io/HiGHS/stable/options/definitions/ diff --git a/docs/model/dispatch_optimisation.md b/docs/model/dispatch_optimisation.md index 8881722bd..7ff88500f 100644 --- a/docs/model/dispatch_optimisation.md +++ b/docs/model/dispatch_optimisation.md @@ -5,7 +5,7 @@ milestone year (MSY), the dispatch model calculates the least-cost operation of energy system assets to satisfy commodity demands. It is solved as a Linear Programme (LP) using the HiGHS solver. -This formulation is used by the `system_dispatch` [simulation protocol](README.md#simulation-protocols). +This formulation is used by the `system_dispatch` [simulation protocol](index.html#simulation-protocols). ## Sets and Indices