diff --git a/.github/workflows/test-notebooks.yml b/.github/workflows/test-notebooks.yml index dfe025d2..2a5cec6e 100644 --- a/.github/workflows/test-notebooks.yml +++ b/.github/workflows/test-notebooks.yml @@ -40,8 +40,8 @@ jobs: # Skip notebooks that require credentials or special setup case "$name" in - solve-on-oetc.ipynb|solve-on-remote.ipynb) - echo "Skipping $name (requires credentials or special setup)" + remote-machines.ipynb) + echo "Skipping $name (requires credentials or remote machine)" continue ;; esac diff --git a/doc/api.rst b/doc/api.rst index f0afc322..b9876f68 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -70,6 +70,8 @@ Modifying a model model.Model.remove_objective model.Model.remove_sos_constraints model.Model.copy + model.Model.apply_sos_reformulation + model.Model.undo_sos_reformulation model.Model.reformulate_sos_constraints Solving @@ -498,10 +500,75 @@ Type aliases Solvers ======== +The stateful :class:`~linopy.solvers.Solver` instance owns the solver-side +model and exposes a two-step :meth:`~linopy.solvers.Solver.from_name` / +:meth:`~linopy.solvers.Solver.solve` workflow. :meth:`Model.solve` is a +thin wrapper around it. + +.. autosummary:: + :toctree: generated/ + + solvers.Solver + +Construction +------------ + +.. autosummary:: + :toctree: generated/ + + solvers.Solver.from_name + solvers.Solver.from_model + +Solving +------- + +.. autosummary:: + :toctree: generated/ + + solvers.Solver.solve + solvers.Solver.update_solver_model + solvers.Solver.close + +Post-solve state +---------------- + +.. autosummary:: + :toctree: generated/ + + solvers.Solver.status + solvers.Solver.solution + solvers.Solver.report + solvers.Solver.solver_model + +Capabilities +------------ + +.. autosummary:: + :toctree: generated/ + + solvers.Solver.is_available + solvers.Solver.license_status + solvers.Solver.supports + solvers.Solver.supported_features + solvers.Solver.runtime_features + +Discovery +--------- + .. autosummary:: :toctree: generated/ solvers.available_solvers + solvers.licensed_solvers + solvers.SolverFeature + solvers.LicenseStatus + +Implementations +--------------- + +.. autosummary:: + :toctree: generated/ + solvers.CBC solvers.COPT solvers.Cplex @@ -519,17 +586,29 @@ Solvers Remote solving ============== +Solve a model on a remote machine via SSH or on the OET Cloud (OETC). +See :doc:`remote-machines` for usage. + .. autosummary:: :toctree: generated/ + remote.SSH + remote.SshSettings + remote.Oetc + remote.OetcSettings remote.RemoteHandler + remote.OetcHandler + remote.OetcCredentials Solver status and result types ============================== Types returned by or compared against :attr:`Model.status`, -:attr:`Model.termination_condition`, and :attr:`Model.solution`. +:attr:`Model.termination_condition`, and :attr:`Model.solution`, plus +:class:`~linopy.constants.SolverReport` surfaced on +:attr:`Solver.report ` and +:attr:`Result.report `. .. autosummary:: :toctree: generated/ @@ -538,6 +617,7 @@ Types returned by or compared against :attr:`Model.status`, constants.TerminationCondition constants.Status constants.Solution + constants.SolverReport constants.Result diff --git a/doc/benchmark.rst b/doc/benchmark.rst deleted file mode 100644 index f56d5dff..00000000 --- a/doc/benchmark.rst +++ /dev/null @@ -1,27 +0,0 @@ -.. _benchmark: - -Performance comparison -====================== - - -Linopy's performance scales well with the problem size. Its overall speed is comparable with the famous `JuMP `_ package written in `Julia `_. It even outperforms `JuMP` in total memory efficiency when it comes to large models. Compared to `Pyomo `_, the common optimization package in python, one can expect - -* a **speedup of times 4-6** -* a **memory reduction of roughly 50%** - -for large problems. The following figure shows the memory usage and speed for solving the problem - -.. math:: - - & \min \;\; \sum_{i,j} 2 x_{i,j} + y_{i,j} \\ - s.t. & \\ - & x_{i,j} - y_{i,j} \; \ge \; i-1 \qquad \forall \; i,j \in \{1,...,N\} \\ - & x_{i,j} + y_{i,j} \; \ge \; 0 \qquad \forall \; i,j \in \{1,...,N\} - - -with the different API's using the `Gurobi `_ solver. The workflow, that produces the figure, can be found `here `_. - -.. image:: benchmark.png - :width: 1500 - :alt: benchmark - :align: center diff --git a/doc/conf.py b/doc/conf.py index c6b3b90c..b028e589 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -123,6 +123,7 @@ html_theme_options = { "repository_url": "https://github.com/pypsa/linopy", "use_repository_button": True, + "show_toc_level": 2, } diff --git a/doc/create-a-model-with-coordinates.nblink b/doc/create-a-model-with-coordinates.nblink deleted file mode 100644 index fc84769f..00000000 --- a/doc/create-a-model-with-coordinates.nblink +++ /dev/null @@ -1,3 +0,0 @@ -{ - "path": "../examples/create-a-model-with-coordinates.ipynb" -} diff --git a/doc/create-a-model.nblink b/doc/create-a-model.nblink deleted file mode 100644 index 0c96b54f..00000000 --- a/doc/create-a-model.nblink +++ /dev/null @@ -1,3 +0,0 @@ -{ - "path": "../examples/create-a-model.ipynb" -} diff --git a/doc/first-real-model.nblink b/doc/first-real-model.nblink new file mode 100644 index 00000000..cf7eae2c --- /dev/null +++ b/doc/first-real-model.nblink @@ -0,0 +1,3 @@ +{ + "path": "../examples/first-real-model.ipynb" +} diff --git a/doc/gpu-acceleration.rst b/doc/gpu-acceleration.rst index 2498993a..46cf368a 100644 --- a/doc/gpu-acceleration.rst +++ b/doc/gpu-acceleration.rst @@ -1,6 +1,6 @@ -======================== -GPU-Accelerated Solving -======================== +================ +GPU acceleration +================ .. warning:: @@ -8,7 +8,7 @@ GPU-Accelerated Solving Linopy supports GPU-accelerated optimization solvers that can significantly speed up solving large-scale linear programming problems by leveraging the parallel processing capabilities of modern GPUs. -Supported GPU Solvers +Supported GPU solvers ===================== cuPDLPx @@ -42,7 +42,7 @@ To install it, you have to have the `CUDA Toolkit `_. -Xpress with GPU Acceleration +Xpress with GPU acceleration ----------------------------- `FICO Xpress `_ version 9.8 and later includes GPU acceleration support for certain operations. @@ -56,7 +56,7 @@ Xpress with GPU Acceleration Prerequisites ============= -Hardware Requirements +Hardware requirements --------------------- GPU solvers require: @@ -65,13 +65,13 @@ GPU solvers require: - Sufficient GPU memory for your problem size (varies by problem) - PCIe 3.0 or higher for optimal data transfer -Software Requirements +Software requirements --------------------- 1. **CUDA Toolkit**: Most GPU solvers require CUDA 11.0 or later 2. **Compatible GPU drivers**: Match your CUDA version -Verifying Installation +Verifying installation ====================== To verify that the GPU solvers are properly installed and detected: diff --git a/doc/gurobi-double-logging.rst b/doc/gurobi-double-logging.rst deleted file mode 100644 index c53f6866..00000000 --- a/doc/gurobi-double-logging.rst +++ /dev/null @@ -1,26 +0,0 @@ - -======================== -Double logging in gurobi -======================== - -When using the gurobi solver you may see some of the logs generated by gurobi during the solve -are duplicated. - -e.g. - -.. code-block:: bash - - Total elapsed time = 498.27s - [INFO] Total elapsed time = 498.27s - -This is because the gurobi logger both prints to the console and propagates to the root logger. - -Adding the following to your application code before call solve should fix the issue. - -.. code-block:: python - - logger = logging.getLogger("gurobipy") - logger.propagate = False - - -.. _Further information: https://groups.google.com/g/gurobi/c/sV7xxN_mzCk diff --git a/doc/index.rst b/doc/index.rst index 39846607..a74ba885 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -98,69 +98,61 @@ This package is published under MIT license. :maxdepth: 2 :caption: Getting Started - prerequisites - create-a-model - create-a-model-with-coordinates + installation + quick-start + first-real-model .. toctree:: :hidden: :maxdepth: 2 - :caption: User Guide + :caption: Creating a Model - user-guide creating-variables creating-expressions creating-constraints coordinate-alignment - manipulating-models - -.. toctree:: - :hidden: - :maxdepth: 2 - :caption: Examples - - transport-tutorial - migrating-from-pyomo .. toctree:: :hidden: :maxdepth: 2 - :caption: Advanced Features + :caption: Solving a Model - sos-constraints - piecewise-linear-constraints - testing-framework + solving + solver-api + remote-machines + gpu-acceleration + solving-troubleshooting .. toctree:: :hidden: :maxdepth: 2 - :caption: Solving + :caption: Modifying a Model - solve-on-remote - solve-on-oetc - gpu-acceleration + manipulating-models .. toctree:: :hidden: :maxdepth: 2 - :caption: Troubleshooting + :caption: Advanced Formulations - infeasible-model - gurobi-double-logging + sos-constraints + piecewise-linear-constraints .. toctree:: :hidden: :maxdepth: 2 - :caption: Comparisons + :caption: Coming from Other Tools - benchmark - syntax + tool-comparison + migrating-from-pyomo + transport-tutorial .. toctree:: :hidden: :maxdepth: 2 :caption: Reference + testing-framework api release_notes contributing diff --git a/doc/infeasible-model.nblink b/doc/infeasible-model.nblink deleted file mode 100644 index e989b4bd..00000000 --- a/doc/infeasible-model.nblink +++ /dev/null @@ -1,3 +0,0 @@ -{ - "path": "../examples/infeasible-model.ipynb" -} diff --git a/doc/prerequisites.rst b/doc/installation.rst similarity index 74% rename from doc/prerequisites.rst rename to doc/installation.rst index 7a9e83bd..5e7d78b0 100644 --- a/doc/prerequisites.rst +++ b/doc/installation.rst @@ -1,7 +1,7 @@ -Getting Started -=============== +Installation +============ -This guide will provide you with the necessary steps to get started with Linopy, from installation to creating your first model and beyond. +This page covers everything you need to install Linopy and the solver(s) you want to use. Once you have it installed, head to the :doc:`quick-start` for your first model. Before you start, make sure you have the following: @@ -30,9 +30,6 @@ Install a solver Linopy won't work without a solver. Currently, the following solvers are supported: -CPU-based solvers -~~~~~~~~~~~~~~~~~ - - `Cbc `__ - open source, free, fast - `GLPK `__ - open source, free, not very fast - `HiGHS `__ - open source, free, fast @@ -58,23 +55,12 @@ We recommend the HiGHS solver, which is free, open source, and fast across a wide range of problem sizes. It is included in both the ``solvers`` and ``dev`` extras. - -GPU-accelerated solvers -~~~~~~~~~~~~~~~~~~~~~~~ - -For large-scale optimization problems, GPU-accelerated solvers can provide significant performance improvements: - -- `cuPDLPx `__ - open source, GPU-accelerated first-order solver - -**Note:** GPU solvers require compatible NVIDIA GPU hardware and CUDA installation. See the :doc:`gpu-acceleration` guide for detailed setup instructions. - -.. code:: bash - - uv pip install cupdlpx - - For most of the other solvers, please click on the links to get further installation information. +GPU-accelerated solving (cuPDLPx, and Xpress in v9.8+) can speed up +large-scale problems considerably; setup is covered in the +:doc:`gpu-acceleration` guide. + If you're ready, let's dive in! diff --git a/doc/quick-start.nblink b/doc/quick-start.nblink new file mode 100644 index 00000000..c48cd4db --- /dev/null +++ b/doc/quick-start.nblink @@ -0,0 +1,3 @@ +{ + "path": "../examples/quick-start.ipynb" +} diff --git a/doc/release_notes.rst b/doc/release_notes.rst index e5b7033f..53d7cdcc 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -46,9 +46,39 @@ Most users should keep calling ``model.solve(...)``. If you want more control, y * Xpress now supports ``io_api="direct"``: the linopy model is loaded via the native ``loadproblem`` array API instead of being serialised through an LP/MPS file, with SOS constraints attached in-place. Adds ``model.to_xpress()`` matching the existing ``to_gurobipy`` / ``to_highspy`` / ``to_mosek`` helpers. * Writing the solution back to the model after solving is faster: it no longer rebuilds the constraint matrix, and now uses positional (rather than label-based) indexing — roughly 2× faster overall. +*Remote solves* + +* Pass ``remote=`` to ``Model.solve`` to run the inner solver on a remote worker: + + .. code-block:: python + + m.solve("gurobi", remote=OetcSettings(...), Method=2) + m.solve("highs", remote=SshSettings(hostname=...), presolve="on") + + ``solver_name`` and ``**solver_options`` work the same as for local solves; ``remote=`` selects *where* to run. After the call, ``model.remote`` holds the remote instance (mirrors :attr:`Model.solver`). +* ``SshSettings.setup_commands: list[str]`` — shell commands run on the remote before the solve, e.g. ``setup_commands=["conda activate linopy-env"]``. + **Deprecations** * ``Solver.solve_problem``, ``Solver.solve_problem_from_model``, and ``Solver.solve_problem_from_file`` still work but emit a ``DeprecationWarning``. Use ``Solver.from_name(...).solve()`` (or simply ``model.solve(...)``) instead. They will be removed in a future release. +* ``linopy.remote.OetcHandler`` and ``linopy.remote.RemoteHandler`` are deprecated. Construction emits a ``DeprecationWarning``; the ``solve_on_oetc`` / ``solve_on_remote`` return contracts are unchanged. Migrate: + + .. code-block:: python + + # Before + handler = OetcHandler( + OetcSettings(credentials=OetcCredentials(email=..., password=...), ...) + ) + solved = handler.solve_on_oetc(m, TimeLimit=100) + + # After + m.solve( + "gurobi", remote=OetcSettings(email=..., password=..., ...), TimeLimit=100 + ) + + Passing an existing handler via ``Model.solve(remote=handler, ...)`` is also deprecated — pass the settings dataclass instead. +* ``linopy.remote.OetcCredentials`` is deprecated. Pass ``email`` and ``password`` directly to :class:`OetcSettings` instead of wrapping them. The ``OetcSettings(credentials=OetcCredentials(...))`` shape still works for one deprecation cycle and emits a ``DeprecationWarning``. +* :class:`linopy.remote.SSH` only exposes ``solve(model)``. For env activation use ``SshSettings.setup_commands``; for arbitrary remote shell commands, drop to :class:`RemoteHandler` (during deprecation) or paramiko directly. **Bug Fixes** @@ -60,6 +90,7 @@ Most users should keep calling ``model.solve(...)``. If you want more control, y * ``available_solvers`` now lists all *installed* solvers, even ones without a working license. If you used it to decide "can I actually solve with X?", switch to ``linopy.licensed_solvers`` or ``SolverClass.license_status()``. * ``Model.solver_model`` and ``Model.solver_name`` are now read-only properties that delegate to ``model.solver``. You can't reassign them (only ``= None`` is allowed, which closes the solver), and ``solver_name`` is ``None`` before the first solve. * ``result.solution.primal`` and ``result.solution.dual`` are now ``numpy`` arrays indexed by linopy's integer labels (with ``NaN`` for slots without a value), instead of pandas Series keyed by variable/constraint name. If you accessed them by name, use ``model.variables[name].solution`` (or ``model.constraints[name].dual``) instead. +* The pip extra ``linopy[remote]`` has been renamed to ``linopy[ssh]`` to match what it installs (only ``paramiko``, for SSH transport — OETC has its own ``linopy[oetc]`` extra). ``linopy[remote]`` no longer exists; update your install commands. **Internal** @@ -68,6 +99,12 @@ Most users should keep calling ``model.solve(...)``. If you want more control, y * ``linopy.common`` gains ``values_to_lookup_array``; the legacy pandas-based helpers ``series_to_lookup_array`` and ``lookup_vals`` are removed. ``model.to_gurobipy()`` / ``model.to_highspy()`` / ``to_cupdlpx(model)`` (and similar) all return the underlying solver model as before; internally they now go through ``Solver.from_model(model, io_api="direct")``. No user-visible change. +**Documentation** + +* The guide has been reorganised into a clearer path — *Getting Started*, *Creating a Model*, *Modifying a Model*, *Solving a Model*, *Advanced Formulations*, *Coming from Other Tools* — with a rewritten quick start and a new economic-dispatch walkthrough (*Your First Real Model*). +* New *Coming from Other Tools* section: a *Benchmarks and syntax* page comparing linopy with JuMP, Pyomo, and GAMS; a runnable *Migrating from Pyomo* notebook for the rule-function modelling pattern; and the GAMS transport tutorial. +* The *Using Solvers* guide is split into a task-focused *Solving a model* page and a lower-level *Solver API* reference. + Version 0.7.0 ------------- diff --git a/doc/remote-machines.nblink b/doc/remote-machines.nblink new file mode 100644 index 00000000..f273fb0c --- /dev/null +++ b/doc/remote-machines.nblink @@ -0,0 +1,3 @@ +{ + "path": "../examples/remote-machines.ipynb" +} diff --git a/doc/solve-on-oetc.nblink b/doc/solve-on-oetc.nblink deleted file mode 100644 index ab7ed00c..00000000 --- a/doc/solve-on-oetc.nblink +++ /dev/null @@ -1,3 +0,0 @@ -{ - "path": "../examples/solve-on-oetc.ipynb" -} diff --git a/doc/solve-on-remote.nblink b/doc/solve-on-remote.nblink deleted file mode 100644 index 03be52c0..00000000 --- a/doc/solve-on-remote.nblink +++ /dev/null @@ -1,3 +0,0 @@ -{ - "path": "../examples/solve-on-remote.ipynb" -} diff --git a/doc/solver-api.nblink b/doc/solver-api.nblink new file mode 100644 index 00000000..5af2e746 --- /dev/null +++ b/doc/solver-api.nblink @@ -0,0 +1,3 @@ +{ + "path": "../examples/solver-api.ipynb" +} diff --git a/doc/solving-troubleshooting.nblink b/doc/solving-troubleshooting.nblink new file mode 100644 index 00000000..b1571d01 --- /dev/null +++ b/doc/solving-troubleshooting.nblink @@ -0,0 +1,3 @@ +{ + "path": "../examples/solving-troubleshooting.ipynb" +} diff --git a/doc/solving.nblink b/doc/solving.nblink new file mode 100644 index 00000000..7400f805 --- /dev/null +++ b/doc/solving.nblink @@ -0,0 +1,3 @@ +{ + "path": "../examples/solving.ipynb" +} diff --git a/doc/sos-constraints.rst b/doc/sos-constraints.rst index caa4b5e5..1fbf21c6 100644 --- a/doc/sos-constraints.rst +++ b/doc/sos-constraints.rst @@ -268,12 +268,13 @@ as binary + linear constraints using the Big-M method. .. code-block:: python - # Automatic reformulation during solve + # Automatic reformulation during solve (apply / undo bracketed by Model.solve) m.solve(solver_name="highs", reformulate_sos=True) - # Or reformulate manually - m.reformulate_sos_constraints() + # Or stage the reformulation manually — e.g. to inspect or export the MILP + m.apply_sos_reformulation() m.solve(solver_name="highs") + m.undo_sos_reformulation() **Requirements:** @@ -392,4 +393,4 @@ See Also - :doc:`creating-variables`: Creating variables with coordinates - :doc:`creating-constraints`: Adding regular constraints -- :doc:`user-guide`: General linopy usage patterns +- :doc:`piecewise-linear-constraints`: Building piecewise-linear constraints (one of its methods uses SOS2 internally) diff --git a/doc/syntax.rst b/doc/syntax.rst deleted file mode 100644 index 38370962..00000000 --- a/doc/syntax.rst +++ /dev/null @@ -1,82 +0,0 @@ - -Syntax comparison -================= - -In order to compare the syntax between different API's, let's initialize the following problem in the different API's: - -.. math:: - - & \min \;\; \sum_{i,j} 2 x_{i,j} + \; y_{i,j} \\ - s.t. & \\ - & x_{i,j} - y_{i,j} \; \ge \; i-1 \qquad \forall \; i,j \in \{1,...,N\} \\ - & x_{i,j} + y_{i,j} \; \ge \; 0 \qquad \forall \; i,j \in \{1,...,N\} - - - - - -In ``JuMP`` the formulation translates to the following code: - - .. code-block:: julia - - using JuMP - - function create_model(N) - m = Model() - @variable(m, x[1:N, 1:N]) - @variable(m, y[1:N, 1:N]) - @constraint(m, x - y .>= 0:(N-1)) - @constraint(m, x + y .>= 0) - @objective(m, Min, 2 * sum(x) + sum(y)) - return m - end - -The same model in ``linopy`` is initialized by - - .. code-block:: python - - from linopy import Model - from numpy import arange - - - def create_model(N): - m = Model() - x = m.add_variables(coords=[arange(N), arange(N)]) - y = m.add_variables(coords=[arange(N), arange(N)]) - m.add_constraints(x - y >= arange(N)) - m.add_constraints(x + y >= 0) - m.add_objective((2 * x).sum() + y.sum()) - return m - -Note that the syntax is quite similar. - -In ``Pyomo`` the code would look like - - .. code-block:: python - - from numpy import arange - from pyomo.environ import ConcreteModel, Constraint, Objective, Set, Var - - - def create_model(N): - m = ConcreteModel() - m.N = Set(initialize=arange(N)) - - m.x = Var(m.N, m.N, bounds=(None, None)) - m.y = Var(m.N, m.N, bounds=(None, None)) - - def bound1(m, i, j): - return m.x[(i, j)] - m.y[(i, j)] >= i - - def bound2(m, i, j): - return m.x[(i, j)] + m.y[(i, j)] >= 0 - - def objective(m): - return sum(2 * m.x[(i, j)] + m.y[(i, j)] for i in m.N for j in m.N) - - m.con1 = Constraint(m.N, m.N, rule=bound1) - m.con2 = Constraint(m.N, m.N, rule=bound2) - m.obj = Objective(rule=objective) - return m - -which is heavily based on the internal call of functions in order to define the constraints. diff --git a/doc/tool-comparison.rst b/doc/tool-comparison.rst new file mode 100644 index 00000000..df68ab70 --- /dev/null +++ b/doc/tool-comparison.rst @@ -0,0 +1,118 @@ +.. _tool-comparison: + +Benchmarks and syntax +===================== + +This page is for users arriving from JuMP, Pyomo, or GAMS. It reports linopy's +performance against those tools and compares the API surface on a common toy +problem. + +- **Porting a Pyomo model?** See :doc:`migrating-from-pyomo` — linopy supports + Pyomo's rule-function pattern directly. +- **Coming from GAMS?** See :doc:`transport-tutorial` — the classic GAMS + transport problem worked in linopy alongside the original GAMS syntax. + + +Performance +----------- + +linopy's performance scales well with problem size, comparable to +`JuMP `_ on speed and ahead of it on memory efficiency for +large models. Against `Pyomo `_, linopy typically delivers: + +* a **speedup of 4–6×** +* a **memory reduction of roughly 50%** + +The figure below shows memory usage and build time on the toy problem from the +next section, solved with the `Gurobi `_ solver. The +benchmark workflow is +`available here `_. + +.. image:: benchmark.png + :width: 1500 + :alt: benchmark + :align: center + + +Syntax cheatsheet +----------------- + +The benchmark above solves this toy problem: + +.. math:: + + & \min \;\; \sum_{i,j} 2 x_{i,j} + \; y_{i,j} \\ + s.t. & \\ + & x_{i,j} - y_{i,j} \; \ge \; i-1 \qquad \forall \; i,j \in \{1,...,N\} \\ + & x_{i,j} + y_{i,j} \; \ge \; 0 \qquad \forall \; i,j \in \{1,...,N\} + +Here is the same problem in each tool. + +**JuMP** (Julia): + +.. code-block:: julia + + using JuMP + + function create_model(N) + m = Model() + @variable(m, x[1:N, 1:N]) + @variable(m, y[1:N, 1:N]) + @constraint(m, x - y .>= 0:(N-1)) + @constraint(m, x + y .>= 0) + @objective(m, Min, 2 * sum(x) + sum(y)) + return m + end + +**linopy** (Python): + +.. code-block:: python + + from linopy import Model + from numpy import arange + + + def create_model(N): + m = Model() + x = m.add_variables(coords=[arange(N), arange(N)]) + y = m.add_variables(coords=[arange(N), arange(N)]) + m.add_constraints(x - y >= arange(N)) + m.add_constraints(x + y >= 0) + m.add_objective((2 * x).sum() + y.sum()) + return m + +The linopy and JuMP formulations are close in spirit: both rely on broadcasting +and array-style operations rather than explicit per-element loops. + +**Pyomo** (Python): + +.. code-block:: python + + from numpy import arange + from pyomo.environ import ConcreteModel, Constraint, Objective, Set, Var + + + def create_model(N): + m = ConcreteModel() + m.N = Set(initialize=arange(N)) + + m.x = Var(m.N, m.N, bounds=(None, None)) + m.y = Var(m.N, m.N, bounds=(None, None)) + + def bound1(m, i, j): + return m.x[(i, j)] - m.y[(i, j)] >= i + + def bound2(m, i, j): + return m.x[(i, j)] + m.y[(i, j)] >= 0 + + def objective(m): + return sum(2 * m.x[(i, j)] + m.y[(i, j)] for i in m.N for j in m.N) + + m.con1 = Constraint(m.N, m.N, rule=bound1) + m.con2 = Constraint(m.N, m.N, rule=bound2) + m.obj = Objective(rule=objective) + return m + +Pyomo builds constraints from element-wise rule functions rather than +vectorised expressions. linopy supports that shape too — see +:doc:`migrating-from-pyomo`. diff --git a/doc/user-guide.rst b/doc/user-guide.rst deleted file mode 100644 index 8b7ee5bd..00000000 --- a/doc/user-guide.rst +++ /dev/null @@ -1,60 +0,0 @@ -.. _user-guide: - -Overview -======== - -In :doc:`Getting Started ` you installed linopy, built -a first scalar model, and saw N-D variables on coordinates. The User -Guide reopens each of those pieces in depth and adds the rest of the -modelling surface. - -Each page is a runnable Jupyter notebook — read it top to bottom, or -use it as a reference once you know what you're looking for. - - -Core building blocks --------------------- - -The four notebooks below cover the model object you'll interact with -most. Read them in order the first time; come back to them whenever -you're unsure what a particular operator or argument does. - -- :doc:`creating-variables` — declaring decision variables, with bounds - and coordinates. Continuous, integer, binary, and semi-continuous. -- :doc:`creating-expressions` — combining variables into linear (and - quadratic) expressions; arithmetic, broadcasting, ``sum``, - ``groupby``, ``rolling``, ``where``. -- :doc:`creating-constraints` — turning expressions into ``≤`` / ``≥`` - / ``==`` constraints, and the ``CSRConstraint`` memory-efficient - alternative. -- :doc:`coordinate-alignment` — how linopy lines up operands that live - on different coordinates, and how to control it with ``join``. - -After these four you can build any LP/MIP/QP linopy supports. - - -Working with an existing model ------------------------------- - -Once you've built a model, you'll often want to inspect it, change a -bound, swap a constraint, or copy it for what-if analysis. - -- :doc:`manipulating-models` — modifying or removing variables and - constraints in place; ``Model.copy()``; ``fix`` / ``relax`` for - variables. - - -Where to go next ----------------- - -- **Examples** — end-to-end problem walkthroughs: - :doc:`transport-tutorial`, :doc:`migrating-from-pyomo`. -- **Advanced features** — :doc:`sos-constraints`, - :doc:`piecewise-linear-constraints`, and the - :doc:`testing-framework` for asserting structural properties of a - model. -- **Solving** — :doc:`solve-on-remote` (SSH), - :doc:`solve-on-oetc` (OET Cloud), :doc:`gpu-acceleration` (cuPDLPx). -- **Troubleshooting** — :doc:`infeasible-model` (diagnosing infeasible - problems), :doc:`gurobi-double-logging` (and other solver quirks). -- **Reference** — the full :doc:`api` listing. diff --git a/examples/coordinate-alignment.ipynb b/examples/coordinate-alignment.ipynb index 1547bd9d..8e03daf4 100644 --- a/examples/coordinate-alignment.ipynb +++ b/examples/coordinate-alignment.ipynb @@ -6,13 +6,27 @@ "source": [ "# Coordinate Alignment\n", "\n", - "Since linopy builds on xarray, coordinate alignment matters when combining variables or expressions that live on different coordinates. By default, linopy aligns operands automatically and fills missing entries with sensible defaults. This guide shows how alignment works and how to control it with the ``join`` parameter." + "linopy builds on xarray, so whenever you combine two variables or expressions their coordinates have to be lined up first. Most of the time linopy does this automatically and the result is exactly what you'd expect — :doc:`creating-expressions` and :doc:`creating-constraints` rely on it without comment. This guide is the reference for *how* that automatic alignment works, and for the one case where the default can surprise you.\n", + "\n", + "The automatic behaviour has two modes, depending on whether the operands have the **same shape** along the shared dimension:\n", + "\n", + "- **Different shapes** → linopy keeps the union of coordinates and fills the gaps — zeros for arithmetic, `NaN` (i.e. \"skip this row\") for a constraint's right-hand side.\n", + "- **Same shape** → linopy aligns by **position** and ignores the coordinate labels. Fast, but it means two operands with mismatched labels get combined silently.\n", + "\n", + "When the default isn't what you want, the `.add()` / `.sub()` / `.mul()` / `.div()` methods — and `.le()` / `.ge()` / `.eq()` for constraints — take an explicit `join` parameter. The rest of this guide walks through both." ] }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:08:36.836203Z", + "iopub.status.busy": "2026-05-20T07:08:36.835994Z", + "iopub.status.idle": "2026-05-20T07:08:37.547368Z", + "shell.execute_reply": "2026-05-20T07:08:37.547161Z" + } + }, "outputs": [], "source": [ "import numpy as np\n", @@ -26,7 +40,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Default Alignment Behavior\n", + "## Default alignment behavior\n", "\n", "When two operands share a dimension but have different coordinates, linopy keeps the **larger** (superset) coordinate range and fills missing positions with zeros (for addition) or zero coefficients (for multiplication)." ] @@ -34,7 +48,14 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:08:37.549229Z", + "iopub.status.busy": "2026-05-20T07:08:37.549044Z", + "iopub.status.idle": "2026-05-20T07:08:37.728449Z", + "shell.execute_reply": "2026-05-20T07:08:37.728210Z" + } + }, "outputs": [], "source": [ "m = linopy.Model()\n", @@ -56,7 +77,14 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:08:37.729882Z", + "iopub.status.busy": "2026-05-20T07:08:37.729751Z", + "iopub.status.idle": "2026-05-20T07:08:37.738224Z", + "shell.execute_reply": "2026-05-20T07:08:37.738016Z" + } + }, "outputs": [], "source": [ "x + y" @@ -72,7 +100,14 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:08:37.752284Z", + "iopub.status.busy": "2026-05-20T07:08:37.752191Z", + "iopub.status.idle": "2026-05-20T07:08:37.755950Z", + "shell.execute_reply": "2026-05-20T07:08:37.755732Z" + } + }, "outputs": [], "source": [ "factor = xr.DataArray([2, 3, 4], dims=[\"time\"], coords={\"time\": [0, 1, 2]})\n", @@ -89,7 +124,14 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:08:37.756894Z", + "iopub.status.busy": "2026-05-20T07:08:37.756833Z", + "iopub.status.idle": "2026-05-20T07:08:37.761095Z", + "shell.execute_reply": "2026-05-20T07:08:37.760916Z" + } + }, "outputs": [], "source": [ "x + factor" @@ -99,15 +141,22 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Constraints with Subset RHS\n", + "### Constraints with a subset RHS\n", "\n", - "For constraints, missing right-hand-side values are filled with ``NaN``, which tells linopy to **skip** the constraint at those positions:" + "For constraints, missing right-hand-side values are filled with `NaN`, which tells linopy to **skip** the constraint at those positions:" ] }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:08:37.762036Z", + "iopub.status.busy": "2026-05-20T07:08:37.761974Z", + "iopub.status.idle": "2026-05-20T07:08:37.767482Z", + "shell.execute_reply": "2026-05-20T07:08:37.767309Z" + } + }, "outputs": [], "source": [ "rhs = xr.DataArray([10, 20, 30], dims=[\"time\"], coords={\"time\": [0, 1, 2]})\n", @@ -125,12 +174,26 @@ { "cell_type": "markdown", "metadata": {}, - "source": "### Same-Shape Operands: Positional Alignment\n\nWhen two operands have the **same shape** on a shared dimension, linopy uses **positional alignment** by default — coordinate labels are ignored and the left operand's labels are kept. This is a performance optimization but can be surprising:" + "source": [ + "### Same-shape operands: positional alignment\n", + "\n", + ".. warning::\n", + " When two operands have the **same shape** along a shared dimension, linopy aligns them by **position** and ignores the coordinate labels entirely — the result simply takes the left operand's labels. It is a deliberate performance shortcut, but it means two operands whose labels don't actually match get combined anyway, silently. Whenever the labels might differ, pass an explicit ``join`` (shown below).\n", + "\n", + "For example, ``offset_const`` is labelled ``[5, 6, 7, 8, 9]`` while ``x`` is labelled ``[0, 1, 2, 3, 4]`` — same length, so the result keeps ``x``'s labels and matches the two by position:" + ] }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:08:37.768405Z", + "iopub.status.busy": "2026-05-20T07:08:37.768345Z", + "iopub.status.idle": "2026-05-20T07:08:37.772690Z", + "shell.execute_reply": "2026-05-20T07:08:37.772543Z" + } + }, "outputs": [], "source": [ "offset_const = xr.DataArray(\n", @@ -142,12 +205,21 @@ { "cell_type": "markdown", "metadata": {}, - "source": "Even though ``offset_const`` has coordinates ``[5, 6, 7, 8, 9]`` and ``x`` has ``[0, 1, 2, 3, 4]``, the result uses ``x``'s labels. The values are aligned by **position**, not by label. The same applies when adding two variables or expressions of identical shape:" + "source": [ + "The same happens between two variables of identical shape — their labels need not overlap at all:" + ] }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:08:37.773582Z", + "iopub.status.busy": "2026-05-20T07:08:37.773523Z", + "iopub.status.idle": "2026-05-20T07:08:37.779892Z", + "shell.execute_reply": "2026-05-20T07:08:37.779718Z" + } + }, "outputs": [], "source": [ "z = m.add_variables(lower=0, coords=[pd.RangeIndex(5, 10, name=\"time\")], name=\"z\")\n", @@ -157,12 +229,23 @@ { "cell_type": "markdown", "metadata": {}, - "source": "``x`` (time 0–4) and ``z`` (time 5–9) share no coordinate labels, yet the result has 5 entries under ``x``'s coordinates — because they have the same shape, positions are matched directly.\n\nTo force **label-based** alignment, pass an explicit ``join``:" + "source": [ + "``x`` (time 0–4) and ``z`` (time 5–9) share no coordinate labels, yet the result has 5 entries under ``x``'s coordinates — because they have the same shape, positions are matched directly.\n", + "\n", + "To force **label-based** alignment, pass an explicit ``join``:" + ] }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:08:37.780786Z", + "iopub.status.busy": "2026-05-20T07:08:37.780731Z", + "iopub.status.idle": "2026-05-20T07:08:37.787431Z", + "shell.execute_reply": "2026-05-20T07:08:37.787256Z" + } + }, "outputs": [], "source": [ "x.add(z, join=\"outer\")" @@ -171,28 +254,37 @@ { "cell_type": "markdown", "metadata": {}, - "source": "With ``join=\"outer\"``, the result spans all 10 time steps (union of 0–4 and 5–9), filling missing positions with zeros. This is the correct label-based alignment. The same-shape positional shortcut is equivalent to ``join=\"override\"`` — see below." + "source": [ + "With ``join=\"outer\"``, the result spans all 10 time steps (union of 0–4 and 5–9), filling missing positions with zeros. This is the correct label-based alignment. The same-shape positional shortcut is equivalent to ``join=\"override\"`` — see below." + ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## The ``join`` Parameter\n", + "## The `join` parameter\n", "\n", - "For explicit control over alignment, use the ``.add()``, ``.sub()``, ``.mul()``, and ``.div()`` methods with a ``join`` parameter. The supported values follow xarray conventions:\n", + "For explicit control over alignment, use the `.add()`, `.sub()`, `.mul()`, and `.div()` methods with a `join` parameter. The supported values follow xarray conventions:\n", "\n", - "- ``\"inner\"`` — intersection of coordinates\n", - "- ``\"outer\"`` — union of coordinates (with fill)\n", - "- ``\"left\"`` — keep left operand's coordinates\n", - "- ``\"right\"`` — keep right operand's coordinates\n", - "- ``\"override\"`` — positional alignment, ignore coordinate labels\n", - "- ``\"exact\"`` — coordinates must match exactly (raises on mismatch)" + "- `\"inner\"` — intersection of coordinates\n", + "- `\"outer\"` — union of coordinates (with fill)\n", + "- `\"left\"` — keep left operand's coordinates\n", + "- `\"right\"` — keep right operand's coordinates\n", + "- `\"override\"` — positional alignment, ignore coordinate labels\n", + "- `\"exact\"` — coordinates must match exactly (raises on mismatch)" ] }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:08:37.788405Z", + "iopub.status.busy": "2026-05-20T07:08:37.788348Z", + "iopub.status.idle": "2026-05-20T07:08:37.791770Z", + "shell.execute_reply": "2026-05-20T07:08:37.791609Z" + } + }, "outputs": [], "source": [ "m2 = linopy.Model()\n", @@ -214,7 +306,14 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:08:37.792878Z", + "iopub.status.busy": "2026-05-20T07:08:37.792814Z", + "iopub.status.idle": "2026-05-20T07:08:37.799972Z", + "shell.execute_reply": "2026-05-20T07:08:37.799779Z" + } + }, "outputs": [], "source": [ "a.add(b, join=\"inner\")" @@ -230,7 +329,14 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:08:37.800875Z", + "iopub.status.busy": "2026-05-20T07:08:37.800820Z", + "iopub.status.idle": "2026-05-20T07:08:37.807266Z", + "shell.execute_reply": "2026-05-20T07:08:37.807079Z" + } + }, "outputs": [], "source": [ "a.add(b, join=\"outer\")" @@ -246,7 +352,14 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:08:37.808188Z", + "iopub.status.busy": "2026-05-20T07:08:37.808130Z", + "iopub.status.idle": "2026-05-20T07:08:37.814375Z", + "shell.execute_reply": "2026-05-20T07:08:37.814203Z" + } + }, "outputs": [], "source": [ "a.add(b, join=\"left\")" @@ -262,7 +375,14 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:08:37.815356Z", + "iopub.status.busy": "2026-05-20T07:08:37.815292Z", + "iopub.status.idle": "2026-05-20T07:08:37.821588Z", + "shell.execute_reply": "2026-05-20T07:08:37.821436Z" + } + }, "outputs": [], "source": [ "a.add(b, join=\"right\")" @@ -271,12 +391,21 @@ { "cell_type": "markdown", "metadata": {}, - "source": "**Override** — positional alignment, ignore coordinate labels. The result uses the left operand's coordinates. Here ``a`` has i=[0, 1, 2] and ``b`` has i=[1, 2, 3], so positions are matched as 0↔1, 1↔2, 2↔3:" + "source": [ + "**Override** — positional alignment, ignore coordinate labels. The result uses the left operand's coordinates. Here ``a`` has i=[0, 1, 2] and ``b`` has i=[1, 2, 3], so positions are matched as 0↔1, 1↔2, 2↔3:" + ] }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:08:37.822440Z", + "iopub.status.busy": "2026-05-20T07:08:37.822380Z", + "iopub.status.idle": "2026-05-20T07:08:37.828201Z", + "shell.execute_reply": "2026-05-20T07:08:37.828059Z" + } + }, "outputs": [], "source": [ "a.add(b, join=\"override\")" @@ -294,7 +423,14 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:08:37.829086Z", + "iopub.status.busy": "2026-05-20T07:08:37.829026Z", + "iopub.status.idle": "2026-05-20T07:08:37.834373Z", + "shell.execute_reply": "2026-05-20T07:08:37.834186Z" + } + }, "outputs": [], "source": [ "const = xr.DataArray([2, 3, 4], dims=[\"i\"], coords={\"i\": [1, 2, 3]})\n", @@ -305,7 +441,14 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:08:37.835323Z", + "iopub.status.busy": "2026-05-20T07:08:37.835264Z", + "iopub.status.idle": "2026-05-20T07:08:37.839918Z", + "shell.execute_reply": "2026-05-20T07:08:37.839727Z" + } + }, "outputs": [], "source": [ "a.mul(const, join=\"left\")" @@ -315,15 +458,22 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Alignment in Constraints\n", + "## Alignment in constraints\n", "\n", - "The ``.le()``, ``.ge()``, and ``.eq()`` methods create constraints with explicit coordinate alignment. They accept the same ``join`` parameter:" + "The `.le()`, `.ge()`, and `.eq()` methods create constraints with explicit coordinate alignment. They accept the same `join` parameter:" ] }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:08:37.840832Z", + "iopub.status.busy": "2026-05-20T07:08:37.840779Z", + "iopub.status.idle": "2026-05-20T07:08:37.845979Z", + "shell.execute_reply": "2026-05-20T07:08:37.845807Z" + } + }, "outputs": [], "source": [ "rhs = xr.DataArray([10, 20], dims=[\"i\"], coords={\"i\": [0, 1]})\n", @@ -341,7 +491,14 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:08:37.846859Z", + "iopub.status.busy": "2026-05-20T07:08:37.846804Z", + "iopub.status.idle": "2026-05-20T07:08:37.851651Z", + "shell.execute_reply": "2026-05-20T07:08:37.851481Z" + } + }, "outputs": [], "source": [ "a.le(rhs, join=\"left\")" @@ -359,7 +516,14 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:08:37.852594Z", + "iopub.status.busy": "2026-05-20T07:08:37.852535Z", + "iopub.status.idle": "2026-05-20T07:08:37.859020Z", + "shell.execute_reply": "2026-05-20T07:08:37.858775Z" + } + }, "outputs": [], "source": [ "expr = 2 * a + 1\n", @@ -369,12 +533,23 @@ { "cell_type": "markdown", "metadata": {}, - "source": "## Practical Example\n\nConsider a generation dispatch model where solar availability follows a daily profile and a minimum demand constraint only applies during peak hours." + "source": [ + "## Practical example\n", + "\n", + "Consider a generation dispatch model where solar availability follows a daily profile and a minimum demand constraint only applies during peak hours." + ] }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:08:37.860197Z", + "iopub.status.busy": "2026-05-20T07:08:37.860086Z", + "iopub.status.idle": "2026-05-20T07:08:37.863526Z", + "shell.execute_reply": "2026-05-20T07:08:37.863370Z" + } + }, "outputs": [], "source": [ "m3 = linopy.Model()\n", @@ -395,7 +570,14 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:08:37.864428Z", + "iopub.status.busy": "2026-05-20T07:08:37.864375Z", + "iopub.status.idle": "2026-05-20T07:08:37.874793Z", + "shell.execute_reply": "2026-05-20T07:08:37.874629Z" + } + }, "outputs": [], "source": [ "capacity = xr.DataArray([100, 80, 50], dims=[\"tech\"], coords={\"tech\": techs})\n", @@ -405,12 +587,21 @@ { "cell_type": "markdown", "metadata": {}, - "source": "For solar, we build a full 24-hour availability profile — zero at night, sine-shaped during daylight (hours 6–18). Since this covers all hours, standard alignment works directly and solar is properly constrained to zero at night:" + "source": [ + "For solar, we build a full 24-hour availability profile — zero at night, sine-shaped during daylight (hours 6–18). Since this covers all hours, standard alignment works directly and solar is properly constrained to zero at night:" + ] }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:08:37.875847Z", + "iopub.status.busy": "2026-05-20T07:08:37.875790Z", + "iopub.status.idle": "2026-05-20T07:08:37.885613Z", + "shell.execute_reply": "2026-05-20T07:08:37.885452Z" + } + }, "outputs": [], "source": [ "solar_avail = np.zeros(24)\n", @@ -424,12 +615,21 @@ { "cell_type": "markdown", "metadata": {}, - "source": "Now suppose a minimum demand of 120 MW must be met, but only during peak hours (8–20). The demand array covers a subset of hours, so we use ``join=\"inner\"`` to restrict the constraint to just those hours:" + "source": [ + "Now suppose a minimum demand of 120 MW must be met, but only during peak hours (8–20). The demand array covers a subset of hours, so we use ``join=\"inner\"`` to restrict the constraint to just those hours:" + ] }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:08:37.886478Z", + "iopub.status.busy": "2026-05-20T07:08:37.886422Z", + "iopub.status.idle": "2026-05-20T07:08:37.897229Z", + "shell.execute_reply": "2026-05-20T07:08:37.897068Z" + } + }, "outputs": [], "source": [ "peak_hours = pd.RangeIndex(8, 21, name=\"hour\")\n", @@ -444,7 +644,9 @@ { "cell_type": "markdown", "metadata": {}, - "source": "The demand constraint only applies during peak hours (8–20). Outside that range, no minimum generation is required." + "source": [ + "The demand constraint only applies during peak hours (8–20). Outside that range, no minimum generation is required." + ] }, { "cell_type": "markdown", @@ -480,7 +682,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.3" + "version": "3.11.11" } }, "nbformat": 4, diff --git a/examples/create-a-model-with-coordinates.ipynb b/examples/create-a-model-with-coordinates.ipynb deleted file mode 100644 index e8021a35..00000000 --- a/examples/create-a-model-with-coordinates.ipynb +++ /dev/null @@ -1,218 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "4db583af", - "metadata": {}, - "source": [ - "# Use Coordinates\n", - "\n", - "Now, the real power of the package comes into play! \n", - "\n", - "Linopy is structured around the concept that variables, and therefore expressions and constraints, have coordinates. That is, a `Variable` object actually contains multiple variables across dimensions, just as we know it from a `numpy` array or a `pandas.DataFrame`.\n", - "\n", - "Suppose the two variables `x` and `y` are now functions of time `t` and we would modify the problem according to: " - ] - }, - { - "cell_type": "markdown", - "id": "comparable-talent", - "metadata": {}, - "source": [ - "Minimize:\n", - "$$\\sum_t x_t + 2 y_t$$\n", - "\n", - "subject to:\n", - "\n", - "$$\n", - "x_t \\ge 0 \\qquad \\forall t \\\\\n", - "y_t \\ge 0 \\qquad \\forall t \\\\\n", - "3x_t + 7y_t \\ge 10 t \\qquad \\forall t\\\\\n", - "5x_t + 2y_t \\ge 3 t \\qquad \\forall t\n", - "$$\n", - "\n", - "whereas `t` spans all the range from 0 to 10." - ] - }, - { - "cell_type": "markdown", - "id": "proprietary-receipt", - "metadata": {}, - "source": [ - "In order to formulate the new problem with linopy, we start again by initializing a model." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "close-maximum", - "metadata": {}, - "outputs": [], - "source": [ - "import linopy\n", - "\n", - "m = linopy.Model()" - ] - }, - { - "cell_type": "markdown", - "id": "positive-appearance", - "metadata": {}, - "source": [ - "Again, we define `x` and `y` using the `add_variables` function, but now we are adding a `coords` argument. This automatically creates optimization variables for all coordinates, in this case time-steps." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "included-religious", - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "\n", - "time = pd.Index(range(10), name=\"time\")\n", - "\n", - "x = m.add_variables(\n", - " lower=0,\n", - " coords=[time],\n", - " name=\"x\",\n", - ")\n", - "y = m.add_variables(lower=0, coords=[time], name=\"y\")" - ] - }, - { - "cell_type": "markdown", - "id": "terminal-ethernet", - "metadata": {}, - "source": [ - "Following the previous example, we write the constraints out using the syntax from above, while multiplying the rhs with `t`. Note that the coordinates from the lhs and the rhs have to match. \n", - "\n", - ".. note::\n", - " In the beginning, it is recommended to use explicit dimension names. Like that, things remain clear and no unexpected broadcasting (which we show later) will happen. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c24d120a", - "metadata": {}, - "outputs": [], - "source": [ - "factor = pd.Series(time, index=time)\n", - "\n", - "3 * x + 7 * y >= 10 * factor" - ] - }, - { - "cell_type": "markdown", - "id": "f09803f4", - "metadata": {}, - "source": [ - "It always helps to write out the constraints before adding them to the model. Since they look good, let's assign them." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "comprehensive-blend", - "metadata": {}, - "outputs": [], - "source": [ - "con1 = m.add_constraints(3 * x + 7 * y >= 10 * factor, name=\"con1\")\n", - "con2 = m.add_constraints(5 * x + 2 * y >= 3 * factor, name=\"con2\")\n", - "m" - ] - }, - { - "cell_type": "markdown", - "id": "induced-professor", - "metadata": {}, - "source": [ - "Now, when it comes to the objective, we use the `sum` function of `linopy.LinearExpression`. This stacks all terms all terms of the `time` dimension and writes them into one big expression. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "alternate-story", - "metadata": {}, - "outputs": [], - "source": [ - "obj = (x + 2 * y).sum()\n", - "m.add_objective(obj)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "outer-presence", - "metadata": {}, - "outputs": [], - "source": [ - "m.solve(solver_name=\"highs\", output_flag=False)" - ] - }, - { - "cell_type": "markdown", - "id": "495cd082", - "metadata": {}, - "source": [ - "In order to inspect the solution. You can go via the variables, i.e. `y.solution` or via the `solution` aggregator of the model, which combines the solution of all variables. This can sometimes be helpful." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "monthly-census", - "metadata": {}, - "outputs": [], - "source": [ - "m.solution.to_dataframe().plot(grid=True, ylabel=\"Optimal Value\");" - ] - }, - { - "cell_type": "markdown", - "id": "owned-europe", - "metadata": {}, - "source": [ - "Alright! Now you learned how to set up linopy variables and expressions with coordinates. In the User Guide, which follows, we are going to see, how the representation of variables with coordinates allows us to formulate more advanced operations." - ] - }, - { - "cell_type": "markdown", - "id": "4db583af", - "metadata": {}, - "source": [ - "## Where to next\n", - "\n", - "You've now seen the full path from declaring variables on coordinates to solving the model. The [User Guide overview](user-guide.rst) reopens each piece in depth and points you at every topic from here." - ] - } - ], - "metadata": { - "@webio": { - "lastCommId": null, - "lastKernelId": null - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/create-a-model.ipynb b/examples/create-a-model.ipynb deleted file mode 100644 index b6fc9705..00000000 --- a/examples/create-a-model.ipynb +++ /dev/null @@ -1,285 +0,0 @@ -{ - "cells": [ - { - "attachments": {}, - "cell_type": "markdown", - "id": "4db583af", - "metadata": {}, - "source": [ - "# Solve a Basic Model\n", - "\n", - "In this example, we explain the basic functions of the linopy `Model` class. First, we are setting up a very simple linear optimization model, given by " - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "together-ocean", - "metadata": {}, - "source": [ - "Minimize:\n", - "$$ x + 2y $$\n", - " \n", - "subject to:\n", - "\n", - "$$ x \\ge 0 $$\n", - "$$y \\ge 0 $$\n", - "$$3x + 7y \\ge 10 $$\n", - "$$5x + 2y \\ge 3 $$" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "dramatic-cannon", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "43949d36", - "metadata": {}, - "source": [ - "### Initializing a `Model`\n", - "\n", - "The Model class in Linopy is a fundamental part of the library. It serves as a container for all the relevant data associated with a linear optimization problem. This includes variables, constraints, and the objective function." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "technical-conducting", - "metadata": {}, - "outputs": [], - "source": [ - "from linopy import Model\n", - "\n", - "m = Model()" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "e5b16d53", - "metadata": {}, - "source": [ - "This creates a new Model object, which you can then use to define your optimization problem." - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "rolled-delicious", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Adding variables\n", - "\n", - "Variables in a linear optimization problem represent the decision variables. A variable can always be assigned with a lower and an upper bound. In our case, both `x` and `y` have a lower bound of zero (default is unbouded). In Linopy, you can add variables to a Model using the `add_variables` method:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "protecting-power", - "metadata": {}, - "outputs": [], - "source": [ - "x = m.add_variables(lower=0, name=\"x\")\n", - "y = m.add_variables(lower=0, name=\"y\");" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "featured-maria", - "metadata": {}, - "source": [ - "`x` and `y` are linopy variables of the class `linopy.Variable`. Each of them contain all relevant information that define it. The `name` parameter is optional but can be useful for referencing the variables later." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "virtual-anxiety", - "metadata": {}, - "outputs": [], - "source": [ - "x" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "sonic-rebate", - "metadata": {}, - "source": [ - "Since both `x` and `y` are scalar variables (meaning they don't have any dimensions), their underlying data contain only one optimization variable each. \n", - "\n", - "### Adding Constraints\n", - "\n", - "Constraints define the feasible region of the optimization problem. They consist of the left hand side (lhs) and the right hand side (rhs). The first constraint that we want to write down is \n", - "$3x + 7y >= 10$, which we write out exactly in the mathematical way" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fbb46cad", - "metadata": {}, - "outputs": [], - "source": [ - "3 * x + 7 * y >= 10" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "f4666bee", - "metadata": {}, - "source": [ - "Note, we can also mix the constant and the variable expression, like this" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "60f41b76", - "metadata": {}, - "outputs": [], - "source": [ - "3 * x + 7 * y - 10 >= 0" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "02abd938", - "metadata": {}, - "source": [ - "... and linopy will automatically take over the separation of variables expression on the lhs, and constant values on the rhs.\n", - "\n", - "The constraint is currently not assigned to the model. We assign it by calling the function `m.add_constraints`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "hollywood-production", - "metadata": {}, - "outputs": [], - "source": [ - "m.add_constraints(3 * x + 7 * y >= 10)\n", - "m.add_constraints(5 * x + 2 * y >= 3);" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "global-maple", - "metadata": {}, - "source": [ - "## Adding the Objective \n", - "\n", - "The objective function defines what you want to optimize. You can set the objective function of a Model in Linopy using the add_objective method. For our example that would be" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "overall-exhibition", - "metadata": {}, - "outputs": [], - "source": [ - "m.add_objective(x + 2 * y)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "6f9692aa", - "metadata": {}, - "source": [ - "## Solving the Model\n", - "\n", - "Once you've defined your Model with variables, constraints, and an objective function, you can solve it using the `solve` method:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "pressing-copying", - "metadata": {}, - "outputs": [], - "source": [ - "m.solve(solver_name=\"highs\", output_flag=False)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "preceding-limit", - "metadata": {}, - "source": [ - "The solution of the linear problem assigned to the variables under `solution` in form of a `xarray.Dataset`. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "electric-duration", - "metadata": {}, - "outputs": [], - "source": [ - "x.solution" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e6d31751", - "metadata": {}, - "outputs": [], - "source": [ - "y.solution" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "e296f641", - "metadata": {}, - "source": [ - "Well done! You solved your first linopy model!" - ] - } - ], - "metadata": { - "@webio": { - "lastCommId": null, - "lastKernelId": null - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/creating-constraints.ipynb b/examples/creating-constraints.ipynb index 05e2a899..62fa4597 100644 --- a/examples/creating-constraints.ipynb +++ b/examples/creating-constraints.ipynb @@ -2,75 +2,68 @@ "cells": [ { "cell_type": "markdown", - "id": "e8249281", + "id": "cc-01", "metadata": {}, "source": [ "# Creating Constraints\n", "\n", - "Constraints are created and at the same time assigned to the model using the function \n", + "A constraint relates two expressions with `≤`, `≥`, or `=`. In linopy you write the constraint in natural mathematical form, hand it to `model.add_constraints`, and it becomes part of the model. The same conventions you've already seen for expressions — broadcasting by dimension name, explicit alignment when coordinates can mismatch — carry over directly.\n", "\n", - "```\n", - "model.add_constraints\n", - "```\n", - "where `model` is a `linopy.Model` instance. Again, we want to understand this function and its argument. So, let's create a model first." + "This page covers four topics:\n", + "\n", + "- writing and assigning constraints with the `<=` / `>=` / `==` operators;\n", + "- the explicit-alignment forms `.le()` / `.ge()` / `.eq()` for when coordinate ranges might disagree;\n", + "- the CSR backend — a memory-efficient alternative storage format for large models." ] }, { "cell_type": "code", "execution_count": null, - "id": "e0c196e4", - "metadata": {}, + "id": "cc-02", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-19T20:55:43.372185Z", + "iopub.status.busy": "2026-05-19T20:55:43.372019Z", + "iopub.status.idle": "2026-05-19T20:55:44.248825Z", + "shell.execute_reply": "2026-05-19T20:55:44.248569Z" + } + }, "outputs": [], "source": [ "from linopy import Model\n", "\n", - "m = Model()" + "m = Model()\n", + "x = m.add_variables(name=\"x\")" ] }, { "cell_type": "markdown", - "id": "043c0b06", + "id": "cc-03", "metadata": {}, "source": [ - "Given a variable `x` which has to by lower than 10/3, the constraint would be formulated as \n", - "\n", - "$$ x \\le \\frac{10}{3} $$\n", - "\n", - "or\n", - "\n", - "$$ 3 x \\le 10 $$\n", - " \n", - "or \n", + "## Writing constraints\n", "\n", - "$$ x - \\frac{3}{10} \\le 0 $$\n", + "Consider the constraint $x \\le \\tfrac{10}{3}$. linopy accepts every algebraically equivalent rearrangement — write whichever form is most natural for the problem:\n", "\n", + "$$\n", + "x \\le \\tfrac{10}{3} \\qquad\\Longleftrightarrow\\qquad 3x \\le 10 \\qquad\\Longleftrightarrow\\qquad 3x - 10 \\le 0\n", + "$$\n", "\n", - "of which all formulations can be written out with linopy just like that. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6b496b92", - "metadata": {}, - "outputs": [], - "source": [ - "x = m.add_variables(name=\"x\")" - ] - }, - { - "cell_type": "markdown", - "id": "73541c03", - "metadata": {}, - "source": [ - "When applying one of the operators `<=`, `>=`, `==` to the expression, an unassigned constraint is built:" + "Applying `<=`, `>=`, or `==` to an expression builds an **unassigned** constraint — one that exists as a value but isn't yet part of any model:" ] }, { "cell_type": "code", "execution_count": null, - "id": "4c8aba7e", - "metadata": {}, + "id": "cc-04", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-19T20:55:44.250154Z", + "iopub.status.busy": "2026-05-19T20:55:44.250020Z", + "iopub.status.idle": "2026-05-19T20:55:44.254513Z", + "shell.execute_reply": "2026-05-19T20:55:44.254305Z" + } + }, "outputs": [], "source": [ "con = 3 * x <= 10\n", @@ -79,17 +72,24 @@ }, { "cell_type": "markdown", - "id": "0d75781d", + "id": "cc-05", "metadata": {}, "source": [ - "Unasssigned means, it is not yet added to the model. We can inspect the elements of the anonymous constraint: " + "An unassigned constraint exposes its left-hand side, right-hand side, and sign so you can inspect what the math reduced to before committing it to the model:" ] }, { "cell_type": "code", "execution_count": null, - "id": "01f182b5", - "metadata": {}, + "id": "cc-06", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-19T20:55:44.255490Z", + "iopub.status.busy": "2026-05-19T20:55:44.255415Z", + "iopub.status.idle": "2026-05-19T20:55:44.257577Z", + "shell.execute_reply": "2026-05-19T20:55:44.257403Z" + } + }, "outputs": [], "source": [ "con.lhs" @@ -98,8 +98,15 @@ { "cell_type": "code", "execution_count": null, - "id": "783287b3", - "metadata": {}, + "id": "cc-07", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-19T20:55:44.258398Z", + "iopub.status.busy": "2026-05-19T20:55:44.258333Z", + "iopub.status.idle": "2026-05-19T20:55:44.261859Z", + "shell.execute_reply": "2026-05-19T20:55:44.261640Z" + } + }, "outputs": [], "source": [ "con.rhs" @@ -107,54 +114,75 @@ }, { "cell_type": "markdown", - "id": "aac468c3", + "id": "cc-08", "metadata": {}, "source": [ - "We can now add the constraint to the model by passing the unassigned `Constraint` to the `.add_constraint` function. " + "Pass the unassigned constraint to `m.add_constraints` to attach it to the model. Always set `name=` — modifying or looking up the constraint later relies on it:" ] }, { "cell_type": "code", "execution_count": null, - "id": "0adf929b", - "metadata": {}, + "id": "cc-09", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-19T20:55:44.262735Z", + "iopub.status.busy": "2026-05-19T20:55:44.262672Z", + "iopub.status.idle": "2026-05-19T20:55:44.265403Z", + "shell.execute_reply": "2026-05-19T20:55:44.265252Z" + } + }, "outputs": [], "source": [ - "c = m.add_constraints(con, name=\"my-constraint\")\n", + "c = m.add_constraints(con, name=\"capacity\")\n", "c" ] }, { "cell_type": "markdown", - "id": "e78c2635", + "id": "cc-10", "metadata": {}, "source": [ - "The same output would be generated if passing lhs, sign and rhs as separate arguments to the function:" + "You can also write the constraint inline, without the named intermediate — same end result:" ] }, { "cell_type": "code", "execution_count": null, - "id": "c084adec", - "metadata": {}, + "id": "cc-11", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-19T20:55:44.266307Z", + "iopub.status.busy": "2026-05-19T20:55:44.266241Z", + "iopub.status.idle": "2026-05-19T20:55:44.270281Z", + "shell.execute_reply": "2026-05-19T20:55:44.270106Z" + } + }, "outputs": [], "source": [ - "m.add_constraints(3 * x <= 10, name=\"the-same-constraint\")" + "m.add_constraints(3 * x <= 10, name=\"capacity_inline\")" ] }, { "cell_type": "markdown", - "id": "2b4db4d5", + "id": "cc-12", "metadata": {}, "source": [ - "Note that the return value of the operation is a `Constraint` which contains the reference labels to the constraints in the optimization model. Also is redirects to its lhs, sign and rhs, for example we can call" + "The returned `Constraint` carries the reference labels into the optimization model and forwards attribute access to `.lhs`, `.rhs`, and `.sign`:" ] }, { "cell_type": "code", "execution_count": null, - "id": "ea6e990c", - "metadata": {}, + "id": "cc-13", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-19T20:55:44.271181Z", + "iopub.status.busy": "2026-05-19T20:55:44.271119Z", + "iopub.status.idle": "2026-05-19T20:55:44.273335Z", + "shell.execute_reply": "2026-05-19T20:55:44.273166Z" + } + }, "outputs": [], "source": [ "c.lhs" @@ -162,25 +190,26 @@ }, { "cell_type": "markdown", - "id": "e6ae2a19", + "id": "cc-14", "metadata": {}, "source": [ - "to inspect the lhs of a defined constraint." - ] - }, - { - "cell_type": "markdown", - "id": "efb74da3", - "metadata": {}, - "source": [ - "When moving the constant value to the left hand side in the initialization, it will be pulled to the right hand side as soon as the constraint is defined" + "### Automatic LHS/RHS separation\n", + "\n", + "linopy doesn't mind which side of the inequality the constants are on. Constants written on the variable side get pulled across automatically, so every committed constraint ends up with variables on the left and the constant on the right:" ] }, { "cell_type": "code", "execution_count": null, - "id": "e582051e", - "metadata": {}, + "id": "cc-15", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-19T20:55:44.274257Z", + "iopub.status.busy": "2026-05-19T20:55:44.274193Z", + "iopub.status.idle": "2026-05-19T20:55:44.276998Z", + "shell.execute_reply": "2026-05-19T20:55:44.276833Z" + } + }, "outputs": [], "source": [ "3 * x - 10" @@ -189,8 +218,15 @@ { "cell_type": "code", "execution_count": null, - "id": "e2c2dbb3", - "metadata": {}, + "id": "cc-16", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-19T20:55:44.277848Z", + "iopub.status.busy": "2026-05-19T20:55:44.277790Z", + "iopub.status.idle": "2026-05-19T20:55:44.281546Z", + "shell.execute_reply": "2026-05-19T20:55:44.281371Z" + } + }, "outputs": [], "source": [ "3 * x - 10 <= 0" @@ -198,25 +234,34 @@ }, { "cell_type": "markdown", - "id": "15909055", + "id": "cc-17", "metadata": {}, "source": [ - "Like this, the all defined constraints have a clear separation between variable on the left, and constants on the right. " + "Both forms produce the same final constraint, with consistent LHS/RHS shape." ] }, { "cell_type": "markdown", - "id": "b9d31509", + "id": "cc-18", "metadata": {}, "source": [ - "All constraints are added to the `.constraints` container from where all assigned constraints can be accessed." + "### Accessing constraints on the model\n", + "\n", + "All constraints added to the model live in `m.constraints` and are looked up by name:" ] }, { "cell_type": "code", "execution_count": null, - "id": "d205e695", - "metadata": {}, + "id": "cc-19", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-19T20:55:44.282451Z", + "iopub.status.busy": "2026-05-19T20:55:44.282396Z", + "iopub.status.idle": "2026-05-19T20:55:44.284056Z", + "shell.execute_reply": "2026-05-19T20:55:44.283876Z" + } + }, "outputs": [], "source": [ "m.constraints" @@ -225,36 +270,98 @@ { "cell_type": "code", "execution_count": null, - "id": "cc5baaf4", + "id": "cc-20", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-19T20:55:44.284868Z", + "iopub.status.busy": "2026-05-19T20:55:44.284812Z", + "iopub.status.idle": "2026-05-19T20:55:44.286672Z", + "shell.execute_reply": "2026-05-19T20:55:44.286518Z" + } + }, + "outputs": [], + "source": [ + "m.constraints[\"capacity\"]" + ] + }, + { + "cell_type": "markdown", + "id": "cc-21", "metadata": {}, + "source": [ + "## Explicit alignment with `.le()`, `.ge()`, `.eq()`\n", + "\n", + "The `<=`, `>=`, `==` operators are convenient but they inherit the coordinate-alignment behaviour shown in :doc:`creating-expressions`: when the two sides share a dim name with mismatched coordinates, the left operand's coordinates win silently.\n", + "\n", + "For constraints where that matters — typically when LHS and RHS come from different data sources — use the method forms `.le()`, `.ge()`, and `.eq()` instead. They take an explicit `join` (`\"inner\"`, `\"outer\"`, `\"left\"`, `\"right\"`) so you control how the two sides line up:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cc-22", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-19T20:55:44.287568Z", + "iopub.status.busy": "2026-05-19T20:55:44.287501Z", + "iopub.status.idle": "2026-05-19T20:55:44.296241Z", + "shell.execute_reply": "2026-05-19T20:55:44.296093Z" + } + }, "outputs": [], "source": [ - "m.constraints[\"my-constraint\"]" + "import pandas as pd\n", + "\n", + "a = m.add_variables(coords=[pd.Index(range(0, 10), name=\"time\")], name=\"a\")\n", + "b = m.add_variables(coords=[pd.Index(range(5, 15), name=\"time\")], name=\"b\")\n", + "\n", + "# Inner join: constraint applies only on the time steps both variables cover.\n", + "a.le(b, join=\"inner\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11f63852", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-19T20:55:44.297166Z", + "iopub.status.busy": "2026-05-19T20:55:44.297111Z", + "iopub.status.idle": "2026-05-19T20:55:44.304394Z", + "shell.execute_reply": "2026-05-19T20:55:44.304218Z" + } + }, + "outputs": [], + "source": [ + "# Outer join: union of the two ranges, with partial terms where one side is missing.\n", + "a.le(b, join=\"outer\")" ] }, { "cell_type": "markdown", - "id": "r0wxi7v1m7l", + "id": "cc-23", "metadata": {}, - "source": "## Coordinate Alignment in Constraints\n\nAs an alternative to the ``<=``, ``>=``, ``==`` operators, linopy provides ``.le()``, ``.ge()``, and ``.eq()`` methods on variables and expressions. These methods accept a ``join`` parameter (``\"inner\"``, ``\"outer\"``, ``\"left\"``, ``\"right\"``) for explicit control over how coordinates are aligned when creating constraints. See the :doc:`coordinate-alignment` guide for details." + "source": [ + "The :doc:`coordinate-alignment` guide walks through every join option in depth." + ] }, { "cell_type": "markdown", - "id": "csr-backend-intro", + "id": "cc-24", "metadata": {}, "source": [ - "## CSR Backend (Advanced)\n", + "## The CSR backend\n", "\n", - "By default, linopy stores each constraint as an `xarray.Dataset` (`Constraint`). This is flexible and allows full label-based indexing, but can use significant memory when constraints have many terms.\n", + "By default, linopy stores each constraint as an `xarray.Dataset` (`Constraint`). This is flexible and supports full label-based indexing, but can be memory-heavy when constraints carry many terms.\n", "\n", - "For large models, linopy provides an alternative **CSR backend** via the `CSRConstraint` class. It stores the constraint coefficients as a [scipy CSR sparse matrix](https://docs.scipy.org/doc/scipy/reference/generated/scipy.sparse.csr_array.html) with flat numpy arrays for the right-hand side and signs. This can reduce memory usage by up to **90%** and speeds up matrix generation for direct solver APIs by **30–120x**.\n", + "For large models, the **CSR backend** (`CSRConstraint`) stores coefficients as a [scipy CSR sparse matrix](https://docs.scipy.org/doc/scipy/reference/generated/scipy.sparse.csr_array.html), with flat numpy arrays for the right-hand side and signs. It reduces memory usage by up to **90%** and speeds up matrix generation for direct solver APIs by **30–120×**.\n", "\n", - "`CSRConstraint` is **immutable** — once frozen, the constraint data cannot be modified in place. You can always convert back to the mutable xarray-backed `Constraint` if needed." + "`CSRConstraint` is **immutable** — once frozen, the constraint data can't be modified in place. Conversion back to the mutable xarray-backed `Constraint` is always available." ] }, { "cell_type": "markdown", - "id": "csr-per-constraint", + "id": "cc-25", "metadata": {}, "source": [ "### Freezing individual constraints\n", @@ -265,14 +372,19 @@ { "cell_type": "code", "execution_count": null, - "id": "csr-per-constraint-code", - "metadata": {}, + "id": "cc-26", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-19T20:55:44.305421Z", + "iopub.status.busy": "2026-05-19T20:55:44.305364Z", + "iopub.status.idle": "2026-05-19T20:55:44.313328Z", + "shell.execute_reply": "2026-05-19T20:55:44.313120Z" + } + }, "outputs": [], "source": [ "import numpy as np\n", "\n", - "from linopy import Model\n", - "\n", "m2 = Model()\n", "y = m2.add_variables(coords=[np.arange(100)], name=\"y\")\n", "\n", @@ -284,7 +396,7 @@ }, { "cell_type": "markdown", - "id": "csr-global", + "id": "cc-27", "metadata": {}, "source": [ "### Freezing all constraints globally\n", @@ -295,14 +407,21 @@ { "cell_type": "code", "execution_count": null, - "id": "csr-global-code", - "metadata": {}, + "id": "cc-28", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-19T20:55:44.314308Z", + "iopub.status.busy": "2026-05-19T20:55:44.314238Z", + "iopub.status.idle": "2026-05-19T20:55:44.324866Z", + "shell.execute_reply": "2026-05-19T20:55:44.324705Z" + } + }, "outputs": [], "source": [ "m3 = Model(freeze_constraints=True)\n", - "z = m3.add_variables(coords=[np.arange(50)], name=\"z\")\n", - "m3.add_constraints(z >= 0, name=\"lower\")\n", - "m3.add_constraints(z <= 100, name=\"upper\")\n", + "z3 = m3.add_variables(coords=[np.arange(50)], name=\"z\")\n", + "m3.add_constraints(z3 >= 0, name=\"lower\")\n", + "m3.add_constraints(z3 <= 100, name=\"upper\")\n", "\n", "print(type(m3.constraints[\"lower\"]))\n", "print(type(m3.constraints[\"upper\"]))" @@ -310,7 +429,7 @@ }, { "cell_type": "markdown", - "id": "csr-roundtrip", + "id": "cc-29", "metadata": {}, "source": [ "### Converting between representations\n", @@ -321,15 +440,22 @@ { "cell_type": "code", "execution_count": null, - "id": "csr-roundtrip-code", - "metadata": {}, + "id": "cc-30", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-19T20:55:44.325765Z", + "iopub.status.busy": "2026-05-19T20:55:44.325713Z", + "iopub.status.idle": "2026-05-19T20:55:44.328656Z", + "shell.execute_reply": "2026-05-19T20:55:44.328472Z" + } + }, "outputs": [], "source": [ "frozen = m3.constraints[\"lower\"]\n", - "print(f\"Frozen type: {type(frozen).__name__}\")\n", + "print(f\"Frozen type: {type(frozen).__name__}\")\n", "\n", "thawed = frozen.mutable()\n", - "print(f\"Mutable type: {type(thawed).__name__}\")\n", + "print(f\"Mutable type: {type(thawed).__name__}\")\n", "\n", "refrozen = thawed.freeze()\n", "print(f\"Re-frozen type: {type(refrozen).__name__}\")" @@ -337,38 +463,61 @@ }, { "cell_type": "markdown", - "id": "7843d42c", - "source": "### API differences from `Constraint`\n\n`CSRConstraint` deliberately exposes a narrower API than the xarray-backed `Constraint`:\n\n- **No in-place mutation.** Setters such as `con.coeffs = ...`, `con.vars = ...`, `con.sign = ...`, `con.rhs = ...`, and `con.lhs = ...` are only available on `Constraint`.\n- **No label-based indexing.** `con.loc[...]` is only available on `Constraint`.\n- **Accessing `.coeffs` / `.vars` triggers reconstruction.** On a `CSRConstraint` these properties rebuild the full xarray `Dataset` on demand and emit a `PerformanceWarning`. For solver-oriented workflows prefer `con.to_matrix()` or work with the CSR data directly.\n\nIf you need any of the above, call `.mutable()` first to get a `Constraint`:\n\n```python\ncon = m.constraints[\"my_constraint\"].mutable()\ncon.loc[{\"time\": 0}] # label-based indexing now available\ncon.rhs = 5 # mutation now available\n```", - "metadata": {} + "id": "cc-31", + "metadata": {}, + "source": [ + "### API differences from `Constraint`\n", + "\n", + "`CSRConstraint` deliberately exposes a narrower API than the xarray-backed `Constraint`:\n", + "\n", + "- **No in-place mutation.** Setters such as `con.coeffs = ...`, `con.vars = ...`, `con.sign = ...`, `con.rhs = ...`, and `con.lhs = ...` are only available on `Constraint`.\n", + "- **No label-based indexing.** `con.loc[...]` is only available on `Constraint`.\n", + "- **Accessing `.coeffs` / `.vars` triggers reconstruction.** On a `CSRConstraint` these properties rebuild the full xarray `Dataset` on demand and emit a `PerformanceWarning`. For solver-oriented workflows prefer `con.to_matrix()` or work with the CSR data directly.\n", + "\n", + "If you need any of the above, call `.mutable()` first to get a `Constraint`:\n", + "\n", + "```python\n", + "con = m.constraints[\"my_constraint\"].mutable()\n", + "con.loc[{\"time\": 0}] # label-based indexing now available\n", + "con.rhs = 5 # mutation now available\n", + "```" + ] }, { "cell_type": "markdown", - "id": "csr-when-to-use", + "id": "cc-32", "metadata": {}, "source": [ "### When to use the CSR backend\n", "\n", "The CSR backend is most beneficial when:\n", "\n", - "- Your model has **many constraints with many terms**.\n", - "- **Memory** is a bottleneck.\n", - "- You use a **direct solver API** (e.g. HiGHS, Gurobi Python bindings) rather than file-based I/O.\n", + "- the model has **many constraints with many terms**;\n", + "- **memory** is a bottleneck;\n", + "- you're using a **direct solver API** (e.g. HiGHS, Gurobi Python bindings) rather than file-based I/O.\n", "\n", "For small models the overhead is negligible and the default xarray-backed `Constraint` is perfectly fine.\n", "\n", - "Additionally, if you don't need variable and constraint names in the solver (e.g. for batch solves), you can disable name export for extra speed:\n", + "If you don't need variable and constraint names exported to the solver (e.g. for batch solves), you can disable name export for extra speed:\n", "\n", "```python\n", "m = Model(freeze_constraints=True, set_names_in_solver_io=False)\n", "```" ] + }, + { + "cell_type": "markdown", + "id": "cc-33", + "metadata": {}, + "source": [ + "## Where to next\n", + "\n", + "- :doc:`coordinate-alignment` — the deep dive on `join` semantics for both expressions and constraints.\n", + "- :doc:`manipulating-models` — modifying constraints (sign, RHS, removing) after they're on the model." + ] } ], "metadata": { - "@webio": { - "lastCommId": null, - "lastKernelId": null - }, "kernelspec": { "display_name": "Python 3", "language": "python", @@ -384,7 +533,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.3" + "version": "3.11.11" } }, "nbformat": 4, diff --git a/examples/creating-expressions.ipynb b/examples/creating-expressions.ipynb index d0bf0db4..b9b0e8e4 100644 --- a/examples/creating-expressions.ipynb +++ b/examples/creating-expressions.ipynb @@ -1,51 +1,54 @@ { "cells": [ { - "attachments": {}, "cell_type": "markdown", - "id": "4db583af", + "id": "ce-01", "metadata": {}, "source": [ "# Creating Expressions\n", "\n", - "In this notebook, we look at different options to create expressions. A strong focus will be set on the array-like operations: Since variables are represented in array-like structure, we benefit from a lot of well-knwon functionalities which we know from `numpy`, `pandas` or `xarray`.\n", + "Variables in linopy are array-like — they have a shape, named dimensions, and coordinates — so combining them produces array-like *expressions* that work the same way. Every familiar move from `numpy`, `pandas`, and `xarray` is available: arithmetic operators broadcast over shared dimensions, `.loc` selects subsets, `.where` masks, and `.shift` / `.rolling` / `.groupby` aggregate along named dims.\n", "\n", + "Two ideas underpin everything that follows:\n", "\n", - "These are for example\n", - "\n", - "- `arithmetic` operations to create expressions\n", - "- `broadcasting` to combine smaller and larger arrays\n", - "- `.loc` to select a subset of the original array using indexes\n", - "- `.where` to select where the variable or expression should be active or not\n", - "- `.shift` to shift the whole array along one dimension\n", - "- `.groupby` to group by a key and apply operations on the groups \n", - "- `.rolling` to perform a rolling operation and perform operations\n" + "- **Operations align by dimension name.** Two expressions on the same `time` dim combine elementwise; expressions on different dims broadcast over the union.\n", + "- **Coordinates carry semantics.** When same-named dims hold *different* coordinate values, alignment is no longer obvious — linopy has a default rule and an explicit one. Both are shown below, and the case where the default surprises is flagged." ] }, { - "attachments": {}, "cell_type": "markdown", - "id": "f3712718", + "id": "ce-02", "metadata": {}, "source": [ ".. hint::\n", - " Nearly all of the functions and properties, that can be accessed from a `Variable`, can be accesses from a `LinearExpression` and `QuadraticExpression`." + " Nearly every function or property on a `Variable` is also available on a `LinearExpression` and `QuadraticExpression`. The interface is intentionally consistent." ] }, { - "attachments": {}, "cell_type": "markdown", - "id": "95b25d13", + "id": "ce-03", "metadata": {}, "source": [ - "Let's start by creating a model." + "## Setup\n", + "\n", + "Two variables run through every example:\n", + "\n", + "- `x` on a single `time` dimension (10 steps);\n", + "- `y` on two dimensions, `time` (the same 10 steps) and `port` (`a`–`d`)." ] }, { "cell_type": "code", "execution_count": null, - "id": "close-maximum", - "metadata": {}, + "id": "ce-04", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T06:15:50.794130Z", + "iopub.status.busy": "2026-05-20T06:15:50.793926Z", + "iopub.status.idle": "2026-05-20T06:15:51.632145Z", + "shell.execute_reply": "2026-05-20T06:15:51.631918Z" + } + }, "outputs": [], "source": [ "import pandas as pd\n", @@ -63,23 +66,29 @@ ] }, { - "attachments": {}, "cell_type": "markdown", - "id": "582a0cad", + "id": "ce-05", "metadata": {}, "source": [ - "## Arithmetic Operations\n", + "## Arithmetic and broadcasting\n", "\n", - "Arithmetic operations such as addition (`+`), subtraction (`-`), multiplication (`*`) can be used directly on the variables and expressions in Linopy. These operations are applied element-wise on the variables.\n", + "`+`, `-`, and `*` work elementwise on variables and expressions. When two operands share a dimension, the result has that dimension's coordinates; when they don't, the result broadcasts to the union of dimensions. The output is a `LinearExpression`; multiplying two variables produces a `QuadraticExpression`.\n", "\n", - "For example, if you want to create a new combined expr `z` that is the sum of `x` and `y`, you can do so as follows:" + "Adding `x` (shape `(time: 10)`) to `y` (shape `(time: 10, port: 4)`) broadcasts `x` over `port`:" ] }, { "cell_type": "code", "execution_count": null, - "id": "0aec195a", - "metadata": {}, + "id": "ce-06", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T06:15:51.633303Z", + "iopub.status.busy": "2026-05-20T06:15:51.633184Z", + "iopub.status.idle": "2026-05-20T06:15:51.641570Z", + "shell.execute_reply": "2026-05-20T06:15:51.641361Z" + } + }, "outputs": [], "source": [ "z = x + y\n", @@ -87,315 +96,391 @@ ] }, { - "attachments": {}, - "cell_type": "markdown", - "id": "8c4b9ea6", - "metadata": {}, - "source": [ - ".. note::\n", - " In the addition, the variable `x` is broadcasted and the return value has the same set of dimensions as `y`. " - ] - }, - { - "attachments": {}, "cell_type": "markdown", - "id": "a1de2b9a", + "id": "ce-07", "metadata": {}, "source": [ - "Similarly, you can subtract `y` from `x` or multiply `x` and `y` as follows:" + "Subtraction works the same way; multiplying two variables produces a `QuadraticExpression`:" ] }, { "cell_type": "code", "execution_count": null, - "id": "c56761cd", - "metadata": {}, + "id": "ce-08", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T06:15:51.642592Z", + "iopub.status.busy": "2026-05-20T06:15:51.642517Z", + "iopub.status.idle": "2026-05-20T06:15:51.650598Z", + "shell.execute_reply": "2026-05-20T06:15:51.650406Z" + } + }, "outputs": [], "source": [ - "z = x - y\n", - "z" + "x - y" ] }, { "cell_type": "code", "execution_count": null, - "id": "b59fa397", - "metadata": {}, + "id": "ce-09", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T06:15:51.651575Z", + "iopub.status.busy": "2026-05-20T06:15:51.651509Z", + "iopub.status.idle": "2026-05-20T06:15:51.659934Z", + "shell.execute_reply": "2026-05-20T06:15:51.659766Z" + } + }, "outputs": [], "source": [ - "z = x * y\n", - "z" + "x * y # QuadraticExpression" ] }, { - "attachments": {}, "cell_type": "markdown", - "id": "0ca00a73", + "id": "ce-10", "metadata": {}, "source": [ - "In all cases, the returned shape is the same. Note that, the output type of the multiplication is a `QuadraticExpression` and not a `LinearExpression`.\n" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "3ff0d1cd", - "metadata": {}, - "source": [ - "The `z` expression, which carries along `x` and `y`, has different attributes such as `coord_dims`, `dims`, `size`." + "Expressions carry a few introspection attributes — `dims`, `coord_dims`, `size` — for programmatic inspection of the structure:" ] }, { "cell_type": "code", "execution_count": null, - "id": "35c7331f", - "metadata": {}, + "id": "ce-11", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T06:15:51.660873Z", + "iopub.status.busy": "2026-05-20T06:15:51.660806Z", + "iopub.status.idle": "2026-05-20T06:15:51.662488Z", + "shell.execute_reply": "2026-05-20T06:15:51.662300Z" + } + }, "outputs": [], "source": [ "z.coord_dims" ] }, { - "attachments": {}, "cell_type": "markdown", - "id": "f7578221", + "id": "ce-12", "metadata": {}, "source": [ - ".. important::\n", - " When combining variables or expressions with dimensions of the same name and size, the first object determines the coordinates of the resulting expression. For example:" + "### Explicit alignment with `.add()`, `.sub()`, `.mul()`, `.div()`\n", + "\n", + "When two operands share a dim name but hold *different* coordinate values — for example, two variables that both live on `time` but cover overlapping rather than identical ranges — there's a choice to make about how to line them up. The operator forms (`+`, `-`, `*`, `/`) pick a default; the method forms `.add()`, `.sub()`, `.mul()`, `.div()` take an explicit `join` (`\"inner\"`, `\"outer\"`, `\"left\"`, `\"right\"`):" ] }, { "cell_type": "code", "execution_count": null, - "id": "8c511f35", - "metadata": {}, + "id": "ce-13", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T06:15:51.663411Z", + "iopub.status.busy": "2026-05-20T06:15:51.663353Z", + "iopub.status.idle": "2026-05-20T06:15:51.670275Z", + "shell.execute_reply": "2026-05-20T06:15:51.670099Z" + } + }, "outputs": [], "source": [ - "other_time = pd.Index(range(10, 20), name=\"time\")\n", - "b = m.add_variables(coords=[other_time], name=\"b\")\n", - "b" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "59f43468", - "metadata": {}, - "source": [ - "`b` has the same shape as `x`, but they have different coordinates. When we combine `x` and `b` the coordinates on dimension `time` will be taken from the first object and the coordinates of the subsequent object will be ignored:" + "overlap_time = pd.Index(range(5, 15), name=\"time\")\n", + "z2 = m.add_variables(coords=[overlap_time], name=\"z2\")\n", + "\n", + "# Inner join — only the time steps both variables cover (5..9).\n", + "x.add(z2, join=\"inner\")" ] }, { "cell_type": "code", "execution_count": null, - "id": "26edd6ab", - "metadata": {}, + "id": "ce-14", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T06:15:51.671162Z", + "iopub.status.busy": "2026-05-20T06:15:51.671104Z", + "iopub.status.idle": "2026-05-20T06:15:51.677757Z", + "shell.execute_reply": "2026-05-20T06:15:51.677597Z" + } + }, "outputs": [], "source": [ - "x + b" + "# Outer join — full union, with NaN where one side has no coordinate.\n", + "x.add(z2, join=\"outer\")" ] }, { "cell_type": "markdown", - "id": "a8xsfdqrcrn", + "id": "ce-15", + "metadata": {}, "source": [ - ".. tip::\n", - " For explicit control over how coordinates are aligned during arithmetic, use the ``.add()``, ``.sub()``, ``.mul()``, and ``.div()`` methods with a ``join`` parameter (``\"inner\"``, ``\"outer\"``, ``\"left\"``, ``\"right\"``). See the :doc:`coordinate-alignment` guide for details." - ], - "metadata": {} + ".. warning::\n", + " **The default operator inherits coordinates from the left operand.** When two operands share a dim name but disagree on its coordinate *values*, linopy doesn't reconcile them — it takes the left operand's coordinates and proceeds as if the right side matched. The model still builds and still solves; just on a problem with different alignment than you may have intended.\n", + "\n", + " Use `.add()` / `.sub()` / `.mul()` / `.div()` with an explicit `join` whenever the coordinate ranges might disagree. The :doc:`coordinate-alignment` guide walks through every join option in depth." + ] }, { - "attachments": {}, "cell_type": "markdown", - "id": "de6d3073", + "id": "ce-16", "metadata": {}, "source": [ - "## Using `.loc` to select a subset\n", - "\n", - "The `.loc` function allows you to select a subset of the array using indexes. This is useful when you want to apply operations to a specific subset of your variables.\n", - "\n", - "For example, if you want to apply a summation to the variables `x` and `y` only for the first 5 time steps, you can do so as follows:" + "To see the default rule fire: `x` covers `time = 0..9` and a new variable `b` covers `time = 10..19`. Adding them with `+` keeps `x`'s coordinates and silently aligns `b` to those:" ] }, { "cell_type": "code", "execution_count": null, - "id": "93119cfc", - "metadata": {}, + "id": "ce-17", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T06:15:51.678712Z", + "iopub.status.busy": "2026-05-20T06:15:51.678643Z", + "iopub.status.idle": "2026-05-20T06:15:51.685659Z", + "shell.execute_reply": "2026-05-20T06:15:51.685495Z" + } + }, "outputs": [], "source": [ - "x.loc[:5]" + "other_time = pd.Index(range(10, 20), name=\"time\")\n", + "b = m.add_variables(coords=[other_time], name=\"b\")\n", + "x + b" ] }, { - "cell_type": "code", - "execution_count": null, - "id": "ae6a1b29", + "cell_type": "markdown", + "id": "ce-18", "metadata": {}, - "outputs": [], "source": [ - "x.loc[:5] + y.loc[:5]" + "## Selecting subsets\n", + "\n", + "Two kinds of subsetting: picking parts of an expression by label or position (`.sel`, `.isel`, `.loc`), and masking parts out conditionally (`.where`). All return expressions of the same kind, so they keep composing with arithmetic." ] }, { - "attachments": {}, "cell_type": "markdown", - "id": "14b02f7d", + "id": "ce-19", "metadata": {}, "source": [ - "which is the same as" + "### Selecting by label and position\n", + "\n", + "`.sel` selects along a named dimension by coordinate label; `.isel` selects by integer position. Both name the dimension explicitly, and both work identically on variables and on expressions.\n", + "\n", + "`.sel` accepts scalars, lists, and slices:" ] }, { "cell_type": "code", "execution_count": null, - "id": "7281f08c", - "metadata": {}, + "id": "0b2fe61d", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T06:15:51.686605Z", + "iopub.status.busy": "2026-05-20T06:15:51.686550Z", + "iopub.status.idle": "2026-05-20T06:15:51.688865Z", + "shell.execute_reply": "2026-05-20T06:15:51.688713Z" + } + }, "outputs": [], "source": [ - "expr = x + y\n", - "expr.loc[:5]" + "x.sel(time=[2, 4, 6])" ] }, { - "attachments": {}, "cell_type": "markdown", - "id": "c3c97abf", + "id": "72307c2c", "metadata": {}, "source": [ - "In combination with the overwrite of the coordinates, this is useful when you need to combine different selections, like" + "`.isel` takes the same forms but indexes by **position** — \"the first three steps\" regardless of the coordinate labels:" ] }, { "cell_type": "code", "execution_count": null, - "id": "27063ea9", - "metadata": {}, + "id": "f8f5e346", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T06:15:51.689745Z", + "iopub.status.busy": "2026-05-20T06:15:51.689690Z", + "iopub.status.idle": "2026-05-20T06:15:51.691866Z", + "shell.execute_reply": "2026-05-20T06:15:51.691718Z" + } + }, "outputs": [], "source": [ - "x.loc[:4] + y.loc[5:]" + "x.isel(time=slice(0, 3))" ] }, { - "attachments": {}, "cell_type": "markdown", - "id": "cdb53f49", + "id": "258de0b2", "metadata": {}, "source": [ - "## Using `.where` to select active variables or expressions\n", - "\n", - "The `.where` function allows you to select where the variable or expression should be active or not. This is useful when you want to apply constraints or operations only to a specific subset of your variables based on a condition. It is quite similar to the functionality of masking, that we showed earlier.\n", - "\n", - "For example, if you want to create an sum of the variables `x` and `y` where `time` is greater than 2, you can do so as follows:" + "The interface is the same on expressions — `.sel` and `.isel` work on a combined expression exactly as they do on a variable:" ] }, { "cell_type": "code", "execution_count": null, - "id": "ab8f59fd", - "metadata": {}, + "id": "ce-23", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T06:15:51.692749Z", + "iopub.status.busy": "2026-05-20T06:15:51.692698Z", + "iopub.status.idle": "2026-05-20T06:15:51.700631Z", + "shell.execute_reply": "2026-05-20T06:15:51.700454Z" + } + }, "outputs": [], "source": [ - "mask = xr.DataArray(time > 2, coords=[time])\n", - "(x + y).where(mask)" + "(x + y).sel(time=[2, 4, 6])" ] }, { - "attachments": {}, "cell_type": "markdown", - "id": "3ee10a58", + "id": "ce-24", "metadata": {}, "source": [ - "We can use this to make a conditional summation:" + ".. note::\n", + " `.loc` is a pandas-style bracket alternative — `x.loc[:5]` is shorthand for `x.sel(time=slice(None, 5))`, and `x.loc[{\"time\": [2, 4, 6]}]` for the dict form. We sugest to use `sel`/`isel`" + ] + }, + { + "cell_type": "markdown", + "id": "ce-26", + "metadata": {}, + "source": [ + "### `.where` — conditional masking\n", + "\n", + "`.where(condition)` keeps the expression where the condition is `True` and masks it elsewhere. Useful for activating a constraint only on a subset of indices — e.g. \"demand applies only after time step 2\":" ] }, { "cell_type": "code", "execution_count": null, - "id": "21fa9664", - "metadata": {}, + "id": "ce-27", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T06:15:51.701600Z", + "iopub.status.busy": "2026-05-20T06:15:51.701547Z", + "iopub.status.idle": "2026-05-20T06:15:51.709499Z", + "shell.execute_reply": "2026-05-20T06:15:51.709337Z" + } + }, "outputs": [], "source": [ - "(x + y).where(mask) + xr.DataArray(5, coords=[time]).where(~mask, 0)" + "mask = xr.DataArray(time > 2, coords=[time])\n", + "(x + y).where(mask)" ] }, { - "attachments": {}, "cell_type": "markdown", - "id": "652973ea", + "id": "ce-28", "metadata": {}, "source": [ - "## Using `.shift` to shift the Variable along one dimension\n", + "## Operations along a dimension\n", "\n", - "The `.shift` function allows you to shift the whole array along one dimension. This is useful when you want to apply constraints or operations that involve a time delay or a shift in the time steps.\n", + "`.shift`, `.rolling`, and `.groupby` act along a named dimension — typically `time`, but the mechanism is general. Each is the building block for a common modelling pattern." + ] + }, + { + "cell_type": "markdown", + "id": "ce-29", + "metadata": {}, + "source": [ + "### `.shift` — couple a variable to its lagged self\n", "\n", - "For example, if you want to apply a constraint that involves a one time step delay in the variables `x` and `y`, you can do so as follows:" + "`var.shift(time=1)` lines a variable up against itself one step earlier along `time`. The difference `var - var.shift(time=1)` is the natural way to write state-of-charge–style coupling — \"the change in stock between consecutive time steps\":" ] }, { "cell_type": "code", "execution_count": null, - "id": "organized-hampshire", - "metadata": {}, + "id": "ce-30", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T06:15:51.710521Z", + "iopub.status.busy": "2026-05-20T06:15:51.710462Z", + "iopub.status.idle": "2026-05-20T06:15:51.719337Z", + "shell.execute_reply": "2026-05-20T06:15:51.719139Z" + } + }, "outputs": [], "source": [ "y - y.shift(time=1)" ] }, { - "attachments": {}, "cell_type": "markdown", - "id": "eaf3f38c", + "id": "ce-31", "metadata": {}, "source": [ - "## Using `.groupby` to group by a key and apply operations on the groups\n", + "### `.rolling` — windowed sums and means\n", "\n", - "The `.groupby` function allows you to group by a key and apply operations on the groups. This is useful when you want to apply constraints or operations that involve a grouping of the time steps or any other dimension.\n", - "\n", - "For example, if you want to apply a constraint that involves the sum of `x` and `y` over every two time steps, you can do so as follows:" + "Rolling windows aggregate over a fixed-length slice along a dimension. Use them for moving-average constraints, reserve requirements over a window, or accumulation over a horizon:" ] }, { "cell_type": "code", "execution_count": null, - "id": "5170d187", - "metadata": {}, + "id": "ce-32", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T06:15:51.720333Z", + "iopub.status.busy": "2026-05-20T06:15:51.720274Z", + "iopub.status.idle": "2026-05-20T06:15:51.725672Z", + "shell.execute_reply": "2026-05-20T06:15:51.725507Z" + } + }, "outputs": [], "source": [ - "group_key = pd.Series(time.values // 2, index=time)\n", - "(x + y).groupby(group_key).sum()" + "x.rolling(time=3).sum()" ] }, { - "attachments": {}, "cell_type": "markdown", - "id": "7ded9a54", + "id": "ce-33", "metadata": {}, "source": [ - "## Using `.rolling` to perform a rolling operation\n", + "### `.groupby` — aggregate along a key\n", "\n", - "The `.rolling` function allows you to perform a rolling operation and apply operations. This is useful when you want to apply constraints or operations that involve a rolling window of the time steps or any other dimension.\n", - "\n", - "For example, if you want to apply a constraint that involves the sum of `x` over a rolling window of 3 time steps, you can do so as follows:" + "`.groupby(key).sum()` (or `.mean()`, `.max()`, …) groups along a dimension by the values of a key and aggregates each group. Use it to turn hourly variables into daily totals, per-asset variables into per-technology aggregates, or any \"sum by category\" pattern:" ] }, { "cell_type": "code", "execution_count": null, - "id": "d703fb70", - "metadata": {}, + "id": "ce-34", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T06:15:51.726561Z", + "iopub.status.busy": "2026-05-20T06:15:51.726507Z", + "iopub.status.idle": "2026-05-20T06:15:51.736433Z", + "shell.execute_reply": "2026-05-20T06:15:51.736260Z" + } + }, "outputs": [], "source": [ - "x.rolling(time=3).sum()" + "# Aggregate over pairs of time steps; the key just needs to be aligned to `time`.\n", + "group_key = pd.Series(time.values // 2, index=time, name=\"group\")\n", + "(x + y).groupby(group_key).sum()" + ] + }, + { + "cell_type": "markdown", + "id": "ce-35", + "metadata": {}, + "source": [ + "## Where to next\n", + "\n", + "- The :doc:`coordinate-alignment` guide is the deep dive on `join` semantics — when to reach for each option and how it shapes the resulting expression.\n", + "- :doc:`creating-constraints` covers how to turn expressions into model constraints." ] } ], "metadata": { - "@webio": { - "lastCommId": null, - "lastKernelId": null - }, "kernelspec": { "display_name": "Python 3", "language": "python", @@ -411,7 +496,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.3" + "version": "3.11.11" } }, "nbformat": 4, diff --git a/examples/creating-variables.ipynb b/examples/creating-variables.ipynb index 9179a31a..56706694 100644 --- a/examples/creating-variables.ipynb +++ b/examples/creating-variables.ipynb @@ -2,24 +2,33 @@ "cells": [ { "cell_type": "markdown", - "id": "e8249281", + "id": "cv-01", "metadata": {}, "source": [ "# Creating Variables\n", "\n", - "Variables are created and assigned to the model using the function \n", + "Variables are the decision quantities of an optimization problem. In linopy they're created with `model.add_variables(...)`, stored on the model, and used as building blocks for expressions and constraints. A `Variable` has bounds, an optional shape with any number of labelled dimensions, and a name.\n", "\n", - "```\n", - "model.add_variables\n", - "```\n", - "where `model` is a `linopy.Model` instance. In the following we show how this function works and what the resulting variables look like. So, let's create a model and go through it!" + "Throughout this guide every example sets two things explicitly:\n", + "\n", + "- a **`name`** — so the variable is readable in repr, in expressions, and when looking it up on the model;\n", + "- explicit **dimension names** — so coordinates can't silently collide between variables you add separately.\n", + "\n", + "These two habits are the single biggest thing you can do to keep larger models debuggable." ] }, { "cell_type": "code", "execution_count": null, - "id": "e0c196e4", - "metadata": {}, + "id": "cv-02", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:57:22.883250Z", + "iopub.status.busy": "2026-05-20T07:57:22.883072Z", + "iopub.status.idle": "2026-05-20T07:57:23.703611Z", + "shell.execute_reply": "2026-05-20T07:57:23.703362Z" + } + }, "outputs": [], "source": [ "import numpy as np\n", @@ -32,56 +41,43 @@ ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "", - "id": "46c4f2824a2ed8aa" - }, - { - "cell_type": "markdown", - "id": "6c6420a7", + "id": "cv-03", "metadata": {}, "source": [ - "First of all it is crucial to know, that the return value of the `.add_variables` function is a `linopy.Variable` which itself contains all important information and provides helpful functions. It can have an arbitrary number of labeled dimensions. For each combination of coordinates, exactly one representative scalar variable is defined and, in the end, passed to the solver. \n", - "\n", - "The first three arguments of the `.add_variables` function are \n", - "1. `lower` denoting the lower bound of the variables (default `-inf`) \n", - "2. `upper` denoting the upper bound (default `+inf`)\n", - "3. `coords` (default None). \n", - "\n", - "These argument determine the shape of the added variable. \n", - "\n", - "Generally, the function is strongly aligned to the initialization of an `xarray.DataArray`. Therefore `lower` and `upper` can be \n", - "\n", - "* scalar values (int/float)\n", - "* numpy ndarray's\n", - "* pandas Series\n", - "* pandas DataFrame's\n", - "* xarray DataArray's\n", + "## The `add_variables` signature\n", "\n", + "The first three positional arguments determine the variable's shape:\n", "\n", - "Note that scalars, numpy objects and pandas objects do not have or do not require dimension names. Thus, the naming of the dimensions is done by `xarray`. Therefore you can pass the `coords` argument, or alternatively, a `dims` argument in order to name your dimensions. \n", + "1. `lower` — lower bound (default `-inf`)\n", + "2. `upper` — upper bound (default `+inf`)\n", + "3. `coords` — dimension coordinates (default `None`)\n", "\n", - "\n", - ".. hint::\n", - " It is **best practice** to always define variables with explicit `name` and dimension names. This eases the inspection and avoids confusion from the automatically derived names." + "`lower` and `upper` can be scalars, numpy arrays, pandas `Series` / `DataFrame`, or xarray `DataArray` — anything compatible with `xarray.DataArray` construction. The variable's name is set with `name=`, and dimension names come from either `dims=`, named pandas indexes, or named xarray dims." ] }, { "cell_type": "markdown", - "id": "a2283b9a", + "id": "cv-04", "metadata": {}, "source": [ - "Let's start by creating a simple variable:\n", + "## A scalar variable\n", "\n", - "If we just keep the default, which is `-inf` and `+inf` for `lower` and `upper`, the code returns" + "With no bounds and no coordinates, `add_variables` returns a single decision variable. Always pass `name=`:" ] }, { "cell_type": "code", "execution_count": null, - "id": "ee589323", - "metadata": {}, + "id": "cv-05", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:57:23.704958Z", + "iopub.status.busy": "2026-05-20T07:57:23.704835Z", + "iopub.status.idle": "2026-05-20T07:57:23.708439Z", + "shell.execute_reply": "2026-05-20T07:57:23.708291Z" + } + }, "outputs": [], "source": [ "x = m.add_variables(name=\"x\")\n", @@ -90,25 +86,24 @@ }, { "cell_type": "markdown", - "id": "708920a3", - "metadata": {}, - "source": [ - "which is a variable without any coordinates and with just one optimization variable. The variable name is set by `name = 'x'`. " - ] - }, - { - "cell_type": "markdown", - "id": "b276e45d", + "id": "cv-06", "metadata": {}, "source": [ - "Like this the variable appears with its name when defining expression with it:" + "The name follows the variable into any expression it appears in:" ] }, { "cell_type": "code", "execution_count": null, - "id": "68d7e7da", - "metadata": {}, + "id": "cv-07", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:57:23.709401Z", + "iopub.status.busy": "2026-05-20T07:57:23.709324Z", + "iopub.status.idle": "2026-05-20T07:57:23.712258Z", + "shell.execute_reply": "2026-05-20T07:57:23.712086Z" + } + }, "outputs": [], "source": [ "x + 5" @@ -116,463 +111,516 @@ }, { "cell_type": "markdown", - "id": "528a7c40", + "id": "cv-08", "metadata": {}, "source": [ - "We can alter the lower and upper bounds of the variable by assigning scalar values to them." + "Add bounds by passing `lower` and `upper`:" ] }, { "cell_type": "code", "execution_count": null, - "id": "a1c080e0", - "metadata": {}, + "id": "cv-09", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:57:23.713196Z", + "iopub.status.busy": "2026-05-20T07:57:23.713119Z", + "iopub.status.idle": "2026-05-20T07:57:23.715371Z", + "shell.execute_reply": "2026-05-20T07:57:23.715215Z" + } + }, "outputs": [], "source": [ - "y = m.add_variables(lower=0, upper=4, name=\"y\")" + "y = m.add_variables(lower=0, upper=4, name=\"y\")\n", + "y" ] }, { "cell_type": "markdown", - "id": "885ac764", + "id": "cv-10", "metadata": {}, "source": [ - "### Variable Types\n", + "## Variable types\n", "\n", - "Per default the variable type is continuous, that the variables can take any real value in between and including the lower and upper bound. In order to alter the type, you have the option to set `integer` or `binary` to `True`." + "By default variables are continuous within their bounds. For discrete decisions, pass `integer=True` or `binary=True`:" ] }, { "cell_type": "code", "execution_count": null, - "id": "8a5d4543", - "metadata": {}, + "id": "cv-11", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:57:23.716244Z", + "iopub.status.busy": "2026-05-20T07:57:23.716187Z", + "iopub.status.idle": "2026-05-20T07:57:23.718375Z", + "shell.execute_reply": "2026-05-20T07:57:23.718205Z" + } + }, "outputs": [], "source": [ - "m.add_variables(lower=0, upper=10, integer=True)" - ] - }, - { - "cell_type": "markdown", - "id": "3af58bc4", - "metadata": {}, - "source": [ - ".. note::\n", - " Since we did not set the name argument the variable name is automatically determined and set to `var0`.\n", - "\n", - "\n", - "This variable `var0` can take all integer number between 0 and 10 inclusively. On the other hand, when defining a binary variable, we do not specify the lower and upper bounds and set `binary` to true." + "count = m.add_variables(lower=0, upper=10, integer=True, name=\"count\")\n", + "count" ] }, { "cell_type": "code", "execution_count": null, - "id": "ff64db81", - "metadata": {}, + "id": "cv-12", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:57:23.719190Z", + "iopub.status.busy": "2026-05-20T07:57:23.719135Z", + "iopub.status.idle": "2026-05-20T07:57:23.721192Z", + "shell.execute_reply": "2026-05-20T07:57:23.721045Z" + } + }, "outputs": [], "source": [ - "m.add_variables(binary=True)" + "flag = m.add_variables(binary=True, name=\"flag\")\n", + "flag" ] }, { "cell_type": "markdown", - "id": "7b432107", + "id": "cv-13", "metadata": {}, "source": [ - "### Working with dimensions\n", + "## Dimensioned variables\n", "\n", - "When initializing dimensional variables, it is most straight-forward and recommended to create variables with `DataArray`'s which are passed to the as `lower` and/or `upper`. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b4dfc46d", - "metadata": {}, - "outputs": [], - "source": [ - "lower = xr.DataArray([1, 2, 3])\n", - "v = m.add_variables(lower, name=\"v\")\n", - "v" - ] - }, - { - "cell_type": "markdown", - "id": "1ff347f9", - "metadata": {}, - "source": [ - "The returned `Variable` now has the same shape as the `lower` bound that we passed to the initialization. Since we did not specify any dimension name, it defaults to `dim_0`. In order to give the dimension a proper name we can use the `dims` argument. " + "Real models almost always have variables with one or more dimensions: a generation variable indexed by `(generator, hour)`, a flow indexed by `(from, to)`, a stock indexed by `(asset, time)`. The cleanest way to declare one is to pass an xarray `DataArray` with named `dims=` as the bound — the dimension names come along automatically:" ] }, { "cell_type": "code", "execution_count": null, - "id": "f93e5c08", - "metadata": {}, + "id": "cv-14", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:57:23.722138Z", + "iopub.status.busy": "2026-05-20T07:57:23.722079Z", + "iopub.status.idle": "2026-05-20T07:57:23.725249Z", + "shell.execute_reply": "2026-05-20T07:57:23.725057Z" + } + }, "outputs": [], "source": [ - "lower = xr.DataArray([1, 2, 3], dims=[\"my-dim\"])\n", - "m.add_variables(lower)" + "lower = xr.DataArray([1, 2, 3], dims=[\"time\"])\n", + "supply = m.add_variables(lower, name=\"supply\")\n", + "supply" ] }, { "cell_type": "markdown", - "id": "d4b20bb5", + "id": "cv-15", "metadata": {}, "source": [ - "You can arbitrarily broadcast dimensions when passing DataArray's with different set of dimensions. Let's do it and give `lower` another dimension than `upper`:" + "### Broadcasting between bounds\n", + "\n", + "When `lower` and `upper` carry different dimensions, linopy broadcasts: the variable spans every combination. This is a feature, not an accident — but only when the dim names are explicit and intended:" ] }, { "cell_type": "code", "execution_count": null, - "id": "71584630", - "metadata": {}, + "id": "cv-16", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:57:23.726289Z", + "iopub.status.busy": "2026-05-20T07:57:23.726207Z", + "iopub.status.idle": "2026-05-20T07:57:23.729724Z", + "shell.execute_reply": "2026-05-20T07:57:23.729551Z" + } + }, "outputs": [], "source": [ - "lower = xr.DataArray([1, 2, 3], dims=[\"my-dim\"])\n", - "upper = xr.DataArray([10, 11, 12, 13], dims=[\"my-dim-2\"])\n", - "m.add_variables(lower, upper)" - ] - }, - { - "cell_type": "markdown", - "id": "3e4e48c0", - "metadata": {}, - "source": [ - "Now instead of a single dimension, we end up with two dimensions `my-dim` and `my-dim-2` in the variable. This kind of **broadcasting** is a deeply incorporated in the functionality of linopy. " + "lower = xr.DataArray([1, 2, 3], dims=[\"time\"])\n", + "upper = xr.DataArray([10, 11, 12, 13], dims=[\"station\"])\n", + "m.add_variables(lower, upper, name=\"dispatch\")" ] }, { "cell_type": "markdown", - "id": "41893f11", + "id": "c28a59e9", "metadata": {}, - "source": [ - "We recall that, in order to improve the inspection, it is encouraged to define a `name` when creating a variable. So in your model you would rather write something like:" - ] + "source": ".. warning::\n Broadcasting between bounds is convenient, but a bound that *accidentally* carries an extra dimension will silently enlarge the variable. Passing `coords` explicitly guards against this — linopy treats `coords` as the authoritative dimension set and validates every bound against it. A bound carrying a dimension outside `coords` raises instead of broadcasting, a cheap way to catch bound-shape bugs at construction.\n\n A bound that is *missing* one of the `coords` dimensions is fine — it is broadcast up to fit." }, { "cell_type": "code", "execution_count": null, - "id": "e8857233", - "metadata": {}, + "id": "475a99fe", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:57:23.730664Z", + "iopub.status.busy": "2026-05-20T07:57:23.730608Z", + "iopub.status.idle": "2026-05-20T07:57:23.732671Z", + "shell.execute_reply": "2026-05-20T07:57:23.732501Z" + } + }, "outputs": [], "source": [ - "lower = xr.DataArray([1, 2, 3], dims=[\"time\"])\n", - "upper = xr.DataArray([10, 11, 12, 13], dims=[\"station\"])\n", - "m.add_variables(lower, upper, name=\"supply\")" + "# `upper` accidentally carries an extra \"station\" dimension.\n", + "bad_upper = xr.DataArray(\n", + " [[1, 2], [3, 4], [5, 6]],\n", + " dims=[\"time\", \"station\"],\n", + " coords={\"time\": range(3), \"station\": [\"a\", \"b\"]},\n", + ")\n", + "\n", + "try:\n", + " m.add_variables(lower=0, upper=bad_upper, coords=[pd.RangeIndex(3, name=\"time\")])\n", + "except ValueError as e:\n", + " print(\"Raised:\", e)" ] }, { "cell_type": "markdown", - "id": "437cc7a1", + "id": "cv-17", "metadata": {}, "source": [ - "#### Initializing variables with numpy arrays\n", + "### From numpy arrays\n", "\n", - "If `lower` and `upper` are numpy arrays, `linopy` it is recommended to pass a `dims` or a `coords` argument." + "numpy arrays carry no dimension names of their own. Pass `dims=` (the simpler form) or a named `coords` index:" ] }, { "cell_type": "code", "execution_count": null, - "id": "0fe33c34", + "id": "cv-18", "metadata": { - "scrolled": true + "execution": { + "iopub.execute_input": "2026-05-20T07:57:23.733511Z", + "iopub.status.busy": "2026-05-20T07:57:23.733453Z", + "iopub.status.idle": "2026-05-20T07:57:23.736188Z", + "shell.execute_reply": "2026-05-20T07:57:23.736031Z" + } }, "outputs": [], "source": [ "lower = np.array([1, 2])\n", "upper = np.array([10, 10])\n", - "m.add_variables(lower, upper, dims=[\"my-dim\"])" + "m.add_variables(lower, upper, dims=[\"time\"], name=\"x_np\")" ] }, { "cell_type": "markdown", - "id": "2ab6d301", + "id": "cv-19", "metadata": {}, "source": [ - "This is equivalent to the following" + "The same with explicit coordinate values — useful when the coordinate labels matter (e.g. timestamps or asset names):" ] }, { "cell_type": "code", "execution_count": null, - "id": "b8313ce0", - "metadata": {}, + "id": "cv-20", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:57:23.737007Z", + "iopub.status.busy": "2026-05-20T07:57:23.736954Z", + "iopub.status.idle": "2026-05-20T07:57:23.739723Z", + "shell.execute_reply": "2026-05-20T07:57:23.739544Z" + } + }, "outputs": [], "source": [ - "my_dim = pd.RangeIndex(2, name=\"my-dim\")\n", - "lower = np.array([1, 2])\n", - "upper = np.array([10, 10])\n", - "m.add_variables(lower, upper, coords=[my_dim])" + "time = pd.RangeIndex(2, name=\"time\")\n", + "m.add_variables(np.array([1, 2]), np.array([10, 10]), coords=[time], name=\"x_np_coords\")" ] }, { "cell_type": "markdown", - "id": "5052b9b5", + "id": "cv-21", "metadata": {}, "source": [ - "Note that \n", - "\n", - "- `dims` is a list of string defining the dimension names. \n", - " \n", - "- `coords` is an tuple of indexes as expected by `xarray.DataArray`. \n", - " \n", - "- The shape of `lower` and `upper` is aligned with `coords`.\n", - " \n", - "- When defining the index for the coords, a name was set in the index creation. This is helpful as we can ensure which dimension the variable is defined on. \n", + "### From pandas objects\n", "\n", - "Let's make the same example without setting an explicit dimension name:" + "Pandas objects always carry an index, but the index isn't always *named*. Set `.index.name` directly (or use `rename_axis`) so the variable's dimension name lines up with what the data represents:" ] }, { "cell_type": "code", "execution_count": null, - "id": "5f0994da", - "metadata": {}, + "id": "cv-22", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:57:23.740573Z", + "iopub.status.busy": "2026-05-20T07:57:23.740521Z", + "iopub.status.idle": "2026-05-20T07:57:23.743256Z", + "shell.execute_reply": "2026-05-20T07:57:23.743101Z" + } + }, "outputs": [], "source": [ - "coords = (pd.RangeIndex(2),)\n", - "m.add_variables(lower=lower, coords=coords)" + "time = pd.Index([0, 1], name=\"time\")\n", + "lower = pd.Series([1, 1], index=time)\n", + "upper = pd.Series([10, 12], index=time)\n", + "m.add_variables(lower, upper, name=\"x_pd\")" ] }, { "cell_type": "markdown", - "id": "dff8126d", + "id": "cv-23", "metadata": {}, "source": [ - "The dimension is now called `dim_0`, any new assignment of variable without dimension names, will also use that dimension name. When combining the variables to expressions it is important that you make sure that dimension names represent what they should. \n", - "\n", - ".. hint::\n", - " If you want to make sure, you are not messing up with dimensions, create the model with the flag `force_dim_names = True`, i.e." + "Same idea for a `DataFrame` — name both the index and the columns axis:" ] }, { "cell_type": "code", "execution_count": null, - "id": "d133a7a4", - "metadata": {}, + "id": "cv-24", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:57:23.744117Z", + "iopub.status.busy": "2026-05-20T07:57:23.744065Z", + "iopub.status.idle": "2026-05-20T07:57:23.747284Z", + "shell.execute_reply": "2026-05-20T07:57:23.747110Z" + } + }, "outputs": [], "source": [ - "other = Model(force_dim_names=True)\n", - "try:\n", - " other.add_variables(lower=lower, coords=coords)\n", - "except ValueError as e:\n", - " print(\"This raised an error:\", e)" + "lower = pd.DataFrame([[1, 1, 2], [1, 2, 2]]).rename_axis(index=\"from\", columns=\"to\")\n", + "upper = pd.Series([10, 12]).rename_axis(\"from\")\n", + "m.add_variables(lower, upper, name=\"flow\")" ] }, { "cell_type": "markdown", - "id": "9203ff16", + "id": "cv-25", "metadata": {}, "source": [ - "#### Initializing variables with Pandas objects\n", - "\n", - "Pandas objects always have indexes but do not require dimension names. It is again helpful to ensure that the variable have explicit dimension names, when passing `lower` and `upper` without `coords`. This can be done by either passing the `dims` argument to the `.add_variables` function, i.e." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2cf719be", - "metadata": {}, - "outputs": [], - "source": [ - "lower = pd.Series([1, 1])\n", - "upper = pd.Series([10, 12])\n", - "m.add_variables(lower, upper, dims=[\"my-dim\"])" + ".. note::\n", + " Pandas objects carry their own indexes, so the ``coords`` argument is ignored when supplied alongside them. A warning is raised if a ``coords`` value doesn't align with the pandas object." ] }, { "cell_type": "markdown", - "id": "3a4cf2d4", + "id": "cv-26", "metadata": {}, "source": [ - "or naming the indexes and columns of the pandas objects directly, e.g." + "## Masking selectively-active entries\n", + "\n", + "Sometimes you want a variable with a full rectangular shape but only some entries active. Classic case: a transport variable over `(from, to)` ports where shipping from a port to itself doesn't make sense. Pass a boolean `mask` with `False` on the disabled entries:" ] }, { "cell_type": "code", "execution_count": null, - "id": "61896a6f", - "metadata": {}, + "id": "cv-27", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:57:23.748130Z", + "iopub.status.busy": "2026-05-20T07:57:23.748077Z", + "iopub.status.idle": "2026-05-20T07:57:23.750050Z", + "shell.execute_reply": "2026-05-20T07:57:23.749905Z" + } + }, "outputs": [], "source": [ - "lower = pd.Series([1, 1]).rename_axis(\"my-dim\")\n", - "upper = pd.Series([10, 12]).rename_axis(\"my-dim\")\n", - "m.add_variables(lower, upper)" - ] - }, - { - "cell_type": "markdown", - "id": "2b462ff9", - "metadata": {}, - "source": [ - ".. note::\n", - " Again, if `lower` and `upper` do not have the same dimension names, the arrays are broadcasted, meaning the dimensions are spanned: " + "ports = list(\"abcdef\")\n", + "port_from = pd.Index(ports, name=\"from\")\n", + "port_to = pd.Index(ports, name=\"to\")\n", + "\n", + "mask = np.ones((len(ports), len(ports)), dtype=bool)\n", + "np.fill_diagonal(mask, False)\n", + "mask" ] }, { "cell_type": "code", "execution_count": null, - "id": "6ffd5a4e", + "id": "cv-28", "metadata": { - "scrolled": true + "execution": { + "iopub.execute_input": "2026-05-20T07:57:23.750887Z", + "iopub.status.busy": "2026-05-20T07:57:23.750831Z", + "iopub.status.idle": "2026-05-20T07:57:23.754758Z", + "shell.execute_reply": "2026-05-20T07:57:23.754600Z" + } }, "outputs": [], "source": [ - "lower = pd.Series([1, 1]).rename_axis(\"my-dim\")\n", - "upper = pd.Series([10, 12]).rename_axis(\"my-other-dim\")\n", - "m.add_variables(lower, upper)" + "transport = m.add_variables(\n", + " lower=0, coords=[port_from, port_to], name=\"transport\", mask=mask\n", + ")\n", + "transport" ] }, { "cell_type": "markdown", - "id": "31bbdbab", + "id": "cv-29", "metadata": {}, "source": [ - "Now instead of 2 variables, 4 variables were defined. \n", - "\n", - "The similar bahvior accounts for the case when passing a DataFrame and a Series without dimension names. The index axis is the first axis of both objects, thus these are expected to be the same (Note that pandas convention, is that Series are aligned and broadcasted along the column dimension of DataFrames): " + "The diagonal entries (e.g. `[a, a]`) are `None` — they don't appear in any constraint or in the objective." ] }, { - "cell_type": "code", - "execution_count": null, - "id": "fa2adc81", + "cell_type": "markdown", + "id": "cv-30", "metadata": {}, - "outputs": [], "source": [ - "lower = pd.DataFrame([[1, 1, 2], [1, 2, 2]])\n", - "upper = pd.Series([10, 12])\n", - "m.add_variables(lower, upper)" + "## Implicit naming for small models\n", + "\n", + "The recommended habit above — `name=` plus explicit dimension names — is what you want for any model you plan to revisit. For quick prototyping, linopy also accepts variables without those: sensible defaults fill in, and the variables still work end-to-end. This is convenient, deliberate, and what the rest of this section walks through.\n", + "\n", + "You can turn the implicit path off with `force_dim_names=True` (shown at the end of the section)." ] }, { "cell_type": "markdown", - "id": "8b1734df", + "id": "cv-31", "metadata": {}, "source": [ - "Again, one is always safer when explicitly naming the dimensions:" + "### Skipping `name=`\n", + "\n", + "Without `name=`, linopy assigns `var0`, `var1`, … in order. Cheap to write; the trade-off is that inspection by name becomes inspection by position:" ] }, { "cell_type": "code", "execution_count": null, - "id": "21f7db15", - "metadata": {}, + "id": "cv-32", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:57:23.755708Z", + "iopub.status.busy": "2026-05-20T07:57:23.755651Z", + "iopub.status.idle": "2026-05-20T07:57:23.757704Z", + "shell.execute_reply": "2026-05-20T07:57:23.757560Z" + } + }, "outputs": [], "source": [ - "lower = lower.rename_axis(index=\"my-dim\", columns=\"my-other-dim\")\n", - "upper = upper.rename_axis(\"my-dim\")\n", - "m.add_variables(lower, upper)" + "m.add_variables(lower=0, upper=10, integer=True)" ] }, { "cell_type": "markdown", - "id": "e8249281", + "id": "cv-33", "metadata": {}, "source": [ - ".. note::\n", - " As pandas objects already carry indexes, the ``coords`` argument is ignored when supplied alongside them. A warning is raised if a ``coords`` value is passed that does not align with the pandas object." + "### Skipping dimension names — the `dim_0` default\n", + "\n", + "A `DataArray` or `coords` value without a dimension name picks up xarray's default, `dim_0` (the second would be `dim_1`, and so on). Variables you add the same way share the same default name, so combining them aligns by position. Fine when that's what you mean; surprising when it isn't:" ] }, { "cell_type": "code", "execution_count": null, - "id": "d0fc67cf", - "metadata": {}, + "id": "cv-34", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:57:23.758552Z", + "iopub.status.busy": "2026-05-20T07:57:23.758499Z", + "iopub.status.idle": "2026-05-20T07:57:23.760966Z", + "shell.execute_reply": "2026-05-20T07:57:23.760810Z" + } + }, "outputs": [], "source": [ - "unaligned_coords = pd.Index([1, 2]), pd.Index([2, 3, 4])\n", - "m.add_variables(lower, upper, coords=unaligned_coords)" + "m.add_variables(xr.DataArray([1, 2, 3]), name=\"v_unnamed\")" ] }, { "cell_type": "markdown", - "id": "49de1cc3", + "id": "bea988b5", "metadata": {}, "source": [ - "### Masking Arrays\n", - "\n", - "In some cases, you want to create a variable with given dimensions, but not all parts should be active. \n", + ".. warning::\n", + " **Explicit dim names are what keeps broadcasting predictable.** linopy always pairs dimensions by name and broadcasts wherever names differ — that's the rule that lets a `(time × station)` variable emerge naturally when you mean it, and it's the same rule that fires when names mismatch by accident (a typo, a stale `rename_axis`, a `pandas` index whose name didn't carry through). Two length-2 vectors with mismatched names become a silent `2 × 2` variable, not two variables.\n", "\n", - "For example, think about an set of ports between which goods can be transported. However, a port cannot transport goods to itself. For such a case, you would create an variable `transport` which has the dimension (`from`, `to`) with values on the diagonal disabled. \n", - "\n", - "Therefore, you can pass a `mask` argument which has `False` values on the diagonal and `True` elsewhere." + " With explicit names you can tell at a glance whether a broadcast was intended; without them, the model still solves — just on a problem that isn't the one you meant. That's why every variable in this guide carries explicit dim names — and what `force_dim_names=True` below enforces." ] }, { "cell_type": "code", "execution_count": null, - "id": "9d802903", - "metadata": {}, + "id": "9ffb786a", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:57:23.761820Z", + "iopub.status.busy": "2026-05-20T07:57:23.761761Z", + "iopub.status.idle": "2026-05-20T07:57:23.764732Z", + "shell.execute_reply": "2026-05-20T07:57:23.764576Z" + } + }, "outputs": [], "source": [ - "ports = list(\"abcdef\")\n", - "port_from = pd.Index(ports, name=\"from\")\n", - "port_to = pd.Index(ports, name=\"to\")\n", - "\n", - "mask = np.ones((len(ports), len(ports)), dtype=bool)\n", - "np.fill_diagonal(mask, False)\n", - "mask" + "# Mismatched dim names: 'first' on lower, 'second' on upper → 2 × 2 broadcast.\n", + "lower = pd.Series([1, 1]).rename_axis(\"first\")\n", + "upper = pd.Series([10, 12]).rename_axis(\"second\")\n", + "m.add_variables(lower, upper, name=\"surprise\")" ] }, { - "cell_type": "code", - "execution_count": null, - "id": "447d8a8a", + "cell_type": "markdown", + "id": "cv-39", "metadata": {}, - "outputs": [], "source": [ - "transport = m.add_variables(\n", - " lower=0, coords=[port_from, port_to], name=\"transport\", mask=mask\n", - ")\n", - "transport" + "### Turning the implicit path off\n", + "\n", + "For larger projects, set `force_dim_names=True` on the model. Adding any variable with an unnamed dimension then raises immediately — the implicit `dim_0` defaults are closed off, and you can't accidentally rely on them:" ] }, { - "cell_type": "markdown", - "id": "df1c551c", - "metadata": {}, + "cell_type": "code", + "execution_count": null, + "id": "cv-40", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:57:23.765640Z", + "iopub.status.busy": "2026-05-20T07:57:23.765589Z", + "iopub.status.idle": "2026-05-20T07:57:23.768334Z", + "shell.execute_reply": "2026-05-20T07:57:23.768181Z" + } + }, + "outputs": [], "source": [ - "Now the diagonal values, for example at the variable at [a,a], are `None`. " + "strict = Model(force_dim_names=True)\n", + "try:\n", + " strict.add_variables(lower=np.array([1, 2]), coords=[pd.RangeIndex(2)])\n", + "except ValueError as e:\n", + " print(\"Raised:\", e)" ] }, { "cell_type": "markdown", - "id": "23a040d4", + "id": "cv-41", "metadata": {}, "source": [ - "### Accessing assigned variables\n", + "## Accessing variables on the model\n", "\n", - "All variables added to the model are stored in the `.variables` container." + "All variables added to a model live in `m.variables` and can be looked up by name:" ] }, { "cell_type": "code", "execution_count": null, - "id": "2946a80c", - "metadata": {}, + "id": "cv-42", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:57:23.769156Z", + "iopub.status.busy": "2026-05-20T07:57:23.769107Z", + "iopub.status.idle": "2026-05-20T07:57:23.770758Z", + "shell.execute_reply": "2026-05-20T07:57:23.770606Z" + } + }, "outputs": [], "source": [ "m.variables" ] }, - { - "cell_type": "markdown", - "id": "45cf0755", - "metadata": {}, - "source": [ - "You can always access the variables from the `.variables` container either by get-item, i.e." - ] - }, { "cell_type": "code", "execution_count": null, - "id": "d974727d", - "metadata": {}, + "id": "cv-43", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:57:23.771540Z", + "iopub.status.busy": "2026-05-20T07:57:23.771486Z", + "iopub.status.idle": "2026-05-20T07:57:23.773208Z", + "shell.execute_reply": "2026-05-20T07:57:23.773046Z" + } + }, "outputs": [], "source": [ "m.variables[\"x\"]" @@ -580,17 +628,24 @@ }, { "cell_type": "markdown", - "id": "03182836", + "id": "cv-44", "metadata": {}, "source": [ - "or by attribute accessing" + "Or by attribute, when the variable name is a valid Python identifier:" ] }, { "cell_type": "code", "execution_count": null, - "id": "308b879a", - "metadata": {}, + "id": "cv-45", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-20T07:57:23.774079Z", + "iopub.status.busy": "2026-05-20T07:57:23.774012Z", + "iopub.status.idle": "2026-05-20T07:57:23.775644Z", + "shell.execute_reply": "2026-05-20T07:57:23.775500Z" + } + }, "outputs": [], "source": [ "m.variables.x" @@ -598,10 +653,6 @@ } ], "metadata": { - "@webio": { - "lastCommId": null, - "lastKernelId": null - }, "kernelspec": { "display_name": "Python 3", "language": "python", @@ -617,7 +668,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.3" + "version": "3.11.11" } }, "nbformat": 4, diff --git a/examples/first-real-model.ipynb b/examples/first-real-model.ipynb new file mode 100644 index 00000000..6aa0f22a --- /dev/null +++ b/examples/first-real-model.ipynb @@ -0,0 +1,395 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "frm-01", + "metadata": {}, + "source": "# Your First Real Model: Economic Dispatch\n\nThe quick-start showed how a coordinate turns a scalar variable into a vector of decisions, and how `.sum()` and broadcasting line up naturally. Real models usually go a step further: more than one dimension and real input data flowing in from `pandas` or `xarray`.\n\nLet's put both into practice with a small **economic dispatch** problem: decide how much each of three generators should produce in each hour of a day so that demand is met at minimum cost." + }, + { + "cell_type": "markdown", + "id": "frm-02", + "metadata": {}, + "source": [ + "## The Problem\n", + "\n", + "Three generators — `coal`, `gas`, and `wind` — must cover an hourly demand profile over 24 hours. Each generator has a fixed capacity and marginal cost. We want to find the cheapest schedule.\n", + "\n", + "Formally, with generators indexed by $g$ and hours by $t$:\n", + "\n", + "$$\n", + "\\begin{aligned}\n", + "\\min_{p} \\quad & \\sum_{g,t} c_g \\, p_{g,t} \\\\\n", + "\\text{s.t.} \\quad & \\sum_g p_{g,t} = d_t & \\forall t \\\\\n", + "& 0 \\le p_{g,t} \\le \\bar p_g & \\forall g, t\n", + "\\end{aligned}\n", + "$$" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "frm-03", + "metadata": { + "ExecuteTime": { + "end_time": "2026-05-19T19:25:46.677893Z", + "start_time": "2026-05-19T19:25:46.662971Z" + }, + "execution": { + "iopub.execute_input": "2026-05-19T19:20:26.378081Z", + "iopub.status.busy": "2026-05-19T19:20:26.377911Z", + "iopub.status.idle": "2026-05-19T19:20:27.328424Z", + "shell.execute_reply": "2026-05-19T19:20:27.328161Z" + } + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "import linopy" + ] + }, + { + "cell_type": "markdown", + "id": "frm-04", + "metadata": {}, + "source": [ + "## The data\n", + "\n", + "Real input data usually arrives as `pandas` objects. linopy consumes them directly: each named index becomes a coordinate on the resulting variable." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "frm-05", + "metadata": { + "ExecuteTime": { + "end_time": "2026-05-19T19:25:46.720525Z", + "start_time": "2026-05-19T19:25:46.686338Z" + }, + "execution": { + "iopub.execute_input": "2026-05-19T19:20:27.329794Z", + "iopub.status.busy": "2026-05-19T19:20:27.329649Z", + "iopub.status.idle": "2026-05-19T19:20:27.336083Z", + "shell.execute_reply": "2026-05-19T19:20:27.335909Z" + } + }, + "outputs": [], + "source": [ + "generators = pd.DataFrame(\n", + " {\n", + " \"capacity\": [200, 100, 80], # MW\n", + " \"marginal_cost\": [30.0, 60.0, 2.0], # $/MWh\n", + " },\n", + " index=pd.Index([\"coal\", \"gas\", \"wind\"], name=\"generator\"),\n", + ")\n", + "generators" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "frm-06", + "metadata": { + "ExecuteTime": { + "end_time": "2026-05-19T19:25:46.785997Z", + "start_time": "2026-05-19T19:25:46.731856Z" + }, + "execution": { + "iopub.execute_input": "2026-05-19T19:20:27.337188Z", + "iopub.status.busy": "2026-05-19T19:20:27.337113Z", + "iopub.status.idle": "2026-05-19T19:20:27.397639Z", + "shell.execute_reply": "2026-05-19T19:20:27.397433Z" + } + }, + "outputs": [], + "source": [ + "hours = pd.RangeIndex(24, name=\"hour\")\n", + "demand = pd.Series(\n", + " [\n", + " 200,\n", + " 200,\n", + " 200,\n", + " 200,\n", + " 210,\n", + " 230,\n", + " 260,\n", + " 290,\n", + " 310,\n", + " 320,\n", + " 330,\n", + " 340,\n", + " 350,\n", + " 350,\n", + " 350,\n", + " 340,\n", + " 330,\n", + " 310,\n", + " 290,\n", + " 270,\n", + " 250,\n", + " 230,\n", + " 220,\n", + " 210,\n", + " ],\n", + " index=hours,\n", + " name=\"demand\",\n", + ")\n", + "demand.plot(title=\"Hourly demand (MW)\", figsize=(6, 2.5));" + ] + }, + { + "cell_type": "markdown", + "id": "frm-07", + "metadata": {}, + "source": [ + "## Building the model\n", + "\n", + "The decision variable `p` represents the generation of each unit in each hour. We declare it with two coordinates — `generator` and `hour` — so it has shape `(3, 24)`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "frm-08", + "metadata": { + "ExecuteTime": { + "end_time": "2026-05-19T19:25:46.815393Z", + "start_time": "2026-05-19T19:25:46.794534Z" + }, + "execution": { + "iopub.execute_input": "2026-05-19T19:20:27.398679Z", + "iopub.status.busy": "2026-05-19T19:20:27.398602Z", + "iopub.status.idle": "2026-05-19T19:20:27.665745Z", + "shell.execute_reply": "2026-05-19T19:20:27.665542Z" + } + }, + "outputs": [], + "source": [ + "m = linopy.Model()\n", + "\n", + "p = m.add_variables(\n", + " lower=0,\n", + " coords=[generators.index, hours],\n", + " name=\"generation\",\n", + ")\n", + "p" + ] + }, + { + "cell_type": "markdown", + "id": "frm-09", + "metadata": {}, + "source": [ + "## Constraints\n", + "\n", + "**Demand balance.** In every hour, the sum of all generation must equal demand. `p.sum(\"generator\")` collapses `p` along the `generator` dimension, leaving an expression with shape `(hour,)` — which lines up with `demand` automatically." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "frm-10", + "metadata": { + "ExecuteTime": { + "end_time": "2026-05-19T19:25:46.841126Z", + "start_time": "2026-05-19T19:25:46.816893Z" + }, + "execution": { + "iopub.execute_input": "2026-05-19T19:20:27.666852Z", + "iopub.status.busy": "2026-05-19T19:20:27.666729Z", + "iopub.status.idle": "2026-05-19T19:20:27.674537Z", + "shell.execute_reply": "2026-05-19T19:20:27.674345Z" + } + }, + "outputs": [], + "source": [ + "m.add_constraints(\n", + " p.sum(\"generator\") == demand,\n", + " name=\"demand_balance\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "frm-11", + "metadata": {}, + "source": [ + "**Capacity.** Generation of each unit cannot exceed its capacity. The capacity is a `(generator,)`-shaped Series; linopy broadcasts it across the `hour` dimension of `p` automatically." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "frm-12", + "metadata": { + "ExecuteTime": { + "end_time": "2026-05-19T19:25:46.865562Z", + "start_time": "2026-05-19T19:25:46.843084Z" + }, + "execution": { + "iopub.execute_input": "2026-05-19T19:20:27.675549Z", + "iopub.status.busy": "2026-05-19T19:20:27.675474Z", + "iopub.status.idle": "2026-05-19T19:20:27.681869Z", + "shell.execute_reply": "2026-05-19T19:20:27.681700Z" + } + }, + "outputs": [], + "source": [ + "m.add_constraints(\n", + " p <= generators[\"capacity\"].to_xarray(),\n", + " name=\"capacity\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "frm-13", + "metadata": {}, + "source": [ + "## Objective\n", + "\n", + "Minimize total cost: multiply each unit's generation by its marginal cost, then sum across both dimensions." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "frm-14", + "metadata": { + "ExecuteTime": { + "end_time": "2026-05-19T19:25:46.874055Z", + "start_time": "2026-05-19T19:25:46.866534Z" + }, + "execution": { + "iopub.execute_input": "2026-05-19T19:20:27.682743Z", + "iopub.status.busy": "2026-05-19T19:20:27.682683Z", + "iopub.status.idle": "2026-05-19T19:20:27.686201Z", + "shell.execute_reply": "2026-05-19T19:20:27.686049Z" + } + }, + "outputs": [], + "source": [ + "m.add_objective(\n", + " (p * generators[\"marginal_cost\"].to_xarray()).sum(),\n", + " sense=\"min\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "frm-15", + "metadata": {}, + "source": [ + "## Solve" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "frm-16", + "metadata": { + "ExecuteTime": { + "end_time": "2026-05-19T19:25:46.920314Z", + "start_time": "2026-05-19T19:25:46.874802Z" + }, + "execution": { + "iopub.execute_input": "2026-05-19T19:20:27.687346Z", + "iopub.status.busy": "2026-05-19T19:20:27.687281Z", + "iopub.status.idle": "2026-05-19T19:20:27.868850Z", + "shell.execute_reply": "2026-05-19T19:20:27.868681Z" + } + }, + "outputs": [], + "source": [ + "m.solve()" + ] + }, + { + "cell_type": "markdown", + "id": "frm-17", + "metadata": {}, + "source": [ + "## Inspecting the solution\n", + "\n", + "The solution is stored on the variable as an xarray `DataArray`, with the same coordinates we declared. Convert it to a `DataFrame` for a tabular view, or plot it directly." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "frm-18", + "metadata": { + "ExecuteTime": { + "end_time": "2026-05-19T19:25:46.933960Z", + "start_time": "2026-05-19T19:25:46.921985Z" + }, + "execution": { + "iopub.execute_input": "2026-05-19T19:20:27.869907Z", + "iopub.status.busy": "2026-05-19T19:20:27.869761Z", + "iopub.status.idle": "2026-05-19T19:20:27.875267Z", + "shell.execute_reply": "2026-05-19T19:20:27.875118Z" + } + }, + "outputs": [], + "source": [ + "p.solution.to_pandas().T.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "frm-19", + "metadata": { + "ExecuteTime": { + "end_time": "2026-05-19T19:25:46.998731Z", + "start_time": "2026-05-19T19:25:46.934657Z" + }, + "execution": { + "iopub.execute_input": "2026-05-19T19:20:27.876117Z", + "iopub.status.busy": "2026-05-19T19:20:27.876040Z", + "iopub.status.idle": "2026-05-19T19:20:27.929840Z", + "shell.execute_reply": "2026-05-19T19:20:27.929571Z" + } + }, + "outputs": [], + "source": [ + "ax = p.solution.to_pandas().T.plot.area(\n", + " figsize=(8, 3.5),\n", + " title=\"Hourly dispatch (MW)\",\n", + ")\n", + "demand.plot(ax=ax, color=\"black\", linewidth=1.5, label=\"demand\")\n", + "ax.set_ylabel(\"MW\")\n", + "ax.legend();" + ] + }, + { + "cell_type": "markdown", + "id": "frm-20", + "metadata": {}, + "source": "## Recap\n\nA few things to notice in the code above:\n\n- The decision variable `p` is 2-dimensional (`generator × hour`). linopy carries those labels everywhere, including in the solution.\n- `p.sum(\"generator\")` collapses along the named dimension to produce an expression with shape `(hour,)`, which lines up with `demand` automatically.\n- The capacity constraint `p <= capacity` broadcasts a 1-D `(generator,)` array across the 2-D `(generator, hour)` variable — no Python loops, no manual indexing.\n- The cheapest generator (`wind`, at 2 $/MWh) runs flat out, `coal` covers most of the rest, and `gas` only kicks in at peak hours.\n\nThe same patterns scale up: add more generators, longer horizons, additional dimensions like regions or scenarios — the model code does not change shape." + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/infeasible-model.ipynb b/examples/infeasible-model.ipynb deleted file mode 100644 index def2113c..00000000 --- a/examples/infeasible-model.ipynb +++ /dev/null @@ -1,116 +0,0 @@ -{ - "cells": [ - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Trouble shooting - Infeasible Model\n", - "\n", - "From time to time, you encounter models that are infeasible. This means that there is no solution that satisfies all the constraints. Some solvers allow you to get the set of constraints that are infeasible. This is useful for debugging your model. \n", - "\n", - "In the following, we show how `linopy` can help you with the infeasibility diagnostics by finding the set of constraints that are infeasible. So far the functionality is limited to the `gurobi` solver. Hopefully, we will be able to extend this to other solvers in the future.\n", - "\n", - "We start by creating a simple model that is infeasible." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": "import pandas as pd\n\nimport linopy\n\nm = linopy.Model()\n\ntime = pd.RangeIndex(10, name=\"time\")\nx = m.add_variables(lower=0, coords=[time], name=\"x\")\ny = m.add_variables(lower=0, coords=[time], name=\"y\")\n\nm.add_constraints(x <= 5)\nm.add_constraints(y <= 5)\nm.add_constraints(x + y >= 12)\n\n# A trivial objective is required; the model is solved purely to check feasibility.\nm.add_objective(0 * x)" - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If we now try to solve the model, we get an message that the model is infeasible." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "m.solve(solver_name=\"gurobi\")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now, we can use the model in the background to find the set of infeasible constraints. The following code will return a list constraint labels that are infeasible." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "labels = m.compute_infeasibilities()\n", - "labels" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Using the `print_labels` function, we can print the constraints that we found to be infeasible." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "m.constraints.print_labels(labels)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The two function calls above can be combined into a single call:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "m.print_infeasibilities()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "base", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - }, - "orig_nbformat": 4 - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/examples/migrating-from-pyomo.ipynb b/examples/migrating-from-pyomo.ipynb index 3d34ce60..fc0b6e5e 100644 --- a/examples/migrating-from-pyomo.ipynb +++ b/examples/migrating-from-pyomo.ipynb @@ -2,18 +2,32 @@ "cells": [ { "cell_type": "markdown", - "id": "ded90143", + "id": "532d13cf", "metadata": {}, "source": [ - "## Migrating from Pyomo\n", + "# Migrating from Pyomo\n", "\n", - "Similar to the implementation in Pyomo, expressions and constraints can be created using a combination of a function and a set of coordinates to iterate over. For creating expressions, the function itself has to return a `ScalarLinearExpression` which can be obtained by selecting single values of the variables are combining them: " + "Coming to linopy from [Pyomo](https://pyomo.org/), the main shift is from **rule functions indexed over a set** to **vectorised array operations**. linopy's everyday API is vectorised — that is what the *Creating a Model* section covers.\n", + "\n", + "But linopy also supports the Pyomo-style \"function over coordinates\" pattern directly, through `model.linexpr` and the `.at` indexer. This notebook walks through that pattern, so an existing Pyomo model can be ported with minimal restructuring.\n", + "\n", + "For a side-by-side syntax comparison and performance numbers, see [Benchmarks and syntax](tool-comparison.rst)." + ] + }, + { + "cell_type": "markdown", + "id": "a3379e1e", + "metadata": {}, + "source": [ + "## Scalar access with `.at`\n", + "\n", + "A linopy `Variable` holds a whole array of decision variables at once. To pick out a single scalar entry — the equivalent of `m.x[i, j]` in Pyomo — use the `.at` indexer." ] }, { "cell_type": "code", "execution_count": null, - "id": "19f3b954", + "id": "91b693a3", "metadata": {}, "outputs": [], "source": [ @@ -30,7 +44,7 @@ { "cell_type": "code", "execution_count": null, - "id": "2bbfd13b", + "id": "b766171c", "metadata": {}, "outputs": [], "source": [ @@ -39,21 +53,18 @@ }, { "cell_type": "markdown", - "id": "a1631a76", + "id": "a7b8dccd", "metadata": {}, "source": [ - ".. important::\n", - " The creation of scalar variables has changed in version `0.3.10` to use the `.at[]` method. When creating a `ScalarVariable` with the `[]` operator, a future warning is raised. The `[]` operator will reserver for integer and boolean indexing only, aligning to the xarray functionality. \n", - "\n", + "## Expressions from a rule function\n", "\n", - "\n", - "Such a `ScalarVariable` is very light-weight and can be used in functions in order to create expressions, just like you know it from `Pyomo`. The following function shows how:" + "A `ScalarVariable` is light-weight and can be combined inside a function to build an expression, just like in Pyomo. Pass the rule function and the coordinates to `model.linexpr`:" ] }, { "cell_type": "code", "execution_count": null, - "id": "4ed6eafb", + "id": "5d201151", "metadata": {}, "outputs": [], "source": [ @@ -70,34 +81,26 @@ }, { "cell_type": "markdown", - "id": "4faecead", + "id": "dc840fc9", "metadata": {}, "source": [ - "Note that the function's first argument has to be the model itself, even though it might not be used in the function." + "The first argument of the rule function must be the model itself, even when — as here — it is unused." ] }, { "cell_type": "markdown", - "id": "d7368607", + "id": "72938767", "metadata": {}, "source": [ - "This functionality is also supported by the `.add_constraints` function. When passing a function as a first argument, `.add_constraints` expects `coords` to by non-empty. The function itself has to return a `AnonymousScalarConstraint`, as done by " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "eeebb710", - "metadata": {}, - "outputs": [], - "source": [ - "x.at[0, \"a\"] <= 3" + "## Constraints from a rule function\n", + "\n", + "The same shape works for `model.add_constraints`. When the first argument is a function, `add_constraints` expects a non-empty `coords` and a function that returns an `AnonymousScalarConstraint`:" ] }, { "cell_type": "code", "execution_count": null, - "id": "087203ad", + "id": "f5856c17", "metadata": {}, "outputs": [], "source": [ @@ -111,13 +114,23 @@ "con = m.add_constraints(bound, coords=coords)\n", "con" ] + }, + { + "cell_type": "markdown", + "id": "4e296086", + "metadata": {}, + "source": [ + ".. note::\n", + " In most cases linopy's vectorised API is faster to write *and* faster to build than the rule-function shape. Reach for ``linexpr(rule, coords)`` only when per-index logic genuinely varies — otherwise prefer broadcast arithmetic.\n", + "\n", + "## Where to next\n", + "\n", + "- [Benchmarks and syntax](tool-comparison.rst) — linopy versus Pyomo, JuMP and GAMS on speed, memory, and syntax.\n", + "- [Transport tutorial](transport-tutorial.ipynb) — a full GAMS-style worked problem." + ] } ], "metadata": { - "@webio": { - "lastCommId": null, - "lastKernelId": null - }, "kernelspec": { "display_name": "Python 3", "language": "python", @@ -133,7 +146,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.3" + "version": "3.11.11" } }, "nbformat": 4, diff --git a/examples/quick-start.ipynb b/examples/quick-start.ipynb new file mode 100644 index 00000000..10b1f5d9 --- /dev/null +++ b/examples/quick-start.ipynb @@ -0,0 +1,317 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "qs-01", + "metadata": {}, + "source": [ + "# Quick start\n", + "\n", + "This is a fast tour of linopy's core moves: declare variables, write constraints, set an objective, solve. We start with a scalar model so the syntax is bare, then add a coordinate so you see how linopy scales to real data." + ] + }, + { + "cell_type": "markdown", + "id": "qs-02", + "metadata": {}, + "source": [ + "## A scalar model\n", + "\n", + "$$\n", + "\\begin{aligned}\n", + "\\min_{x, y} \\quad & x + 2y \\\\\n", + "\\text{s.t.} \\quad & 3x + 7y \\ge 10 \\\\\n", + "& 5x + 2y \\ge 3 \\\\\n", + "& x, y \\ge 0\n", + "\\end{aligned}\n", + "$$" + ] + }, + { + "cell_type": "code", + "id": "qs-03", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-19T19:26:42.804244Z", + "iopub.status.busy": "2026-05-19T19:26:42.804031Z", + "iopub.status.idle": "2026-05-19T19:26:43.597708Z", + "shell.execute_reply": "2026-05-19T19:26:43.597457Z" + }, + "ExecuteTime": { + "end_time": "2026-05-19T19:26:55.488236Z", + "start_time": "2026-05-19T19:26:54.734695Z" + } + }, + "source": [ + "import linopy\n", + "\n", + "m = linopy.Model()" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "id": "qs-04", + "metadata": {}, + "source": [ + "Each call to `add_variables` returns a `Variable` object. With no coordinates, it represents a single scalar decision." + ] + }, + { + "cell_type": "code", + "id": "qs-05", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-19T19:26:43.598993Z", + "iopub.status.busy": "2026-05-19T19:26:43.598879Z", + "iopub.status.idle": "2026-05-19T19:26:43.602442Z", + "shell.execute_reply": "2026-05-19T19:26:43.602258Z" + }, + "ExecuteTime": { + "end_time": "2026-05-19T19:26:55.505389Z", + "start_time": "2026-05-19T19:26:55.495176Z" + } + }, + "source": [ + "x = m.add_variables(lower=0, name=\"x\")\n", + "y = m.add_variables(lower=0, name=\"y\")\n", + "x" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "id": "qs-06", + "metadata": {}, + "source": [ + "Constraints are written in natural mathematical notation. linopy figures out which side has the variables and which has the constants — the two forms below are equivalent." + ] + }, + { + "cell_type": "code", + "id": "qs-07", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-19T19:26:43.603401Z", + "iopub.status.busy": "2026-05-19T19:26:43.603323Z", + "iopub.status.idle": "2026-05-19T19:26:43.609554Z", + "shell.execute_reply": "2026-05-19T19:26:43.609345Z" + }, + "ExecuteTime": { + "end_time": "2026-05-19T19:26:55.528141Z", + "start_time": "2026-05-19T19:26:55.511197Z" + } + }, + "source": [ + "3 * x + 7 * y >= 10" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "id": "qs-08", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-19T19:26:43.610558Z", + "iopub.status.busy": "2026-05-19T19:26:43.610487Z", + "iopub.status.idle": "2026-05-19T19:26:43.619764Z", + "shell.execute_reply": "2026-05-19T19:26:43.619592Z" + }, + "ExecuteTime": { + "end_time": "2026-05-19T19:26:55.540721Z", + "start_time": "2026-05-19T19:26:55.528865Z" + } + }, + "source": [ + "m.add_constraints(3 * x + 7 * y >= 10, name=\"c1\")\n", + "m.add_constraints(5 * x + 2 * y >= 3, name=\"c2\");" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "id": "qs-09", + "metadata": {}, + "source": [ + "Set the objective and solve." + ] + }, + { + "cell_type": "code", + "id": "qs-10", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-19T19:26:43.620689Z", + "iopub.status.busy": "2026-05-19T19:26:43.620632Z", + "iopub.status.idle": "2026-05-19T19:26:43.658654Z", + "shell.execute_reply": "2026-05-19T19:26:43.658410Z" + }, + "ExecuteTime": { + "end_time": "2026-05-19T19:26:55.582697Z", + "start_time": "2026-05-19T19:26:55.541306Z" + } + }, + "source": [ + "m.add_objective(x + 2 * y)\n", + "m.solve(solver_name=\"highs\", output_flag=False)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "id": "qs-11", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-19T19:26:43.659665Z", + "iopub.status.busy": "2026-05-19T19:26:43.659585Z", + "iopub.status.idle": "2026-05-19T19:26:43.661380Z", + "shell.execute_reply": "2026-05-19T19:26:43.661215Z" + }, + "ExecuteTime": { + "end_time": "2026-05-19T19:26:55.609294Z", + "start_time": "2026-05-19T19:26:55.584415Z" + } + }, + "source": [ + "float(x.solution), float(y.solution)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "id": "qs-12", + "metadata": {}, + "source": [ + "## Adding coordinates\n", + "\n", + "Real optimization problems have data: prices indexed by time, capacities indexed by asset, demand indexed by region. linopy variables can carry the same named coordinates as your `pandas` or `xarray` data — the syntax doesn't change.\n", + "\n", + "Let's parameterise the right-hand side by a time index $t \\in \\{0, \\dots, 9\\}$:\n", + "\n", + "$$\n", + "\\begin{aligned}\n", + "\\min \\quad & \\sum_t (x_t + 2 y_t) \\\\\n", + "\\text{s.t.} \\quad & 3 x_t + 7 y_t \\ge 10 t & \\forall t \\\\\n", + "& 5 x_t + 2 y_t \\ge 3 t & \\forall t \\\\\n", + "& x_t, y_t \\ge 0 & \\forall t\n", + "\\end{aligned}\n", + "$$" + ] + }, + { + "cell_type": "code", + "id": "qs-13", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-19T19:26:43.662261Z", + "iopub.status.busy": "2026-05-19T19:26:43.662188Z", + "iopub.status.idle": "2026-05-19T19:26:43.666991Z", + "shell.execute_reply": "2026-05-19T19:26:43.666837Z" + }, + "ExecuteTime": { + "end_time": "2026-05-19T19:26:55.627868Z", + "start_time": "2026-05-19T19:26:55.610419Z" + } + }, + "source": [ + "import pandas as pd\n", + "\n", + "time = pd.Index(range(10), name=\"time\")\n", + "\n", + "m = linopy.Model()\n", + "x = m.add_variables(lower=0, coords=[time], name=\"x\")\n", + "y = m.add_variables(lower=0, coords=[time], name=\"y\")\n", + "x" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "id": "qs-14", + "metadata": {}, + "source": "`x` and `y` now hold ten variables each, labelled by `time`. Constraints and objectives written with them broadcast over that coordinate automatically — `t` on the right-hand side aligns with the `time` dimension on the left.\n\nThe payoff: because variables share coordinates with your input data, scaling from ten timesteps to ten thousand is just a matter of feeding larger inputs. The model code stays the same shape — no loops to grow, no indexers to rewrite." + }, + { + "cell_type": "code", + "id": "qs-15", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-19T19:26:43.667863Z", + "iopub.status.busy": "2026-05-19T19:26:43.667794Z", + "iopub.status.idle": "2026-05-19T19:26:43.699974Z", + "shell.execute_reply": "2026-05-19T19:26:43.699732Z" + }, + "ExecuteTime": { + "end_time": "2026-05-19T19:26:55.666925Z", + "start_time": "2026-05-19T19:26:55.628902Z" + } + }, + "source": [ + "t = pd.Series(time, index=time)\n", + "\n", + "m.add_constraints(3 * x + 7 * y >= 10 * t, name=\"c1\")\n", + "m.add_constraints(5 * x + 2 * y >= 3 * t, name=\"c2\")\n", + "\n", + "m.add_objective((x + 2 * y).sum())\n", + "m.solve(solver_name=\"highs\", output_flag=False);" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "id": "qs-16", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-19T19:26:43.701016Z", + "iopub.status.busy": "2026-05-19T19:26:43.700946Z", + "iopub.status.idle": "2026-05-19T19:26:43.890568Z", + "shell.execute_reply": "2026-05-19T19:26:43.890333Z" + }, + "ExecuteTime": { + "end_time": "2026-05-19T19:26:55.728605Z", + "start_time": "2026-05-19T19:26:55.667375Z" + } + }, + "source": [ + "m.solution.to_dataframe().plot(grid=True, ylabel=\"value\", figsize=(6, 3));" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "id": "qs-17", + "metadata": {}, + "source": "## Where to next\n\n- [Your first real model](first-real-model.ipynb) walks through a small economic-dispatch problem with two dimensions and real input data.\n- The **Creating a Model** section covers variables, expressions, constraints, and coordinate alignment in depth.\n- [Solving a model](solving.ipynb) covers choosing a solver and inspecting it afterwards; [The Solver API](solver-api.ipynb) is the lower-level construct-then-solve interface." + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/remote-machines.ipynb b/examples/remote-machines.ipynb new file mode 100644 index 00000000..cd1931cd --- /dev/null +++ b/examples/remote-machines.ipynb @@ -0,0 +1,191 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "intro", + "metadata": {}, + "source": [ + "# Remote machines\n", + "\n", + "linopy can ship your model to a remote machine, run a solver there, and pull the solved model back. Two transports are supported:\n", + "\n", + "- **SSH** — connect to a server you own (or have access to) over SSH.\n", + "- **OETC** — submit jobs to [OET Cloud](https://open-energy-transition.org/), a managed optimization service.\n", + "\n", + "Both share the same entry point on `Model.solve`:\n", + "\n", + "```python\n", + "m.solve(\"gurobi\", remote=, **solver_options)\n", + "```\n", + "\n", + "`solver_name` and `**solver_options` work exactly like a local solve; `remote=` selects *where* to run. After the call, `model.remote` holds the transport instance for post-solve introspection (mirrors `model.solver`)." + ] + }, + { + "cell_type": "markdown", + "id": "note", + "metadata": {}, + "source": [ + "> **Note:** This notebook is not executed during the documentation build — it requires either SSH access to a remote server or OETC credentials." + ] + }, + { + "cell_type": "markdown", + "id": "model-header", + "metadata": {}, + "source": [ + "## Create a model\n", + "\n", + "Build the model locally as usual:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "model", + "metadata": {}, + "outputs": [], + "source": [ + "from numpy import arange\n", + "from xarray import DataArray\n", + "\n", + "from linopy import Model\n", + "\n", + "N = 10\n", + "m = Model()\n", + "coords = [arange(N), arange(N)]\n", + "x = m.add_variables(coords=coords, name=\"x\")\n", + "y = m.add_variables(coords=coords, name=\"y\")\n", + "m.add_constraints(x - y >= DataArray(arange(N)))\n", + "m.add_constraints(x + y >= 0)\n", + "m.add_objective((2 * x + y).sum())\n", + "m" + ] + }, + { + "cell_type": "markdown", + "id": "ssh-header", + "metadata": {}, + "source": "## SSH\n\n**What you need**\n\n- `uv pip install \"linopy[ssh]\"` locally (pulls in `paramiko`).\n- A remote server with linopy and a solver installed (e.g. in a conda environment).\n- SSH access to that machine (key-based auth recommended).\n\nBuild an `SshSettings` and pass it as `remote=`. Use `setup_commands` to activate environments or export variables on the remote shell before the solve." + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ssh-solve", + "metadata": {}, + "outputs": [], + "source": [ + "from linopy.remote import SshSettings\n", + "\n", + "ssh_settings = SshSettings(\n", + " hostname=\"your.host.de\",\n", + " username=\"username\",\n", + " # password=\"...\", # not needed when SSH keys are autodetected\n", + " setup_commands=[\"conda activate linopy-env\"],\n", + ")\n", + "\n", + "m.solve(\"gurobi\", remote=ssh_settings)\n", + "m.solution" + ] + }, + { + "cell_type": "markdown", + "id": "oetc-header", + "metadata": {}, + "source": "## OETC\n\n**What you need**\n\n- `uv pip install \"linopy[oetc]\"` locally (pulls in `google-cloud-storage` and `requests`).\n- An OETC account with valid credentials.\n- The OETC authentication and orchestrator server URLs.\n\nBuild an `OetcSettings`. Two construction styles:\n\n1. **Manually** — pass `email`, `password`, `name`, and the server URLs.\n2. **`OetcSettings.from_env()`** — resolve everything from environment variables (`OETC_EMAIL`, `OETC_PASSWORD`, `OETC_NAME`, `OETC_AUTH_URL`, `OETC_ORCHESTRATOR_URL`). Recommended for CI/CD. Keyword arguments override the environment.\n\nlinopy uploads the model to OETC, submits a compute job, polls until it finishes, and downloads the solution — all behind one call." + }, + { + "cell_type": "code", + "execution_count": null, + "id": "oetc-solve", + "metadata": {}, + "outputs": [], + "source": "from linopy.remote import OetcSettings\n\n# Option 1: pass credentials directly\noetc_settings = OetcSettings(\n email=\"your-email@example.com\",\n password=\"your-password\",\n name=\"linopy-example-job\",\n authentication_server_url=\"https://auth.oetcloud.com\",\n orchestrator_server_url=\"https://orchestrator.oetcloud.com\",\n cpu_cores=4,\n disk_space_gb=20,\n)\n\n# Option 2: load from environment (with optional overrides)\noetc_settings = OetcSettings.from_env(cpu_cores=4, disk_space_gb=20)\n\nm.solve(\"gurobi\", remote=oetc_settings, TimeLimit=600, MIPGap=0.01)\n\nprint(f\"Status: {m.status}\")\nprint(f\"Objective: {m.objective.value:.4f}\")\nm.solution" + }, + { + "cell_type": "markdown", + "id": "advanced-header", + "metadata": {}, + "source": [ + "## Advanced: drive the transport directly\n", + "\n", + "For finer control — inspecting the round-tripped solved model, splitting submit from collect for async workflows — use the `Oetc` or `SSH` class directly. `Model.solve(remote=...)` runs the same path internally and then writes the result back onto the local model in place.\n", + "\n", + "### SSH" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "advanced-ssh", + "metadata": {}, + "outputs": [], + "source": [ + "from linopy.remote import SSH\n", + "\n", + "ssh = SSH(\n", + " settings=ssh_settings,\n", + " solver_name=\"gurobi\",\n", + " options={\"presolve\": \"on\"},\n", + ")\n", + "result = ssh.solve(m)\n", + "m.assign_result(result)" + ] + }, + { + "cell_type": "markdown", + "id": "advanced-oetc-header", + "metadata": {}, + "source": [ + "### OETC\n", + "\n", + "`Oetc` exposes the three steps `Model.solve(remote=...)` does internally:\n", + "\n", + "1. `upload(model)` — serialize and push the netcdf to OETC.\n", + "2. `submit()` — submit the compute job; returns the job uuid.\n", + "3. `collect(model)` — wait for completion, download, build the `Result`.\n", + "\n", + "Splitting them lets you fire off a job, do other work, and come back to collect later — useful for long-running jobs or async-style workflows." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "advanced-oetc", + "metadata": {}, + "outputs": [], + "source": [ + "from linopy.remote import Oetc\n", + "\n", + "oetc = Oetc(\n", + " settings=oetc_settings,\n", + " solver_name=\"gurobi\",\n", + " options={\"TimeLimit\": 600, \"MIPGap\": 0.01},\n", + ")\n", + "\n", + "oetc.upload(m)\n", + "job_uuid = oetc.submit()\n", + "print(f\"Submitted job {job_uuid} — do other work here ...\")\n", + "\n", + "# Later (or in another process holding `oetc`):\n", + "result = oetc.collect(m)\n", + "m.assign_result(result)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + }, + "nbsphinx": { + "execute": "never" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/solve-on-oetc.ipynb b/examples/solve-on-oetc.ipynb deleted file mode 100644 index f6c5c67d..00000000 --- a/examples/solve-on-oetc.ipynb +++ /dev/null @@ -1,431 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Solve on OETC (OET Cloud)\n", - "\n", - "This example demonstrates how to use linopy with OETC (OET Cloud) for cloud-based optimization solving. OETC is a cloud platform that provides scalable computing resources for optimization problems.\n", - "\n", - "## What you need to run this example:\n", - "\n", - "* A working installation of the required packages:\n", - " * `pip install google-cloud-storage requests`\n", - "* An OETC account with valid credentials (email and password)\n", - "* Access to OETC authentication and orchestrator servers\n", - "\n", - "## How OETC Cloud Solving Works\n", - "\n", - "The OETC integration follows this workflow:\n", - "\n", - "1. **Model Creation**: Define your optimization model locally using linopy\n", - "2. **Authentication**: Sign in to the OETC platform using your credentials\n", - "3. **File Upload**: Compress and upload your model to Google Cloud Storage\n", - "4. **Job Submission**: Submit a compute job to the OETC orchestrator\n", - "5. **Job Monitoring**: Wait for job completion with automatic status polling\n", - "6. **Solution Download**: Download and decompress the solved model\n", - "7. **Local Integration**: Load the solution back into your local model\n", - "\n", - "All of these steps are handled automatically by linopy's `OetcHandler`." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "> **Note:** This notebook requires Google Cloud credentials and access to the OETC platform. It is not executed during the documentation build, so no cell outputs are shown. To run it yourself, install the `linopy[oetc]` extra and configure your credentials." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Create a Model\n", - "\n", - "First, let's create an optimization model that we want to solve on OETC:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from numpy import arange\n", - "from xarray import DataArray\n", - "\n", - "from linopy import Model\n", - "\n", - "# Create a medium-sized optimization problem\n", - "N = 50\n", - "m = Model()\n", - "\n", - "# Define decision variables with coordinates\n", - "coords = [arange(N), arange(N)]\n", - "x = m.add_variables(coords=coords, name=\"x\", lower=0)\n", - "y = m.add_variables(coords=coords, name=\"y\", lower=0)\n", - "\n", - "# Add constraints\n", - "m.add_constraints(x - y >= DataArray(arange(N)), name=\"constraint1\")\n", - "m.add_constraints(x + y >= DataArray(arange(N) * 0.5), name=\"constraint2\")\n", - "m.add_constraints(x <= DataArray(arange(N) + 10), name=\"upper_bounds\")\n", - "\n", - "# Set objective function\n", - "m.add_objective((2 * x + y).sum())\n", - "\n", - "print(\n", - " f\"Model created with {len(m.variables)} variable groups and {len(m.constraints)} constraint groups\"\n", - ")\n", - "m" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Configure OETC Settings\n", - "\n", - "There are two ways to configure OETC settings:\n", - "\n", - "1. **Manual construction** \u2014 build `OetcCredentials` and `OetcSettings` explicitly\n", - "2. **`OetcSettings.from_env()`** \u2014 resolve credentials and options from environment variables\n", - "\n", - "### Option 1: Manual Construction" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Configure your OETC credentials\n", - "# IMPORTANT: Never hardcode credentials in production code!\n", - "# Use environment variables or secure credential management\n", - "import os\n", - "\n", - "from linopy.remote.oetc import (\n", - " ComputeProvider,\n", - " OetcCredentials,\n", - " OetcHandler,\n", - " OetcSettings,\n", - ")\n", - "\n", - "credentials = OetcCredentials(\n", - " email=os.getenv(\"OETC_EMAIL\", \"your-email@example.com\"),\n", - " password=os.getenv(\"OETC_PASSWORD\", \"your-password\"),\n", - ")\n", - "\n", - "# Configure OETC settings\n", - "settings = OetcSettings(\n", - " credentials=credentials,\n", - " name=\"linopy-example-job\",\n", - " authentication_server_url=\"https://auth.oetcloud.com\", # Replace with actual URL\n", - " orchestrator_server_url=\"https://orchestrator.oetcloud.com\", # Replace with actual URL\n", - " compute_provider=ComputeProvider.GCP,\n", - " cpu_cores=4, # Number of CPU cores to allocate\n", - " disk_space_gb=20, # Disk space in GB\n", - " delete_worker_on_error=False, # Keep worker for debugging if job fails\n", - ")\n", - "\n", - "print(\"OETC settings configured successfully\")\n", - "print(f\"Solver: {settings.solver}\")\n", - "print(f\"CPU cores: {settings.cpu_cores}\")\n", - "print(f\"Disk space: {settings.disk_space_gb} GB\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Option 2: Create Settings from Environment Variables\n", - "\n", - "`OetcSettings.from_env()` reads configuration from environment variables,\n", - "with optional keyword overrides. This is the recommended approach for\n", - "CI/CD pipelines and production deployments.\n", - "\n", - "| Environment Variable | Required | Description |\n", - "|---|---|---|\n", - "| `OETC_EMAIL` | Yes | Account email |\n", - "| `OETC_PASSWORD` | Yes | Account password |\n", - "| `OETC_NAME` | Yes | Job name |\n", - "| `OETC_AUTH_URL` | Yes | Authentication server URL |\n", - "| `OETC_ORCHESTRATOR_URL` | Yes | Orchestrator server URL |\n", - "| `OETC_CPU_CORES` | No | CPU cores (default: 2) |\n", - "| `OETC_DISK_SPACE_GB` | No | Disk space in GB (default: 10) |\n", - "| `OETC_DELETE_WORKER_ON_ERROR` | No | Delete worker on error (default: false) |\n", - "\n", - "Keyword arguments take precedence over environment variables." - ] - }, - { - "cell_type": "code", - "metadata": {}, - "outputs": [], - "source": [ - "# Create settings from environment variables\n", - "# All required env vars must be set: OETC_EMAIL, OETC_PASSWORD,\n", - "# OETC_NAME, OETC_AUTH_URL, OETC_ORCHESTRATOR_URL\n", - "settings = OetcSettings.from_env()\n", - "\n", - "# Or override specific values via keyword arguments\n", - "settings = OetcSettings.from_env(\n", - " cpu_cores=8,\n", - " disk_space_gb=50,\n", - ")" - ], - "execution_count": null - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Initialize OETC Handler\n", - "\n", - "The `OetcHandler` manages the entire cloud solving process:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Initialize the OETC handler\n", - "# This will authenticate with OETC and fetch cloud provider credentials\n", - "oetc_handler = OetcHandler(settings)\n", - "\n", - "print(\"OETC handler initialized successfully\")\n", - "print(f\"Authentication token expires at: {oetc_handler.jwt.expires_at}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Solve the Model on OETC\n", - "\n", - "Now we can solve our model on the OETC cloud platform. The `OetcHandler` is passed to the model's `solve()` method:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Solve the model on OETC\n", - "# This will upload the model, submit a job, wait for completion, and download the solution\n", - "import time\n", - "\n", - "print(\"Starting cloud solving process...\")\n", - "start_time = time.time()\n", - "\n", - "try:\n", - " status, termination_condition = m.solve(remote=oetc_handler, solver_name=\"highs\")\n", - "\n", - " end_time = time.time()\n", - " total_time = end_time - start_time\n", - "\n", - " print(f\"\\nSolving completed in {total_time:.2f} seconds\")\n", - " print(f\"Status: {status}\")\n", - " print(f\"Termination condition: {termination_condition}\")\n", - " print(f\"Objective value: {m.objective.value:.4f}\")\n", - "\n", - "except Exception as e:\n", - " print(f\"Error during solving: {e}\")\n", - " raise" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Examine the Solution\n", - "\n", - "Let's examine the solution returned from OETC:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Display solution summary\n", - "print(f\"Model status: {m.status}\")\n", - "print(f\"Objective value: {m.objective.value}\")\n", - "print(f\"Number of variables: {m.solution.sizes}\")\n", - "\n", - "# Show a subset of the solution\n", - "print(\"\\nSample of solution values:\")\n", - "print(\"x values (first 5x5):\")\n", - "print(m.solution[\"x\"].isel(dim_0=slice(0, 5), dim_1=slice(0, 5)).values)\n", - "\n", - "print(\"\\ny values (first 5x5):\")\n", - "print(m.solution[\"y\"].isel(dim_0=slice(0, 5), dim_1=slice(0, 5)).values)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Advanced OETC Configuration\n", - "\n", - "### Solver Options\n", - "\n", - "Solver name and options can be configured at two levels:\n", - "\n", - "1. **Settings level** \u2014 defaults stored in `OetcSettings.solver` and `OetcSettings.solver_options`\n", - "2. **Call level** \u2014 passed via `m.solve(solver_name=..., **solver_options)`\n", - "\n", - "Call-level options **override** settings-level options. The two dicts are\n", - "merged (call-time takes precedence), and the original settings are never\n", - "mutated." - ] - }, - { - "cell_type": "code", - "metadata": {}, - "outputs": [], - "source": [ - "# Settings-level defaults\n", - "advanced_settings = OetcSettings(\n", - " credentials=credentials,\n", - " name=\"advanced-linopy-job\",\n", - " authentication_server_url=\"https://auth.oetcloud.com\",\n", - " orchestrator_server_url=\"https://orchestrator.oetcloud.com\",\n", - " solver=\"gurobi\",\n", - " solver_options={\n", - " \"TimeLimit\": 600,\n", - " \"MIPGap\": 0.01,\n", - " },\n", - " cpu_cores=8,\n", - " disk_space_gb=50,\n", - ")\n", - "\n", - "advanced_handler = OetcHandler(advanced_settings)\n", - "\n", - "# Call-level overrides: solver_name and solver_options are forwarded\n", - "# to OETC and merged with the settings defaults.\n", - "# Here MIPGap from settings (0.01) is kept, TimeLimit is overridden to 300.\n", - "status, condition = m.solve(\n", - " remote=advanced_handler,\n", - " solver_name=\"gurobi\",\n", - " TimeLimit=300,\n", - " Threads=4,\n", - ")" - ], - "execution_count": null - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Error Handling and Debugging\n", - "\n", - "When working with cloud solving, it's important to handle potential errors gracefully:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def solve_with_error_handling(model, oetc_handler, max_retries=3):\n", - " \"\"\"Solve model with error handling and retries\"\"\"\n", - "\n", - " for attempt in range(max_retries):\n", - " try:\n", - " print(f\"Solving attempt {attempt + 1}/{max_retries}...\")\n", - " status, termination = model.solve(remote=oetc_handler)\n", - "\n", - " if status == \"ok\":\n", - " print(\"Solving successful!\")\n", - " return status, termination\n", - " else:\n", - " print(f\"Solving returned status: {status}\")\n", - "\n", - " except Exception as e:\n", - " print(f\"Attempt {attempt + 1} failed: {e}\")\n", - "\n", - " if attempt < max_retries - 1:\n", - " print(\"Retrying in 30 seconds...\")\n", - " time.sleep(30)\n", - " else:\n", - " print(\"All attempts failed\")\n", - " raise\n", - "\n", - " return None, None\n", - "\n", - "\n", - "# Example usage (commented out to avoid actual execution)\n", - "# status, termination = solve_with_error_handling(m, oetc_handler)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Security Best Practices\n", - "\n", - "When using OETC in production:\n", - "\n", - "1. **Never hardcode credentials**: Use environment variables or secure credential stores\n", - "2. **Use token expiration**: The OETC handler automatically manages token expiration\n", - "3. **Validate inputs**: Ensure your model data doesn't contain sensitive information\n", - "4. **Monitor costs**: Cloud computing resources have associated costs\n", - "5. **Clean up resources**: Set `delete_worker_on_error=True` for automatic cleanup\n", - "\n", - "## Comparison with SSH Remote Solving\n", - "\n", - "| Feature | OETC Cloud | SSH Remote |\n", - "|---------|------------|------------|\n", - "| Setup | Account registration | Server access required |\n", - "| Scalability | Auto-scaling | Fixed server resources |\n", - "| Maintenance | Managed service | Self-managed |\n", - "| Cost | Pay-per-use | Infrastructure costs |\n", - "| Security | Enterprise-grade | Self-managed |\n", - "| Solver Licenses | Included | User-provided |\n", - "\n", - "Choose OETC for:\n", - "- Large-scale problems requiring significant compute resources\n", - "- Temporary or intermittent optimization needs\n", - "- Teams without dedicated infrastructure\n", - "- Access to premium solvers without license management\n", - "\n", - "Choose SSH remote for:\n", - "- Existing infrastructure with optimization solvers\n", - "- Strict data governance requirements\n", - "- Consistent, long-running optimization workloads\n", - "- Full control over the solving environment" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.3" - }, - "nbsphinx": { - "execute": "never" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/examples/solve-on-remote.ipynb b/examples/solve-on-remote.ipynb deleted file mode 100644 index 73e6346b..00000000 --- a/examples/solve-on-remote.ipynb +++ /dev/null @@ -1,655 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "4db583af", - "metadata": {}, - "source": [ - "# Remote Solving with SSH\n", - "\n", - "This example demonstrates how linopy can solve optimization models on remote machines using SSH connections. This is one of two remote solving options available in linopy:\n", - "\n", - "1. **SSH Remote Solving** (this example) - Connect to your own servers via SSH\n", - "2. **OETC Cloud Solving** - Use cloud-based optimization services (see [OETC notebook](solve-on-oetc.ipynb))\n", - "\n", - "## SSH Remote Solving\n", - "\n", - "SSH remote solving is ideal when you have:\n", - "\n", - "* Access to dedicated servers with optimization solvers installed\n", - "* Full control over the computing environment\n", - "* Existing infrastructure for optimization workloads\n", - "\n", - "## What you need for SSH remote solving\n", - "\n", - "* The `remote` extra installed on your local machine (`uv pip install \"linopy[remote]\"`), which pulls in `paramiko`\n", - "* A remote server with a working installation of linopy (e.g., in a conda environment)\n", - "* SSH access to that machine\n", - "\n", - "## How SSH Remote Solving Works\n", - "\n", - "The workflow consists of the following steps, most of which linopy handles automatically:\n", - "\n", - "1. Define a model on the local machine\n", - "2. Save the model on the remote machine via SSH\n", - "3. Load, solve and write out the model on the remote machine\n", - "4. Copy the solved model back to the local machine\n", - "5. Load the solved model on the local machine\n", - "\n", - "The model initialization happens locally, while the actual solving happens remotely.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "> **Note:** This notebook requires SSH access to a remote server with a solver installed. It is not executed during the documentation build, so no cell outputs are shown. To run it yourself, configure SSH access and install a solver on the remote machine." - ] - }, - { - "cell_type": "markdown", - "id": "together-ocean", - "metadata": {}, - "source": [ - "## Create a model\n", - "\n", - "First we are going to build the optimization model we want to solve in our local process." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "dramatic-cannon", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Linopy LP model\n", - "===============\n", - "\n", - "Variables:\n", - "----------\n", - " * x (dim_0, dim_1)\n", - " * y (dim_0, dim_1)\n", - "\n", - "Constraints:\n", - "------------\n", - " * con0 (dim_0, dim_1)\n", - " * con1 (dim_0, dim_1)\n", - "\n", - "Status:\n", - "-------\n", - "initialized" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from numpy import arange\n", - "from xarray import DataArray\n", - "\n", - "from linopy import Model\n", - "\n", - "N = 10\n", - "m = Model()\n", - "coords = [arange(N), arange(N)]\n", - "x = m.add_variables(coords=coords, name=\"x\")\n", - "y = m.add_variables(coords=coords, name=\"y\")\n", - "m.add_constraints(x - y >= DataArray(arange(N)))\n", - "m.add_constraints(x + y >= 0)\n", - "m.add_objective((2 * x + y).sum())\n", - "m" - ] - }, - { - "cell_type": "markdown", - "id": "0f9e9b09", - "metadata": {}, - "source": [ - "## Initialize SSH connection\n", - "\n", - "Now we have to set up the SSH connection. The SSH connection is handled by the `RemoteHandler` class in of the `linopy.remote` module. This is strongly relying on the `paramiko` package. When initializing, you have two options:\n", - "\n", - "1. Pass the standard arguments `host`, `username`. If the SSH keys are stored in a default location, the keys are autodetected and the `RemoteHandler` does not require the `password` argument. Otherwise you also have to pass the password.\n", - "2. Pass a working `paramiko.SSHClient` as `client`. This enables you to set up the SSH connection by others means supported by `paramiko`. \n", - "\n", - "In the following we use the first option." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "protecting-power", - "metadata": {}, - "outputs": [], - "source": [ - "from linopy import RemoteHandler\n", - "\n", - "host = \"your.host.de\"\n", - "username = \"username\"\n", - "\n", - "handler = RemoteHandler(host, username=username)" - ] - }, - { - "cell_type": "markdown", - "id": "featured-maria", - "metadata": {}, - "source": [ - "## Optionally: Activate a conda environment on the remote \n", - "\n", - "The `RemoteHandler` keeps an interactive shell in the background. You can execute any code in order to prepare the solving process (install linopy, activate an environment). \n", - "\n", - "Assuming you have a conda environment `linopy-env` that contains the `linopy` package with dependencies, you can run " - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "virtual-anxiety", - "metadata": {}, - "outputs": [], - "source": [ - "handler.execute(\"conda activate linopy-env\")" - ] - }, - { - "cell_type": "markdown", - "id": "sonic-rebate", - "metadata": {}, - "source": [ - "## Solve the model on remote\n", - "\n", - "Now the only thing you have to do is to pass the `RemoteHandler` as an argument to the `solve` function. Other keyword arguments like `solver_name` and solver options are propagated to the remote machine. " - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "ongoing-desktop", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Set parameter Username\n", - "Academic license - for non-commercial use only - expires 2023-02-06\n", - "Read LP format model from file /tmp/linopy-problem-uh4gvjyp.lp\n", - "Reading time = 0.00 seconds\n", - "obj: 200 rows, 200 columns, 400 nonzeros\n", - "Gurobi Optimizer version 9.5.1 build v9.5.1rc2 (linux64)\n", - "Thread count: 12 physical cores, 24 logical processors, using up to 24 threads\n", - "Optimize a model with 200 rows, 200 columns and 400 nonzeros\n", - "Model fingerprint: 0xf2bcac49\n", - "Coefficient statistics:\n", - "Matrix range [1e+00, 1e+00]\n", - "Objective range [1e+00, 2e+00]\n", - "Bounds range [0e+00, 0e+00]\n", - "RHS range [1e+00, 9e+00]\n", - "Presolve removed 200 rows and 200 columns\n", - "Presolve time: 0.00s\n", - "Presolve: All rows and columns removed\n", - "Iteration Objective Primal Inf. Dual Inf. Time\n", - "0 2.2500000e+02 0.000000e+00 0.000000e+00 0s\n", - "\n", - "Solved in 0 iterations and 0.00 seconds (0.00 work units)\n", - "Optimal objective 2.250000000e+02\n" - ] - }, - { - "data": { - "text/plain": [ - "('ok', '')" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "m.solve(remote=handler)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "sustained-portrait", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
<xarray.Dataset>\n",
-       "Dimensions:  (dim_0: 10, dim_1: 10)\n",
-       "Coordinates:\n",
-       "  * dim_0    (dim_0) int64 0 1 2 3 4 5 6 7 8 9\n",
-       "  * dim_1    (dim_1) int64 0 1 2 3 4 5 6 7 8 9\n",
-       "Data variables:\n",
-       "    x        (dim_0, dim_1) float64 0.0 0.0 0.0 0.0 0.0 ... 4.5 4.5 4.5 4.5 4.5\n",
-       "    y        (dim_0, dim_1) float64 0.0 0.0 0.0 0.0 0.0 ... -4.5 -4.5 -4.5 -4.5
" - ], - "text/plain": [ - "\n", - "Dimensions: (dim_0: 10, dim_1: 10)\n", - "Coordinates:\n", - " * dim_0 (dim_0) int64 0 1 2 3 4 5 6 7 8 9\n", - " * dim_1 (dim_1) int64 0 1 2 3 4 5 6 7 8 9\n", - "Data variables:\n", - " x (dim_0, dim_1) float64 0.0 0.0 0.0 0.0 0.0 ... 4.5 4.5 4.5 4.5 4.5\n", - " y (dim_0, dim_1) float64 0.0 0.0 0.0 0.0 0.0 ... -4.5 -4.5 -4.5 -4.5" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "m.solution" - ] - } - ], - "metadata": { - "@webio": { - "lastCommId": null, - "lastKernelId": null - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - }, - "nbsphinx": { - "execute": "never" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/solver-api.ipynb b/examples/solver-api.ipynb new file mode 100644 index 00000000..3fcb7a20 --- /dev/null +++ b/examples/solver-api.ipynb @@ -0,0 +1,181 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "sa-01", + "metadata": {}, + "source": "# The Solver API\n\n`model.solve(...)` is enough for almost every use case — see [Solving a model](solving.ipynb). When you need finer control — adjusting the native solver model before it runs, obtaining the `Result` object directly, or bracketing model transformations by hand — you can build a `Solver` in one step and run it in another." + }, + { + "cell_type": "code", + "execution_count": null, + "id": "sa-02", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "from linopy import Model\n", + "from linopy.solvers import Solver\n", + "\n", + "\n", + "def build_model():\n", + " m = Model()\n", + " x = m.add_variables(lower=0, name=\"x\")\n", + " y = m.add_variables(lower=0, name=\"y\")\n", + " m.add_constraints(3 * x + 7 * y >= 10)\n", + " m.add_constraints(5 * x + 2 * y >= 3)\n", + " m.add_objective(x + 2 * y)\n", + " return m" + ] + }, + { + "cell_type": "markdown", + "id": "sa-03", + "metadata": {}, + "source": [ + "## Building the solver\n", + "\n", + "Build the solver with `Solver.from_name`, then run it with `.solve()`. The two steps are separate, so you can inspect or adjust the solver between them:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "sa-04", + "metadata": {}, + "outputs": [], + "source": [ + "m = build_model()\n", + "solver = Solver.from_name(\"highs\", m, io_api=\"direct\", options={\"output_flag\": False})\n", + "solver" + ] + }, + { + "cell_type": "markdown", + "id": "sa-05", + "metadata": {}, + "source": [ + "`solver.solver_model` is the native solver handle — here a `highspy.Highs` instance. You could tweak it directly before running:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "sa-06", + "metadata": {}, + "outputs": [], + "source": [ + "solver.solver_model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "sa-07", + "metadata": {}, + "outputs": [], + "source": [ + "result = solver.solve()\n", + "result" + ] + }, + { + "cell_type": "markdown", + "id": "sa-08", + "metadata": {}, + "source": [ + "`Result` carries the status, solution, solver name, and report. Writing it back into the `Model`, combining numeric values with labels and coordinates, is a separate call:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "sa-09", + "metadata": {}, + "outputs": [], + "source": [ + "m.assign_result(result)\n", + "m.objective.value" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "sa-10", + "metadata": {}, + "outputs": [], + "source": [ + "m.solution.to_pandas()" + ] + }, + { + "cell_type": "markdown", + "id": "sa-11", + "metadata": {}, + "source": [ + "## Model transformations\n", + "\n", + "**Model transformations live on the `Model`, not the `Solver`.** A `Solver` only declares which features it supports and raises during `_build()` if it can't handle the model it's been handed; it never mutates the model. The transformations currently exposed:\n", + "\n", + "| Transformation | Methods on `Model` | Reversible? |\n", + "|---|---|---|\n", + "| SOS reformulation (rewrite SOS constraints as Big-M binary + linear) | `model.apply_sos_reformulation()` / `model.undo_sos_reformulation()` | yes |\n", + "| Drop zero-coefficient terms | `model.constraints.sanitize_zeros()` | one-way |\n", + "| Replace ±inf bounds in constraints | `model.constraints.sanitize_infinities()` | one-way |\n", + "\n", + "`model.solve(reformulate_sos=True, sanitize_zeros=True, sanitize_infinities=True)` is a convenience that **brackets** these around the one-shot solve (and undoes the SOS reformulation afterwards). The two-step `Solver` API does **not** do this for you — when you go through `Solver.from_name(...).solve()`, you call the transformations yourself first, and use `try`/`finally` to keep the model in a known state if the solve raises:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "sa-12", + "metadata": {}, + "outputs": [], + "source": [ + "m = Model()\n", + "i = pd.Index([0, 1, 2], name=\"i\")\n", + "x = m.add_variables(lower=0, upper=1, coords=[i], name=\"x\")\n", + "m.add_sos_constraints(x, sos_type=1, sos_dim=\"i\")\n", + "m.add_objective(x.sum(), sense=\"max\")\n", + "\n", + "m.constraints.sanitize_zeros()\n", + "m.constraints.sanitize_infinities()\n", + "\n", + "m.apply_sos_reformulation()\n", + "try:\n", + " solver = Solver.from_name(\n", + " \"highs\", m, io_api=\"direct\", options={\"output_flag\": False}\n", + " )\n", + " result = solver.solve()\n", + " m.assign_result(result)\n", + "finally:\n", + " m.undo_sos_reformulation() # restore original SOS form on the Model\n", + "\n", + "list(m.variables.sos)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/solving-troubleshooting.ipynb b/examples/solving-troubleshooting.ipynb new file mode 100644 index 00000000..2caa6035 --- /dev/null +++ b/examples/solving-troubleshooting.ipynb @@ -0,0 +1,119 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Troubleshooting\n", + "\n", + "Things that can go wrong when solving a linopy model, and how to diagnose\n", + "them. Each section below is self-contained -- jump to whichever matches your\n", + "symptom. This page will grow as more solver issues are documented." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Infeasible models\n\nFrom time to time you hit a model that is **infeasible** -- no assignment of\nthe variables satisfies every constraint at once. Some solvers can report\n*which* constraints are in conflict, which is the fastest way to track down\nthe cause.\n\nlinopy exposes this through `compute_infeasibilities` and\n`format_infeasibilities`.\n\nWe start by creating a small model that is infeasible." + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2026-05-20T11:32:52.564804Z", + "start_time": "2026-05-20T11:32:51.692874Z" + } + }, + "source": "import pandas as pd\n\nimport linopy\n\nm = linopy.Model()\n\ntime = pd.RangeIndex(10, name=\"time\")\nx = m.add_variables(lower=0, coords=[time], name=\"x\")\ny = m.add_variables(lower=0, coords=[time], name=\"y\")\n\nm.add_constraints(x <= 5)\nm.add_constraints(y <= 5)\nm.add_constraints(x + y >= 12)\n\n# A trivial objective is required; the model is solved purely to check feasibility.\nm.add_objective(0 * x)", + "outputs": [], + "execution_count": null + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": "If we now try to solve the model, we get a message that the model is infeasible." + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2026-05-20T11:32:52.667586Z", + "start_time": "2026-05-20T11:32:52.569893Z" + } + }, + "source": [ + "m.solve(solver_name=\"gurobi\")" + ], + "outputs": [], + "execution_count": null + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": "When the solver reports infeasibility, `format_infeasibilities()` runs an IIS (irreducible infeasible subsystem) analysis and returns a human-readable summary of the conflicting constraints. This depends on the solver -- it currently works with `gurobi`, and not every solver supports it.\n\nIf you instead need those constraints as data -- to relax or remove them in code -- `compute_infeasibilities()` returns their labels as a list." + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2026-05-20T11:32:52.772257Z", + "start_time": "2026-05-20T11:32:52.747757Z" + } + }, + "source": "print(m.format_infeasibilities())", + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Duplicate Gurobi log lines\n", + "\n", + "When using the Gurobi solver you may see some of the log lines it emits during\n", + "the solve duplicated:\n", + "\n", + "```\n", + "Total elapsed time = 498.27s\n", + "[INFO] Total elapsed time = 498.27s\n", + "```\n", + "\n", + "This happens because the Gurobi logger both prints to the console and\n", + "propagates to the root logger. Adding the following to your application code\n", + "before calling `solve` fixes it:\n", + "\n", + "```python\n", + "import logging\n", + "\n", + "logging.getLogger(\"gurobipy\").propagate = False\n", + "```\n", + "\n", + "See [this thread](https://groups.google.com/g/gurobi/c/sV7xxN_mzCk) for more." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "base", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/solving.ipynb b/examples/solving.ipynb new file mode 100644 index 00000000..e8f03ad1 --- /dev/null +++ b/examples/solving.ipynb @@ -0,0 +1,226 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "sv-01", + "metadata": {}, + "source": [ + "# Solving a model\n", + "\n", + "Once a model is built, `model.solve(...)` hands it to a solver backend — HiGHS, Gurobi, CPLEX, CBC, GLPK, SCIP, Xpress, MOSEK, MindOpt, COPT, Knitro, or the GPU solver cuPDLPx — runs it, and writes the solution back into the variables. This page covers the everyday workflow:\n", + "\n", + "- the one-step `model.solve(...)` call,\n", + "- inspecting the solver afterwards via `model.solver` and `SolverReport`,\n", + "- listing installed and licensed solvers.\n", + "\n", + "For finer control — building a `Solver` yourself, adjusting the native solver model, or bracketing model transformations by hand — see [The Solver API](solver-api.ipynb)." + ] + }, + { + "cell_type": "markdown", + "id": "sv-02", + "metadata": {}, + "source": [ + "## A small example model\n", + "\n", + "We'll use a tiny LP throughout the page. Minimize $x + 2y$ subject to $x, y \\ge 0$, $3x + 7y \\ge 10$, $5x + 2y \\ge 3$." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "sv-03", + "metadata": {}, + "outputs": [], + "source": [ + "import linopy\n", + "from linopy import Model\n", + "\n", + "\n", + "def build_model():\n", + " m = Model()\n", + " x = m.add_variables(lower=0, name=\"x\")\n", + " y = m.add_variables(lower=0, name=\"y\")\n", + " m.add_constraints(3 * x + 7 * y >= 10)\n", + " m.add_constraints(5 * x + 2 * y >= 3)\n", + " m.add_objective(x + 2 * y)\n", + " return m" + ] + }, + { + "cell_type": "markdown", + "id": "sv-04", + "metadata": {}, + "source": [ + "## One-step solving\n", + "\n", + "`model.solve` picks the first available solver, runs it, writes the solution back into the variables, and returns a `(status, termination_condition)` tuple. You can specify which solver you want." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "sv-05", + "metadata": {}, + "outputs": [], + "source": [ + "m = build_model()\n", + "status, termination = m.solve(solver_name=\"highs\", output_flag=False)\n", + "status, termination" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "sv-06", + "metadata": {}, + "outputs": [], + "source": [ + "m.objective.value" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "sv-07", + "metadata": {}, + "outputs": [], + "source": [ + "m.solution.to_pandas()" + ] + }, + { + "cell_type": "markdown", + "id": "sv-08", + "metadata": {}, + "source": [ + "## After solving\n", + "\n", + "After `model.solve(...)` the solver instance stays attached to the model as `model.solver`. You can read off the solver name, the native solver model, the status and — new in this release — a `SolverReport` with runtime, MIP gap, dual (best) bound, and iteration counts." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "sv-09", + "metadata": {}, + "outputs": [], + "source": [ + "m.solver" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "sv-10", + "metadata": {}, + "outputs": [], + "source": [ + "m.solver.report" + ] + }, + { + "cell_type": "markdown", + "id": "sv-11", + "metadata": {}, + "source": [ + "Not every backend fills in every field of `SolverReport` — if a solver doesn't expose a value it stays `None`. `mip_gap` and `dual_bound` are most informative on MIPs." + ] + }, + { + "cell_type": "markdown", + "id": "sv-12", + "metadata": {}, + "source": [ + "Some solvers (Gurobi, MOSEK, …) hold a license while the underlying handle is alive. You can release it explicitly:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "sv-13", + "metadata": {}, + "outputs": [], + "source": [ + "m.solver.close() # frees the native handle (and license)\n", + "# or, to also detach the wrapper:\n", + "m.solver = None\n", + "m.solver, m.solver_name" + ] + }, + { + "cell_type": "markdown", + "id": "sv-14", + "metadata": {}, + "source": [ + "## Available solvers\n", + "\n", + "Two registries are exposed at the top level:\n", + "\n", + "- `linopy.available_solvers` — solvers whose Python package or binary is **installed**. Cheap; does not acquire a license.\n", + "- `linopy.licensed_solvers` — the subset that currently passes a **license** probe. Useful in tests or to pick a solver at runtime." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "sv-15", + "metadata": {}, + "outputs": [], + "source": [ + "list(linopy.available_solvers)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "sv-16", + "metadata": {}, + "outputs": [], + "source": [ + "list(linopy.licensed_solvers)" + ] + }, + { + "cell_type": "markdown", + "id": "sv-17", + "metadata": {}, + "source": [ + "Both are lazy and refreshable — call `linopy.available_solvers.refresh()` after installing or licensing a new solver in the same process. For a per-solver probe use `SolverClass.license_status()`, which returns a `LicenseStatus` dataclass:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "sv-18", + "metadata": {}, + "outputs": [], + "source": [ + "from linopy.solvers import Highs\n", + "\n", + "Highs.license_status()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/transport-tutorial.ipynb b/examples/transport-tutorial.ipynb index cd5cdccd..61a89285 100644 --- a/examples/transport-tutorial.ipynb +++ b/examples/transport-tutorial.ipynb @@ -3,57 +3,7 @@ { "cell_type": "markdown", "metadata": {}, - "source": [ - "# The Transport Problem\n", - "\n", - "## Summary\n", - "\n", - "The goal of the Transport Problem is to select the quantities of a homogeneous good that has several production plants and several punctiform markets as to minimise the transportation costs.\n", - "\n", - "It is the default tutorial for the GAMS language, and GAMS equivalent code is inserted as single-dash comments. The original GAMS code needs slighly different ordering of the commands and it's available at http://www.gams.com/mccarl/trnsport.gms. The Pyomo version of the tutorial can be found at https://nbviewer.org/github/Pyomo/PyomoGallery/blob/master/transport/transport.ipynb. \n", - "\n", - "For comparison, the variable names have been kept the same, though they are not pythonic. The equivalent GAMS code is included at the top of each cell as comments, while the Pyomo-equivalent is stated as text comments below.\n", - "\n", - "## Problem Statement\n", - "\n", - "The Transport Problem can be formulated mathematically as a linear programming problem using the following model. \n", - "\n", - "### Sets\n", - "\n", - " $I$ = set of canning plants \n", - " $J$ = set of markets\n", - "\n", - "### Parameters\n", - "\n", - " $a_i$ = capacity of plant $i$ in cases, $\\forall i \\in I$
\n", - " $b_j$ = demand at market $j$ in cases, $\\forall j \\in J$
\n", - " $d_{i,j}$ = distance in thousands of miles, $\\forall i \\in I, \\forall j \\in J$
\n", - " $f$ = freight in dollars per case per thousand miles
\n", - " $c_{i,j}$ = transport cost in thousands of dollars per case\n", - " \n", - " $c_{i,j}$ is obtained exougenously to the optimisation problem as $c_{i,j} = f \\cdot d_{i,j}$, $\\forall i \\in I, \\forall j \\in J$\n", - " \n", - "### Variables\n", - " $x_{i,j}$ = shipment quantities in cases
\n", - " z = total transportation costs in thousands of dollars\n", - "\n", - "### Objective\n", - "\n", - "Minimize the total cost of the shipments:
\n", - "$\\min_{x} z = \\sum_{i \\in I} \\sum_{j \\in J} c_{i,j} x_{i,j}$\n", - "\n", - "### Constraints\n", - "\n", - "\n", - "Observe supply limit at plant i:
\n", - " $\\sum_{j \\in J} x_{i,j} \\leq a_{i}$, $\\forall i \\in I$\n", - " \n", - "Satisfy demand at market j:
\n", - " $\\sum_{i \\in I} x_{i,j} \\geq b_{j}$, $\\forall j \\in J$\n", - "\n", - "Non-negative transportation quantities
\n", - " $x_{i,j} \\geq 0$, $\\forall i \\in I, \\forall j \\in J$" - ] + "source": "# The Transport Problem\n\nThis notebook reproduces the classic [GAMS transport problem](http://www.gams.com/mccarl/trnsport.gms) — the default tutorial for the GAMS language — in linopy. It sits in the *Coming from Other Tools* section as a **migration aid**: every code cell carries the equivalent GAMS code as single-dash comments, with the Pyomo equivalent shown below it, so a reader porting a model can follow the construct-by-construct mapping directly. For a broader linopy-versus-other-tools comparison, see [Benchmarks and syntax](tool-comparison.rst).\n\nTo keep the comparison faithful, the variable names match the GAMS original and are deliberately not pythonic. A [Pyomo version](https://nbviewer.org/github/Pyomo/PyomoGallery/blob/master/transport/transport.ipynb) of the same tutorial is also available.\n\n## Problem Statement\n\nThe goal of the transport problem is to select shipment quantities of a homogeneous good — produced at several canning plants, consumed at several punctiform markets — that minimise total transportation cost. It is formulated as a linear program using the following model.\n\n### Sets\n\n $I$ = set of canning plants \n $J$ = set of markets\n\n### Parameters\n\n $a_i$ = capacity of plant $i$ in cases, $\\forall i \\in I$
\n $b_j$ = demand at market $j$ in cases, $\\forall j \\in J$
\n $d_{i,j}$ = distance in thousands of miles, $\\forall i \\in I, \\forall j \\in J$
\n $f$ = freight in dollars per case per thousand miles
\n $c_{i,j}$ = transport cost in thousands of dollars per case\n \n $c_{i,j}$ is obtained exogenously to the optimisation problem as $c_{i,j} = f \\cdot d_{i,j}$, $\\forall i \\in I, \\forall j \\in J$\n \n### Variables\n $x_{i,j}$ = shipment quantities in cases
\n z = total transportation costs in thousands of dollars\n\n### Objective\n\nMinimize the total cost of the shipments:
\n$\\min_{x} z = \\sum_{i \\in I} \\sum_{j \\in J} c_{i,j} x_{i,j}$\n\n### Constraints\n\n\nObserve supply limit at plant i:
\n $\\sum_{j \\in J} x_{i,j} \\leq a_{i}$, $\\forall i \\in I$\n \nSatisfy demand at market j:
\n $\\sum_{i \\in I} x_{i,j} \\geq b_{j}$, $\\forall j \\in J$\n\nNon-negative transportation quantities
\n $x_{i,j} \\geq 0$, $\\forall i \\in I, \\forall j \\in J$" }, { "cell_type": "markdown", @@ -391,7 +341,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "A [range of solvers](https://linopy.readthedocs.io/en/latest/prerequisites.html#install-a-solver) can be used for linopy. You can check which solvers are available using: " + "A [range of solvers](https://linopy.readthedocs.io/en/latest/installation.html#install-a-solver) can be used for linopy. You can check which solvers are available using: " ] }, { diff --git a/linopy/model.py b/linopy/model.py index 250d65fe..0f197485 100644 --- a/linopy/model.py +++ b/linopy/model.py @@ -79,11 +79,13 @@ add_piecewise_formulation, ) from linopy.remote import RemoteHandler +from linopy.remote.ssh import SshSettings try: - from linopy.remote import OetcHandler + from linopy.remote import OetcHandler, OetcSettings except ImportError: OetcHandler = None # type: ignore + OetcSettings = None # type: ignore from linopy.solver_capabilities import solver_supports from linopy.solvers import ( IO_APIS, @@ -94,6 +96,7 @@ SOSReformulationResult, reformulate_sos_constraints, sos_reformulation_context, + suppress_serialization_warning, undo_sos_reformulation, ) from linopy.types import ( @@ -111,6 +114,14 @@ logger = logging.getLogger(__name__) +# Types accepted as ``remote=`` for the standalone-class dispatch in +# :meth:`Model.solve` (as opposed to the legacy ``OetcHandler`` / +# ``RemoteHandler`` deprecation path). The OETC entry is conditional on +# the optional google-cloud / requests deps being available. +_REMOTE_SETTINGS_TYPES: tuple[type, ...] = (SshSettings,) +if OetcSettings is not None: + _REMOTE_SETTINGS_TYPES = (*_REMOTE_SETTINGS_TYPES, OetcSettings) + def _coords_to_dict( coords: Sequence[Sequence | pd.Index | DataArray] | Mapping, @@ -196,6 +207,7 @@ class Model: """ _solver: solvers.Solver | None + _remote: Any _variables: Variables _constraints: Constraints _objective: Objective @@ -243,6 +255,7 @@ class Model: "_relaxed_registry", "_piecewise_formulations", "_solver", + "_remote", "_sos_reformulation_state", "__weakref__", ) @@ -314,6 +327,7 @@ def __init__( gettempdir() if solver_dir is None else solver_dir ) self._solver: solvers.Solver | None = None + self._remote: Any = None self._sos_reformulation_state: SOSReformulationResult | None = None @property @@ -326,6 +340,24 @@ def solver(self, value: solvers.Solver | None) -> None: self._solver.close() self._solver = value + @property + def remote(self) -> Any: + """ + Standalone remote-handler instance from the most recent solve, or ``None``. + + Set by :meth:`solve` when called with ``remote=``; lets + callers introspect handler state after the solve (e.g. + ``model.remote._job_uuid`` on OETC). ``None`` for local solves + and after a legacy ``remote=OetcHandler/RemoteHandler`` solve + (those are routed through the same path but the legacy handlers + aren't designed for post-solve inspection). + """ + return self._remote + + @remote.setter + def remote(self, value: Any) -> None: + self._remote = value + @property def solver_model(self) -> Any: return self.solver.solver_model if self.solver is not None else None @@ -1622,7 +1654,7 @@ def solve( sanitize_zeros: bool = True, sanitize_infinities: bool = True, slice_size: int = 2_000_000, - remote: RemoteHandler | OetcHandler | None = None, + remote: RemoteHandler | OetcHandler | OetcSettings | SshSettings | None = None, progress: bool | None = None, mock_solve: bool = False, reformulate_sos: bool | Literal["auto"] = False, @@ -1691,8 +1723,7 @@ def solve( is used to split large variables and constraints into smaller chunks to avoid memory issues. The default is 2_000_000. remote : linopy.remote.RemoteHandler | linopy.oetc.OetcHandler, optional - Remote handler to use for solving model on a server. Note that when - solving on a rSee + Remote handler to use for solving the model on a server. See linopy.remote.RemoteHandler for more details. progress : bool, optional Whether to show a progress bar of writing the lp file. The default is @@ -1727,50 +1758,37 @@ def solve( f"Keyword argument `io_api` has to be one of {IO_APIS} or None" ) - if remote is not None: - # The remote branch short-circuits before reaching Solver.solve(), - # which is where the empty-objective check normally fires. Replicate - # it here. This duplication becomes obsolete once OETC is folded - # into the Solver pipeline (see PyPSA/linopy#683). - if self.objective.expression.empty: - raise ValueError( - "No objective has been set on the model. Use " - "`m.add_objective(...)` first (e.g. `m.add_objective(0 * x)` " - "for a pure feasibility problem)." - ) - if isinstance(remote, OetcHandler): - solved = remote.solve_on_oetc( - self, - solver_name=solver_name, - reformulate_sos=reformulate_sos, - **solver_options, - ) - else: - solved = remote.solve_on_remote( - self, - solver_name=solver_name, - io_api=io_api, - problem_fn=problem_fn, - solution_fn=solution_fn, - log_fn=log_fn, - basis_fn=basis_fn, - warmstart_fn=warmstart_fn, - keep_files=keep_files, - sanitize_zeros=sanitize_zeros, - reformulate_sos=reformulate_sos, - **solver_options, - ) + # New standalone Oetc / SSH remote handlers are selected by passing + # their settings dataclass via ``remote=``. ``solver_name`` and + # ``**solver_options`` describe the *inner* solver to run on the + # worker. + if isinstance(remote, _REMOTE_SETTINGS_TYPES): + return self._solve_with_remote_settings( + remote, + inner_solver=solver_name, + solver_options=solver_options, + reformulate_sos=reformulate_sos, + ) - if solved.objective.value is not None: - self.objective.set_value(float(solved.objective.value)) - self.status = solved.status - self.termination_condition = solved.termination_condition - for k, v in self.variables.items(): - v.solution = solved.variables[k].solution - for k, c in self.constraints.items(): - if "dual" in solved.constraints[k]: - c.dual = solved.constraints[k].dual - return self.status, self.termination_condition + if remote is not None: + # Back-compat shim: the legacy ``remote=OetcHandler/RemoteHandler`` + # shape pre-dates the standalone Oetc/SSH classes. Route to the + # new entrypoint and warn. Slated for removal once one release of + # overlap has shipped. + return self._solve_via_legacy_remote( + remote, + solver_name=solver_name, + io_api=io_api, + problem_fn=problem_fn, + solution_fn=solution_fn, + log_fn=log_fn, + basis_fn=basis_fn, + warmstart_fn=warmstart_fn, + keep_files=keep_files, + sanitize_zeros=sanitize_zeros, + reformulate_sos=reformulate_sos, + solver_options=solver_options, + ) if len(available_solvers) == 0: raise RuntimeError("No solver installed.") @@ -1855,6 +1873,173 @@ def solve( return self.assign_result(result) + def _solve_with_remote_settings( + self, + settings: Any, + *, + inner_solver: str | None, + solver_options: dict[str, Any], + reformulate_sos: bool | Literal["auto"], + ) -> tuple[str, str]: + """ + Dispatch a remote solve from an ``OetcSettings`` / ``SshSettings`` instance. + + The new standalone remote handlers (``Oetc``, ``SSH`` in + :mod:`linopy.remote`) are *not* :class:`linopy.solvers.Solver` + subclasses — they're a parallel concept. The instance is attached + to :attr:`Model.remote` after the call so callers can introspect + e.g. the OETC job uuid. + """ + effective_inner: str | None + effective_options: dict[str, Any] + if OetcSettings is not None and isinstance(settings, OetcSettings): + from linopy.remote.oetc import Oetc + + remote_cls: Any = Oetc + # ``OetcSettings`` carries defaults for solver/solver_options + # (preserves the legacy ``OetcHandler(settings).solve_on_oetc`` + # config style). Outer ``Model.solve(solver_name, **opts)`` + # wins when given. + effective_inner = inner_solver or settings.solver + effective_options = {**settings.solver_options, **solver_options} + elif isinstance(settings, SshSettings): + from linopy.remote.ssh import SSH + + remote_cls = SSH + effective_inner = inner_solver + effective_options = solver_options + else: + raise TypeError( # pragma: no cover — checked by _REMOTE_SETTINGS_TYPES + f"Unknown remote settings type: {type(settings).__name__}" + ) + + if not effective_inner: + raise ValueError( + f"`m.solve(remote=<{type(settings).__name__}>)` requires " + "an explicit `solver_name=` for the inner solver to run " + "on the worker." + ) + + if self.objective.expression.empty: + raise ValueError( + "No objective has been set on the model. Use " + "`m.add_objective(...)` first (e.g. `m.add_objective(0 * x)` " + "for a pure feasibility problem)." + ) + + # Apply SOS reformulation before the remote handler serializes the + # model; the worker just solves a plain MILP, the lifecycle stays + # on this Model. ``sos_reformulation_context`` handles the + # apply/undo bracket, ``suppress_serialization_warning`` silences + # the ``to_netcdf`` UserWarning that fires when serializing in + # reformulated form (intentional here). + with sos_reformulation_context( + self, effective_inner, reformulate_sos + ) as applied: + with suppress_serialization_warning(active=applied): + remote_instance = remote_cls( + settings=settings, + solver_name=effective_inner, + options=effective_options, + ) + self.remote = remote_instance + self.solver = None # remote-solve clears any prior local solver + result = remote_instance.solve(self) + return self.assign_result(result) + + def _solve_via_legacy_remote( + self, + remote: Any, + *, + solver_name: str | None, + io_api: str | None, + problem_fn: str | Path | None, + solution_fn: str | Path | None, + log_fn: str | Path | None, + basis_fn: str | Path | None, + warmstart_fn: str | Path | None, + keep_files: bool, + sanitize_zeros: bool, + reformulate_sos: bool | Literal["auto"], + solver_options: dict[str, Any], + ) -> tuple[str, str]: + """ + Back-compat path for ``Model.solve(remote=)``. + + Calls ``handler.solve_on_oetc(...)`` / ``handler.solve_on_remote(...)`` + as before — preserves the behavior tests on master are asserting + against — and emits a :class:`DeprecationWarning` pointing users at + the new ``remote=`` shape. + """ + if OetcHandler is not None and isinstance(remote, OetcHandler): + warnings.warn( + "Passing an OetcHandler via `remote=` is deprecated; pass " + "the OetcSettings directly: " + "`m.solve(remote=OetcSettings(...))`. The " + "`remote=OetcHandler/RemoteHandler` shape will be removed " + "in a future release.", + DeprecationWarning, + stacklevel=3, + ) + elif isinstance(remote, RemoteHandler): + warnings.warn( + "Passing a RemoteHandler via `remote=` is deprecated; pass " + "an SshSettings via `remote=` with a `solver_name=` for " + "the inner solver (`m.solve(solver_name, remote=SshSettings" + "(...))`). The `remote=OetcHandler/RemoteHandler` shape " + "will be removed in a future release.", + DeprecationWarning, + stacklevel=3, + ) + else: + raise TypeError( + f"`remote` must be an OetcHandler, RemoteHandler, " + f"OetcSettings, or SshSettings, got {type(remote).__name__}" + ) + + # The remote handlers short-circuit before reaching Solver.solve(), + # which is where the empty-objective check normally fires. Replicate + # it here. + if self.objective.expression.empty: + raise ValueError( + "No objective has been set on the model. Use " + "`m.add_objective(...)` first (e.g. `m.add_objective(0 * x)` " + "for a pure feasibility problem)." + ) + if OetcHandler is not None and isinstance(remote, OetcHandler): + solved = remote.solve_on_oetc( + self, + solver_name=solver_name, + reformulate_sos=reformulate_sos, + **solver_options, + ) + else: + solved = remote.solve_on_remote( + self, + solver_name=solver_name, + io_api=io_api, + problem_fn=problem_fn, + solution_fn=solution_fn, + log_fn=log_fn, + basis_fn=basis_fn, + warmstart_fn=warmstart_fn, + keep_files=keep_files, + sanitize_zeros=sanitize_zeros, + reformulate_sos=reformulate_sos, + **solver_options, + ) + + if solved.objective.value is not None: + self.objective.set_value(float(solved.objective.value)) + self.status = solved.status + self.termination_condition = solved.termination_condition + for k, v in self.variables.items(): + v.solution = solved.variables[k].solution + for k, c in self.constraints.items(): + if "dual" in solved.constraints[k]: + c.dual = solved.constraints[k].dual + return self.status, self.termination_condition + def assign_result( self, result: Result, diff --git a/linopy/remote/__init__.py b/linopy/remote/__init__.py index d3d5e162..c8642ec2 100644 --- a/linopy/remote/__init__.py +++ b/linopy/remote/__init__.py @@ -8,16 +8,19 @@ - OetcHandler: Cloud-based execution via OET Cloud service """ -from linopy.remote.ssh import RemoteHandler +from linopy.remote.ssh import SSH, RemoteHandler, SshSettings try: - from linopy.remote.oetc import OetcCredentials, OetcHandler, OetcSettings + from linopy.remote.oetc import Oetc, OetcCredentials, OetcHandler, OetcSettings except ImportError: pass __all__ = [ "RemoteHandler", + "SSH", + "SshSettings", "OetcHandler", + "Oetc", "OetcSettings", "OetcCredentials", ] diff --git a/linopy/remote/_common.py b/linopy/remote/_common.py new file mode 100644 index 00000000..33a3e395 --- /dev/null +++ b/linopy/remote/_common.py @@ -0,0 +1,85 @@ +""" +Shared helpers for the standalone remote-handler classes (``Oetc``, ``SSH``). + +These handlers do not inherit from :class:`linopy.solvers.Solver` — they're +a parallel concept. The helpers here cover the two pieces of plumbing +both handlers need: validating the inner-solver string locally, and +mapping a round-tripped solved :class:`~linopy.model.Model` back onto +the source model's label space. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import numpy as np + +from linopy.constants import Solution + +if TYPE_CHECKING: + from linopy.model import Model + + +def _validate_inner_solver(inner_solver_name: str, model: Model) -> None: + """ + Check that the inner-solver string is locally known and + that the inner solver's feature set covers the model. + + Local installation is *not* required — feature flags are class-level + metadata. We only need the class to introspect ``supports(...)``. + Unknown solver names raise so typos fail fast instead of incurring a + round-trip to the worker. + """ + # Imported here to avoid a circular import at module load. + from linopy.solvers import SolverFeature, SolverName, _solver_class_for + + cls = _solver_class_for(inner_solver_name) + if cls is None: + valid = ", ".join(sorted(n.value for n in SolverName)) + raise ValueError( + f"Unknown inner solver name {inner_solver_name!r}. Pick one of: {valid}." + ) + if model.is_quadratic and not cls.supports(SolverFeature.QUADRATIC_OBJECTIVE): + raise ValueError( + f"Inner solver {inner_solver_name!r} does not support quadratic problems." + ) + if model.variables.semi_continuous and not cls.supports( + SolverFeature.SEMI_CONTINUOUS_VARIABLES + ): + raise ValueError( + f"Inner solver {inner_solver_name!r} does not support semi-continuous " + "variables. Use a solver that supports them (gurobi, cplex, highs)." + ) + if model.variables.sos and not cls.supports(SolverFeature.SOS_CONSTRAINTS): + raise ValueError( + f"Inner solver {inner_solver_name!r} does not support SOS constraints. " + "Reformulate first via `Model.solve(reformulate_sos=True)` or " + "`model.apply_sos_reformulation()`, or pick a solver that supports SOS." + ) + + +def _scatter_solution_from_solved_model( + local_model: Model, solved: Model, n_vars: int, n_cons: int +) -> Solution: + """ + Build a label-indexed :class:`~linopy.constants.Solution` from a + round-tripped solved model. + + The labels on ``solved`` match ``local_model`` because both sides + serialize/load with the same linopy version; we use the local labels + as the index. Missing slots stay ``NaN``; constraints without + ``dual`` are skipped. + """ + primal = np.full(n_vars, np.nan, dtype=float) + dual = np.full(n_cons, np.nan, dtype=float) + for name, var in local_model.variables.items(): + sol = solved.variables[name].solution + primal[var.labels.values.ravel()] = sol.values.ravel() + for name, con in local_model.constraints.items(): + if "dual" not in solved.constraints[name]: + continue + dual[con.labels.values.ravel()] = solved.constraints[name].dual.values.ravel() + + objective_value = solved.objective.value + objective = float(objective_value) if objective_value is not None else float("nan") + return Solution(primal=primal, dual=dual, objective=objective) diff --git a/linopy/remote/oetc.py b/linopy/remote/oetc.py index beef5873..3741e7b0 100644 --- a/linopy/remote/oetc.py +++ b/linopy/remote/oetc.py @@ -1,6 +1,7 @@ from __future__ import annotations import base64 +import contextlib import gzip import json import logging @@ -12,6 +13,8 @@ from enum import Enum from typing import TYPE_CHECKING, Any, Literal +from linopy.constants import Result, SolverReport, Status + if TYPE_CHECKING: from linopy.model import Model @@ -25,6 +28,8 @@ except ImportError: _oetc_deps_available = False +import warnings + import linopy from linopy.sos_reformulation import ( sos_reformulation_context, @@ -40,16 +45,45 @@ class ComputeProvider(str, Enum): @dataclass class OetcCredentials: + """ + .. deprecated:: + Pass ``email`` and ``password`` directly to :class:`OetcSettings` + instead of wrapping them in ``OetcCredentials``. This class will be + removed in a future release. + """ + email: str password: str + def __post_init__(self) -> None: + warnings.warn( + "`OetcCredentials` is deprecated; pass `email=` and `password=` " + "directly to `OetcSettings`. `OetcCredentials` will be removed " + "in a future release.", + DeprecationWarning, + stacklevel=2, + ) + @dataclass class OetcSettings: - credentials: OetcCredentials + """ + Config for the OET Cloud (OETC) remote service. + + Carries the auth/orchestrator endpoints, the worker resource sizing, + and **defaults** for the inner solver and its options. The defaults + can be overridden per call: + + >>> m.solve("gurobi", remote=OetcSettings(...), Method=2) # doctest: +SKIP + >>> m.solve(remote=OetcSettings(..., solver="gurobi")) # doctest: +SKIP + """ + name: str authentication_server_url: str orchestrator_server_url: str + email: str | None = None + password: str | None = None + credentials: OetcCredentials | None = None compute_provider: ComputeProvider = ComputeProvider.GCP solver: str = "highs" solver_options: dict[str, Any] = field(default_factory=dict) @@ -57,6 +91,19 @@ class OetcSettings: disk_space_gb: int = 10 delete_worker_on_error: bool = False + def __post_init__(self) -> None: + if self.credentials is not None: + # `credentials=` warns from its own __post_init__; carry its + # values over unless `email` / `password` were also explicitly + # given (in which case the call site wins). + if self.email is None: + self.email = self.credentials.email + if self.password is None: + self.password = self.credentials.password + self.credentials = None + if not self.email or not self.password: + raise ValueError("`OetcSettings` requires `email` and `password`.") + @classmethod def from_env( cls, @@ -100,9 +147,8 @@ def from_env( ) kwargs: dict[str, Any] = { - "credentials": OetcCredentials( - email=resolved["email"], password=resolved["password"] - ), + "email": resolved["email"], + "password": resolved["password"], "name": resolved["name"], "authentication_server_url": resolved["authentication_server_url"], "orchestrator_server_url": resolved["orchestrator_server_url"], @@ -185,12 +231,30 @@ class JobResult: class OetcHandler: - def __init__(self, settings: OetcSettings) -> None: + """ + .. deprecated:: + Use :class:`~linopy.remote.Oetc` or :meth:`Model.solve(remote=OetcSettings(...)) + ` instead. This class will be removed in a + future release. The new :class:`Oetc` class owns the public lifecycle + (``upload`` / ``submit`` / ``collect`` / ``solve``); ``OetcHandler`` + remains only for back-compat with code that holds a long-lived + handler instance. + """ + + def __init__(self, settings: OetcSettings, *, _internal: bool = False) -> None: if not _oetc_deps_available: raise ImportError( "The 'google-cloud-storage' and 'requests' packages are required " "for OetcHandler. Install them with: pip install linopy[oetc]" ) + if not _internal: + warnings.warn( + "`OetcHandler` is deprecated; use `Oetc(settings, solver_name, " + "options)` from `linopy.remote` or `Model.solve(remote=OetcSettings" + "(...))`. `OetcHandler` will be removed in a future release.", + DeprecationWarning, + stacklevel=2, + ) self.settings = settings self.jwt = self.__sign_in() self.cloud_provider_credentials = self.__get_cloud_provider_credentials() @@ -208,8 +272,8 @@ def __sign_in(self) -> AuthenticationResult: try: logger.info("OETC - Signing in...") payload = { - "email": self.settings.credentials.email, - "password": self.settings.credentials.password, + "email": self.settings.email, + "password": self.settings.password, } response = requests.post( @@ -645,11 +709,17 @@ def solve_on_oetc( """ Solve a linopy model on the OET Cloud compute app. + .. deprecated:: + Use :class:`Oetc` or + :meth:`Model.solve(remote=OetcSettings(...)) `. + Parameters ---------- model : linopy.model.Model solver_name : str, optional Override the solver from settings. + reformulate_sos : bool | "auto", optional + See :meth:`linopy.model.Model.solve`. **solver_options Override/extend solver_options from settings. @@ -657,55 +727,36 @@ def solve_on_oetc( ------- linopy.model.Model Solved model. - - Raises - ------ - Exception: If solving fails at any stage """ + # Delegates to ``Oetc.solve`` so the upload→submit→poll→download + # orchestration lives in one place. This handler is reused as the + # underlying transport so existing auth/credentials are not refetched. + effective_solver = solver_name or self.settings.solver + merged_solver_options = {**self.settings.solver_options, **solver_options} + + oetc = Oetc( + settings=self.settings, + solver_name=effective_solver, + options=merged_solver_options, + ) + oetc._handler = self try: - effective_solver = solver_name or self.settings.solver - merged_solver_options = {**self.settings.solver_options, **solver_options} - with sos_reformulation_context( model, effective_solver, reformulate_sos ) as applied: - with tempfile.NamedTemporaryFile(prefix="linopy-", suffix=".nc") as fn: - fn.file.close() - with suppress_serialization_warning(active=applied): - model.to_netcdf(fn.name) - input_file_name = self._upload_file_to_gcp(fn.name) - - job_uuid = self._submit_job_to_compute_service( - input_file_name, effective_solver, merged_solver_options - ) - job_result = self.wait_and_get_job_data(job_uuid) - - if not job_result.output_files: - raise Exception("No output files found in completed job") - - output_file_name = job_result.output_files[0] - if isinstance(output_file_name, dict) and "name" in output_file_name: - output_file_name = output_file_name["name"] - - solution_file_path = self._download_file_from_gcp(output_file_name) - - solved_model = linopy.read_netcdf(solution_file_path) - - os.remove(solution_file_path) - - logger.info( - f"OETC - Model solved successfully. Status: {solved_model.status}" - ) - if solved_model.objective.value is not None: - logger.info( - f"OETC - Objective value: {solved_model.objective.value:.2e}" - ) - - return solved_model - + with suppress_serialization_warning(active=applied): + oetc.upload(model) + oetc.submit() + oetc.collect(model) except Exception as e: raise Exception(f"Error solving model on OETC: {e}") from e + solved_model = oetc._solved_model + logger.info(f"OETC - Model solved successfully. Status: {solved_model.status}") + if solved_model.objective.value is not None: + logger.info(f"OETC - Objective value: {solved_model.objective.value:.2e}") + return solved_model + def _gzip_compress(self, source_path: str) -> str: """ Compress a file using gzip compression. @@ -786,3 +837,114 @@ def _upload_file_to_gcp(self, file_path: str) -> str: except Exception as e: raise Exception(f"Failed to upload file to GCP: {e}") + + +@dataclass +class Oetc: + """ + Remote handler that solves a linopy model on the OET Cloud (OETC) service. + + This is a standalone class — *not* a :class:`linopy.solvers.Solver` + subclass. It ships a netcdf to a cloud worker which runs the inner + solver (``solver_name``) and returns a solved netcdf. The lifecycle + splits into ``upload`` / ``submit`` / ``collect`` so future async work + can drive the seam without changing callers. + + Parameters + ---------- + settings : OetcSettings + Auth + orchestrator config (where to talk to). + solver_name : str + Inner solver to run on the worker (e.g. ``"gurobi"``, ``"highs"``). + options : dict, optional + Solver options passed through to the inner solver. + + Notes + ----- + Construction is cheap; network I/O happens at :meth:`upload` / + :meth:`submit` / :meth:`collect`. :meth:`solve` runs all three + synchronously. + """ + + settings: OetcSettings + solver_name: str + options: dict[str, Any] = field(default_factory=dict) + + _handler: OetcHandler | None = field(init=False, default=None, repr=False) + _input_file_name: str | None = field(init=False, default=None, repr=False) + _job_uuid: str | None = field(init=False, default=None, repr=False) + _solved_model: Any = field(init=False, default=None, repr=False) + _n_vars: int = field(init=False, default=0, repr=False) + _n_cons: int = field(init=False, default=0, repr=False) + + @classmethod + def is_available(cls) -> bool: + """Return True iff the OETC network deps are importable.""" + return _oetc_deps_available + + def upload(self, model: Model) -> None: + """Serialize the model to netcdf and upload it to the cloud bucket.""" + if self._handler is None: + self._handler = OetcHandler(self.settings, _internal=True) + self._n_vars = model._xCounter + self._n_cons = model._cCounter + + with tempfile.NamedTemporaryFile(prefix="linopy-", suffix=".nc") as fn: + fn.file.close() + model.to_netcdf(fn.name) + self._input_file_name = self._handler._upload_file_to_gcp(fn.name) + + def submit(self) -> str: + """Submit the prepared job to the orchestrator; return the job uuid.""" + if self._handler is None or self._input_file_name is None: + raise RuntimeError("Call `upload(model)` before `submit()`.") + self._job_uuid = self._handler._submit_job_to_compute_service( + self._input_file_name, self.solver_name, dict(self.options) + ) + return self._job_uuid + + def collect(self, model: Model) -> Result: + """Poll, download, parse, and return a label-indexed Result.""" + from linopy.remote._common import _scatter_solution_from_solved_model + + if self._handler is None or self._job_uuid is None: + raise RuntimeError( + "Call `upload(model)` and `submit()` before `collect()`." + ) + + job_result = self._handler.wait_and_get_job_data(self._job_uuid) + if not job_result.output_files: + raise Exception("No output files found in completed job") + output_file_name = job_result.output_files[0] + if isinstance(output_file_name, dict) and "name" in output_file_name: + output_file_name = output_file_name["name"] + + solution_file_path = self._handler._download_file_from_gcp(output_file_name) + try: + solved = linopy.read_netcdf(solution_file_path) + finally: + with contextlib.suppress(OSError): + os.remove(solution_file_path) + + self._solved_model = solved + + status = Status.from_termination_condition(solved.termination_condition) + solution = _scatter_solution_from_solved_model( + model, solved, self._n_vars, self._n_cons + ) + report = SolverReport(runtime=job_result.duration_in_seconds) + return Result( + status=status, + solution=solution, + solver_name=self.solver_name, + report=report, + ) + + def solve(self, model: Model) -> Result: + """Run the full upload → submit → collect pipeline synchronously.""" + from linopy.remote._common import _validate_inner_solver + + _validate_inner_solver(self.solver_name, model) + self.upload(model) + self.submit() + return self.collect(model) diff --git a/linopy/remote/ssh.py b/linopy/remote/ssh.py index 7c0a0644..e5eb8ade 100644 --- a/linopy/remote/ssh.py +++ b/linopy/remote/ssh.py @@ -8,10 +8,12 @@ import logging import os import tempfile +import warnings from collections.abc import Callable -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Literal, Union +from linopy.constants import Result from linopy.io import read_netcdf from linopy.sos_reformulation import ( sos_reformulation_context, @@ -37,11 +39,42 @@ """ +@dataclass +class SshSettings: + """ + Transport-only config for the :class:`linopy.solvers.SSH` solver. + + Inner solver name and solver options come from :meth:`Model.solve` — + ``m.solve("gurobi", remote=SshSettings(hostname=...), presolve="on")``. + + Use ``setup_commands`` to prepare the remote shell before the solve — + e.g. activate a conda environment or set ``PATH``:: + + SshSettings(hostname=..., setup_commands=["conda activate linopy-env"]) + """ + + hostname: str + port: int = 22 + username: str | None = None + password: str | None = None + python_executable: str = "python" + python_file: str = "/tmp/linopy-execution.py" + model_unsolved_file: str = "/tmp/linopy-unsolved-model.nc" + model_solved_file: str = "/tmp/linopy-solved-model.nc" + setup_commands: list[str] = field(default_factory=list) + + @dataclass class RemoteHandler: """ Handler class for solving models on a remote machine via an SSH connection. + .. deprecated:: + ``RemoteHandler`` is the legacy low-level entry point and will be + removed in a future release. Prefer + ``Model.solve("gurobi", remote=SshSettings(hostname=...))`` or + instantiate :class:`SSH` directly. + The basic idea of the handler is to provide a workflow that: 1. defines a model on the local machine @@ -133,9 +166,20 @@ class RemoteHandler: model_unsolved_file: str = "/tmp/linopy-unsolved-model.nc" model_solved_file: str = "/tmp/linopy-solved-model.nc" + _internal: bool = field(default=False, repr=False) + def __post_init__(self) -> None: assert paramiko_present, "The required paramiko package is not installed." + if not self._internal: + warnings.warn( + "`RemoteHandler` is deprecated; use `SSH(settings, solver_name, " + "options)` from `linopy.remote` or `Model.solve(remote=SshSettings" + "(hostname=...))`. `RemoteHandler` will be removed in a future release.", + DeprecationWarning, + stacklevel=2, + ) + if self.client is None: client = paramiko.SSHClient() client.load_system_host_keys() @@ -256,3 +300,82 @@ def solve_on_remote( self.sftp_client.remove(self.model_solved_file) return solved + + +@dataclass +class SSH: + """ + Remote handler that solves a linopy model on a remote machine over SSH. + + This is a standalone class — *not* a :class:`linopy.solvers.Solver` + subclass. It ships the model to a remote host and runs + ``read_netcdf(...).solve(solver_name=...)`` there, pulling the solved + netcdf back. + + Parameters + ---------- + settings : SshSettings + Connection + remote-execution paths. + solver_name : str + Inner solver to run on the remote (e.g. ``"gurobi"``). + options : dict, optional + Solver options passed through to the inner solver. + + Notes + ----- + Synchronous; unlike OETC the remote shell job is short-lived and + doesn't expose a useful submit/collect seam. + """ + + settings: SshSettings + solver_name: str + options: dict[str, Any] = field(default_factory=dict) + + _handler: "RemoteHandler | None" = field(init=False, default=None, repr=False) + _solved_model: Any = field(init=False, default=None, repr=False) + + @classmethod + def is_available(cls) -> bool: + """Return True iff paramiko is importable.""" + return paramiko_present + + def solve(self, model: "Model") -> Result: + """Ship the model, run the inner solver on the remote, return a Result.""" + from linopy.constants import Status + from linopy.remote._common import ( + _scatter_solution_from_solved_model, + _validate_inner_solver, + ) + + _validate_inner_solver(self.solver_name, model) + + if self._handler is None: + self._handler = RemoteHandler( + hostname=self.settings.hostname, + port=self.settings.port, + username=self.settings.username, + password=self.settings.password, + python_executable=self.settings.python_executable, + python_file=self.settings.python_file, + model_unsolved_file=self.settings.model_unsolved_file, + model_solved_file=self.settings.model_solved_file, + _internal=True, + ) + for cmd in self.settings.setup_commands: + self._handler.execute(cmd) + + solve_kwargs: dict[str, Any] = {"solver_name": self.solver_name} + if self.options: + solve_kwargs.update(self.options) + solved = self._handler.solve_on_remote(model, **solve_kwargs) + self._solved_model = solved + + status = Status.from_termination_condition(solved.termination_condition) + solution = _scatter_solution_from_solved_model( + model, solved, model._xCounter, model._cCounter + ) + return Result( + status=status, + solution=solution, + solver_name=self.solver_name, + ) diff --git a/linopy/solvers.py b/linopy/solvers.py index 44db983f..b9207869 100644 --- a/linopy/solvers.py +++ b/linopy/solvers.py @@ -497,11 +497,22 @@ def from_model( model: Model, io_api: str | None = None, options: dict[str, Any] | None = None, - **build_kwargs: Any, + **kwargs: Any, ) -> Solver: - """Instantiate and build the solver against ``model``.""" - instance = cls(model=model, io_api=io_api, options=options or {}) - instance._build(**build_kwargs) + """ + Instantiate and build the solver against ``model``. + + Any ``kwargs`` whose name matches an ``init=True`` dataclass field on + the subclass (e.g. ``settings`` on :class:`Oetc` / :class:`SSH`) are + forwarded to the constructor; the rest go to ``_build`` as + ``build_kwargs``. + """ + from dataclasses import fields + + field_names = {f.name for f in fields(cls) if f.init} + ctor_kw = {k: kwargs.pop(k) for k in list(kwargs) if k in field_names} + instance = cls(model=model, io_api=io_api, options=options or {}, **ctor_kw) + instance._build(**kwargs) return instance def _build(self, **build_kwargs: Any) -> None: diff --git a/pyproject.toml b/pyproject.toml index 67297677..ac916eb0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ oetc = [ "google-cloud-storage", "requests", ] -remote = [ +ssh = [ "paramiko", ] docs = [ diff --git a/test/remote/test_oetc.py b/test/remote/test_oetc.py index 7b2d75f2..dd54b07d 100644 --- a/test/remote/test_oetc.py +++ b/test/remote/test_oetc.py @@ -1530,9 +1530,14 @@ def test_solve_on_oetc_file_upload( """Test solve_on_oetc method complete workflow""" # Setup mock_model = Mock() + mock_model._xCounter = 0 + mock_model._cCounter = 0 + mock_model.variables.items.return_value = [] + mock_model.constraints.items.return_value = [] mock_solved_model = Mock() mock_solved_model.status = "optimal" mock_solved_model.objective.value = 42.0 + mock_solved_model.termination_condition = "optimal" mock_temp_file = Mock() mock_temp_file.name = "/tmp/linopy-abc123.nc" @@ -1655,9 +1660,14 @@ def test_solve_on_oetc_with_job_submission( """Test solve_on_oetc method including job submission, waiting, and download""" # Setup mock_model = Mock() + mock_model._xCounter = 0 + mock_model._cCounter = 0 + mock_model.variables.items.return_value = [] + mock_model.constraints.items.return_value = [] mock_solved_model = Mock() mock_solved_model.status = "optimal" mock_solved_model.objective.value = 100.5 + mock_solved_model.termination_condition = "optimal" mock_temp_file = Mock() mock_temp_file.name = "/tmp/linopy-abc123.nc" diff --git a/test/remote/test_remotes.py b/test/remote/test_remotes.py new file mode 100644 index 00000000..b8ba5d19 --- /dev/null +++ b/test/remote/test_remotes.py @@ -0,0 +1,373 @@ +""" +Tests for the standalone remote classes (``Oetc`` / ``SSH``) and the +``Model.solve(remote=)`` entry point. + +The deprecated ``OetcHandler`` / ``RemoteHandler`` are covered by +``test_oetc.py`` and ``test_ssh.py`` separately; this file focuses on +the *new* public surface and its deprecation warnings. +""" + +from __future__ import annotations + +import warnings +from typing import Any +from unittest.mock import MagicMock, patch + +import numpy as np +import pandas as pd +import pytest + +from linopy import Model +from linopy.constants import ( + Result, + Solution, + SolverReport, + Status, +) +from linopy.remote import ( + Oetc, + OetcCredentials, + OetcHandler, + OetcSettings, + RemoteHandler, + SshSettings, +) + +pytest.importorskip("paramiko") +from linopy.remote.ssh import SSH # noqa: E402 + +# --------------------------------------------------------------------------- +# Helpers + + +def _build_model() -> Model: + m = Model() + idx = pd.Index([0, 1, 2], name="i") + x = m.add_variables(lower=0, upper=1, coords=[idx], name="x") + m.add_constraints(x >= 0, name="c") + m.add_objective(1.0 * x.sum()) + return m + + +def _settings_oetc() -> OetcSettings: + return OetcSettings( + email="a@b.com", + password="pw", + name="test-job", + authentication_server_url="https://auth", + orchestrator_server_url="https://orch", + ) + + +def _settings_ssh() -> SshSettings: + return SshSettings(hostname="example.org", username="me") + + +def _fake_oetc_handler() -> MagicMock: + """A MagicMock(spec=OetcHandler) with the methods Oetc.upload/submit/collect call.""" + h = MagicMock(spec=OetcHandler) + h._upload_file_to_gcp = MagicMock(return_value="model.nc.gz") + h._submit_job_to_compute_service = MagicMock(return_value="job-uuid") + job_result = MagicMock() + job_result.output_files = [{"name": "result.nc.gz"}] + job_result.duration_in_seconds = 42 + h.wait_and_get_job_data = MagicMock(return_value=job_result) + h._download_file_from_gcp = MagicMock(return_value="/tmp/fake-result.nc") + return h + + +def _solved_model_like(m: Model) -> Model: + """Build a Model with the same labels as ``m`` plus dummy solution data.""" + solved = Model() + for name, var in m.variables.items(): + solved_var = solved.add_variables( + lower=var.lower, upper=var.upper, coords=var.coords, name=name + ) + solved_var.solution = solved_var.lower * 0 # zeros, real DataArray + for name, con in m.constraints.items(): + solved.add_constraints(con.lhs >= con.rhs, name=name) + solved.add_objective(m.objective.expression) + solved.objective._value = 0.0 + solved.termination_condition = "optimal" + solved.status = "ok" + return solved + + +# --------------------------------------------------------------------------- +# Oetc class + + +class TestOetcClass: + def test_solve_runs_upload_submit_collect( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + m = _build_model() + oetc = Oetc(settings=_settings_oetc(), solver_name="highs") + oetc._handler = _fake_oetc_handler() # bypass auth + + monkeypatch.setattr( + "linopy.remote.oetc.linopy.read_netcdf", + lambda path: _solved_model_like(m), + ) + + result = oetc.solve(m) + + assert isinstance(result, Result) + assert result.solver_name == "highs" + oetc._handler._upload_file_to_gcp.assert_called_once() + oetc._handler._submit_job_to_compute_service.assert_called_once() + oetc._handler.wait_and_get_job_data.assert_called_once_with("job-uuid") + oetc._handler._download_file_from_gcp.assert_called_once_with("result.nc.gz") + + def test_validates_unknown_solver_name(self) -> None: + m = _build_model() + oetc = Oetc(settings=_settings_oetc(), solver_name="not-a-solver") + oetc._handler = _fake_oetc_handler() + with pytest.raises(ValueError, match="Unknown inner solver"): + oetc.solve(m) + + def test_upload_submit_collect_separable( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The three-step lifecycle can be driven manually, e.g. for async work.""" + m = _build_model() + oetc = Oetc(settings=_settings_oetc(), solver_name="highs") + oetc._handler = _fake_oetc_handler() + monkeypatch.setattr( + "linopy.remote.oetc.linopy.read_netcdf", + lambda path: _solved_model_like(m), + ) + + oetc.upload(m) + assert oetc._input_file_name == "model.nc.gz" + assert oetc._handler._upload_file_to_gcp.call_count == 1 + + job_id = oetc.submit() + assert job_id == "job-uuid" + assert oetc._handler._submit_job_to_compute_service.call_count == 1 + + result = oetc.collect(m) + assert isinstance(result, Result) + assert oetc._handler.wait_and_get_job_data.call_count == 1 + + def test_submit_before_upload_raises(self) -> None: + oetc = Oetc(settings=_settings_oetc(), solver_name="highs") + oetc._handler = _fake_oetc_handler() + with pytest.raises(RuntimeError, match="upload"): + oetc.submit() + + def test_collect_before_submit_raises(self) -> None: + m = _build_model() + oetc = Oetc(settings=_settings_oetc(), solver_name="highs") + oetc._handler = _fake_oetc_handler() + with pytest.raises(RuntimeError, match="upload.*submit"): + oetc.collect(m) + + +# --------------------------------------------------------------------------- +# SSH class + + +class TestSSHClass: + def test_solve_runs_setup_commands_then_delegates(self) -> None: + m = _build_model() + ssh = SSH( + settings=SshSettings( + hostname="example.org", + setup_commands=["conda activate linopy-env", "export FOO=bar"], + ), + solver_name="highs", + ) + fake_handler = MagicMock(spec=RemoteHandler) + fake_handler.execute = MagicMock() + fake_handler.solve_on_remote = MagicMock(return_value=_solved_model_like(m)) + ssh._handler = fake_handler + + result = ssh.solve(m) + + assert isinstance(result, Result) + # solve_on_remote is the public surface from the deprecated handler + fake_handler.solve_on_remote.assert_called_once() + # setup_commands run only on first handler construction; here _handler + # was injected, so they shouldn't run automatically: + fake_handler.execute.assert_not_called() + + def test_setup_commands_run_when_handler_is_built_internally( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """First .solve() with a fresh SSH builds a RemoteHandler and runs setup.""" + m = _build_model() + ssh = SSH( + settings=SshSettings( + hostname="example.org", + setup_commands=["conda activate linopy-env"], + ), + solver_name="highs", + ) + + built: list[Any] = [] + + class FakeRemoteHandler: + def __init__(self, **kwargs: Any) -> None: + self.kwargs = kwargs + self.execute = MagicMock() + self.solve_on_remote = MagicMock(return_value=_solved_model_like(m)) + built.append(self) + + monkeypatch.setattr("linopy.remote.ssh.RemoteHandler", FakeRemoteHandler) + ssh.solve(m) + + assert len(built) == 1 + built[0].execute.assert_called_once_with("conda activate linopy-env") + assert built[0].kwargs.get("_internal") is True + + def test_validates_unknown_solver_name(self) -> None: + m = _build_model() + ssh = SSH(settings=_settings_ssh(), solver_name="not-a-solver") + ssh._handler = MagicMock(spec=RemoteHandler) + with pytest.raises(ValueError, match="Unknown inner solver"): + ssh.solve(m) + + +# --------------------------------------------------------------------------- +# Model.solve(remote=) end-to-end + + +class TestModelSolveRemote: + def test_oetc_settings_dispatches_to_oetc( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + m = _build_model() + captured: dict[str, Any] = {} + + def fake_solve(self: Oetc, model: Model) -> Result: + captured["solver_name"] = self.solver_name + captured["options"] = self.options + captured["instance"] = self + return Result( + status=Status.from_termination_condition("optimal"), + solution=Solution( + primal=np.zeros(model._xCounter, dtype=float), + dual=np.full(model._cCounter, np.nan, dtype=float), + objective=0.0, + ), + solver_name=self.solver_name, + report=SolverReport(runtime=1.0), + ) + + monkeypatch.setattr(Oetc, "solve", fake_solve) + + m.solve("gurobi", remote=_settings_oetc(), Method=2) + + assert captured["solver_name"] == "gurobi" + assert captured["options"] == {"Method": 2} + assert m.remote is captured["instance"] + assert m.solver is None # remote-solve clears any prior local solver + + def test_ssh_settings_dispatches_to_ssh( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + m = _build_model() + captured: dict[str, Any] = {} + + def fake_solve(self: SSH, model: Model) -> Result: + captured["solver_name"] = self.solver_name + captured["options"] = self.options + captured["instance"] = self + return Result( + status=Status.from_termination_condition("optimal"), + solution=Solution( + primal=np.zeros(model._xCounter, dtype=float), + dual=np.full(model._cCounter, np.nan, dtype=float), + objective=0.0, + ), + solver_name=self.solver_name, + ) + + monkeypatch.setattr(SSH, "solve", fake_solve) + + m.solve("highs", remote=_settings_ssh(), presolve="on") + + assert captured["solver_name"] == "highs" + assert captured["options"] == {"presolve": "on"} + assert m.remote is captured["instance"] + + +# --------------------------------------------------------------------------- +# Deprecation warnings + + +class TestDeprecations: + def test_oetc_credentials_construction_warns(self) -> None: + with pytest.warns(DeprecationWarning, match="OetcCredentials"): + OetcCredentials(email="a@b.com", password="pw") + + def test_oetc_settings_credentials_kwarg_carries_values_through(self) -> None: + # Constructing OetcCredentials warns (its own __post_init__). + with pytest.warns(DeprecationWarning, match="OetcCredentials"): + creds = OetcCredentials(email="a@b.com", password="pw") + + s = OetcSettings( + credentials=creds, + name="n", + authentication_server_url="https://a", + orchestrator_server_url="https://o", + ) + assert s.email == "a@b.com" + assert s.password == "pw" + # `credentials` is consumed and cleared. + assert s.credentials is None + + def test_oetc_settings_requires_email_and_password(self) -> None: + with pytest.raises(ValueError, match="email.*password"): + OetcSettings( + name="n", + authentication_server_url="https://a", + orchestrator_server_url="https://o", + ) + + def test_oetc_handler_construction_warns(self) -> None: + with ( + patch.object(OetcHandler, "_OetcHandler__sign_in"), + patch.object(OetcHandler, "_OetcHandler__get_cloud_provider_credentials"), + ): + with pytest.warns(DeprecationWarning, match="OetcHandler"): + OetcHandler(_settings_oetc()) + + def test_oetc_handler_internal_construction_silent(self) -> None: + with ( + patch.object(OetcHandler, "_OetcHandler__sign_in"), + patch.object(OetcHandler, "_OetcHandler__get_cloud_provider_credentials"), + ): + with warnings.catch_warnings(): + warnings.simplefilter("error") + OetcHandler(_settings_oetc(), _internal=True) + + def test_remote_handler_construction_warns( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + fake_client = MagicMock() + fake_client.invoke_shell.return_value.makefile.return_value = MagicMock() + fake_client.open_sftp.return_value = MagicMock() + + with pytest.warns(DeprecationWarning, match="RemoteHandler"): + RemoteHandler(hostname="x", client=fake_client) + + def test_remote_handler_internal_construction_silent(self) -> None: + fake_client = MagicMock() + fake_client.invoke_shell.return_value.makefile.return_value = MagicMock() + fake_client.open_sftp.return_value = MagicMock() + + with warnings.catch_warnings(): + warnings.simplefilter("error") + RemoteHandler(hostname="x", client=fake_client, _internal=True) + + def test_model_solve_remote_handler_warns( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + m = _build_model() + handler = MagicMock(spec=OetcHandler) + handler.settings = _settings_oetc() + handler.solve_on_oetc = MagicMock(return_value=_solved_model_like(m)) + with pytest.warns(DeprecationWarning, match="OetcHandler.*remote="): + m.solve(solver_name="highs", remote=handler) diff --git a/test/test_oetc_settings.py b/test/test_oetc_settings.py index 12deeb66..3206eaff 100644 --- a/test/test_oetc_settings.py +++ b/test/test_oetc_settings.py @@ -7,7 +7,6 @@ from linopy.remote.oetc import ( ComputeProvider, - OetcCredentials, OetcHandler, OetcSettings, ) @@ -48,8 +47,8 @@ def test_from_env_all_set(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("OETC_DELETE_WORKER_ON_ERROR", "true") s = OetcSettings.from_env() - assert s.credentials.email == "test@example.com" - assert s.credentials.password == "secret" + assert s.email == "test@example.com" + assert s.password == "secret" assert s.name == "test-job" assert s.cpu_cores == 8 assert s.disk_space_gb == 20 @@ -62,7 +61,7 @@ def test_from_env_kwargs_override(monkeypatch: pytest.MonkeyPatch) -> None: _set_required_env(monkeypatch) s = OetcSettings.from_env(email="override@example.com") - assert s.credentials.email == "override@example.com" + assert s.email == "override@example.com" def test_from_env_missing_required(monkeypatch: pytest.MonkeyPatch) -> None: @@ -93,7 +92,7 @@ def test_from_env_partial_kwargs(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("OETC_ORCHESTRATOR_URL", "https://orch.example.com") s = OetcSettings.from_env(email="a@b.com", password="pw") - assert s.credentials.email == "a@b.com" + assert s.email == "a@b.com" assert s.name == "env-name" @@ -169,7 +168,8 @@ def _make_handler(settings: OetcSettings) -> OetcHandler: def _default_settings(**overrides: Any) -> OetcSettings: defaults: dict[str, Any] = dict( - credentials=OetcCredentials(email="a@b.com", password="pw"), + email="a@b.com", + password="pw", name="test", authentication_server_url="https://auth", orchestrator_server_url="https://orch", diff --git a/test/test_sos_reformulation.py b/test/test_sos_reformulation.py index 51ec1770..4a6264d3 100644 --- a/test/test_sos_reformulation.py +++ b/test/test_sos_reformulation.py @@ -6,16 +6,13 @@ import warnings from collections.abc import Callable from pathlib import Path -from typing import Literal, cast import numpy as np import pandas as pd import pytest -import xarray as xr from linopy import Model, Variable, available_solvers from linopy.constants import SOS_TYPE_ATTR -from linopy.remote import RemoteHandler from linopy.sos_reformulation import ( compute_big_m_values, reformulate_sos1, @@ -1139,64 +1136,57 @@ def _sos_model() -> Model: m.add_objective(x * np.array([1.0, 2.0, 3.0]), sense="max") return m - def _fake_handler( - self, observed: dict[str, object], tmp_path: Path - ) -> RemoteHandler: + @staticmethod + def _patch_ssh_solve( + monkeypatch: pytest.MonkeyPatch, + observed: dict[str, object], + tmp_path: Path, + ) -> None: """ - Non-OetcHandler stand-in with the SSH-shaped `solve_on_remote`. - - Records whether the model arrives in reformulated form, then runs - `model.to_netcdf(...)` and `read_netcdf(...)` (naturally — no - warning recording here, so we can observe at the call-site whether - Model.solve's suppression worked). + Replace ``linopy.remote.ssh.SSH.solve`` with a stub that records + whether the model arrives in reformulated form, exercises the + ``to_netcdf`` warning path, and returns a synthetic + :class:`Result` so ``Model.assign_result`` is exercised end to end. """ - from linopy.io import read_netcdf - from linopy.sos_reformulation import ( - sos_reformulation_context, - suppress_serialization_warning, - ) + from linopy.constants import Result, Solution, Status + from linopy.remote.ssh import SSH + + def fake_solve(self: SSH, model: Model) -> Result: + observed["state_active"] = model._sos_reformulation_state is not None + observed["solver_name_arg"] = self.solver_name + model.to_netcdf(tmp_path / "sent.nc") # triggers any to_netcdf warning + n_vars = model._xCounter + n_cons = model._cCounter + return Result( + status=Status.from_termination_condition("optimal"), + solution=Solution( + primal=np.zeros(n_vars, dtype=float), + dual=np.full(n_cons, np.nan, dtype=float), + objective=0.0, + ), + solver_name=self.solver_name, + ) + + monkeypatch.setattr(SSH, "solve", fake_solve) + + def test_remote_brackets_and_suppresses_warning( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from linopy.remote.ssh import SshSettings - class _Handler: - def solve_on_remote( - _self, - model: Model, - *, - reformulate_sos: bool | Literal["auto"] = False, - **kwargs: object, - ) -> Model: - solver_name = kwargs.get("solver_name") - assert solver_name is None or isinstance(solver_name, str) - with sos_reformulation_context( - model, solver_name, reformulate_sos - ) as applied: - observed["state_active"] = ( - model._sos_reformulation_state is not None - ) - observed["solver_name_arg"] = solver_name - with suppress_serialization_warning(active=applied): - model.to_netcdf(tmp_path / "sent.nc") - solved = read_netcdf(tmp_path / "sent.nc") - for _name, var in solved.variables.items(): - arr = np.zeros(var.labels.shape, dtype=float) - var.solution = xr.DataArray(arr, dims=var.labels.dims) - solved.objective.set_value(0.0) - solved.status = "ok" - solved.termination_condition = "optimal" - return solved - - return cast(RemoteHandler, _Handler()) - - def test_remote_brackets_and_suppresses_warning(self, tmp_path: Path) -> None: m = self._sos_model() observed: dict[str, object] = {} - handler = self._fake_handler(observed, tmp_path) + self._patch_ssh_solve(monkeypatch, observed, tmp_path) with warnings.catch_warnings(record=True) as captured: warnings.simplefilter("always") - m.solve(solver_name="highs", remote=handler, reformulate_sos=True) + m.solve( + solver_name="highs", + remote=SshSettings(hostname="ignored"), + reformulate_sos=True, + ) - # Reformulation was active when the handler ran (apply happened - # before the remote dispatch). + # Reformulation was active when the transport ran. assert observed["state_active"] is True assert observed["solver_name_arg"] == "highs" @@ -1209,26 +1199,38 @@ def test_remote_brackets_and_suppresses_warning(self, tmp_path: Path) -> None: assert "_sos_reform_x_y" not in m.variables def test_remote_skips_bracket_when_reformulate_sos_false( - self, tmp_path: Path + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: + from linopy.remote.ssh import SshSettings + m = self._sos_model() observed: dict[str, object] = {} - handler = self._fake_handler(observed, tmp_path) + self._patch_ssh_solve(monkeypatch, observed, tmp_path) with warnings.catch_warnings(record=True) as captured: warnings.simplefilter("always") - m.solve(solver_name="highs", remote=handler, reformulate_sos=False) + m.solve( + solver_name="highs", + remote=SshSettings(hostname="ignored"), + reformulate_sos=False, + ) # No reformulation happened — model still has the original SOS var - # when the handler sees it, and to_netcdf never warns. + # when the transport sees it, and to_netcdf never warns. assert observed["state_active"] is False assert not any("active SOS reformulation" in str(w.message) for w in captured) assert m._sos_reformulation_state is None - def test_remote_auto_requires_solver_name_with_sos(self, tmp_path: Path) -> None: + def test_remote_auto_requires_solver_name_with_sos( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from linopy.remote.ssh import SshSettings + m = self._sos_model() observed: dict[str, object] = {} - handler = self._fake_handler(observed, tmp_path) + self._patch_ssh_solve(monkeypatch, observed, tmp_path) - with pytest.raises(ValueError, match="requires an explicit `solver_name`"): - m.solve(remote=handler, reformulate_sos="auto") + # Without an explicit solver_name, the transport dispatch refuses + # to run because there's no inner solver to ship. + with pytest.raises(ValueError, match="explicit `solver_name=`"): + m.solve(remote=SshSettings(hostname="ignored"), reformulate_sos="auto")