From 8bf7b4bd7507c251d8163ac8cf7623732e9089db Mon Sep 17 00:00:00 2001 From: Fabian Date: Mon, 18 May 2026 10:30:41 +0200 Subject: [PATCH 01/18] feat(xpress): add direct API support via loadproblem Adds io_api='direct' for Xpress: loads the model through the native loadproblem array API instead of writing an LP/MPS file. Includes a model.to_xpress() helper mirroring to_gurobipy/to_highspy/to_mosek, SOS attachment, and feature flags (DIRECT_API, SOS_CONSTRAINTS). Refactors _run_file to share a _solve helper with the new _run_direct. --- doc/release_notes.rst | 3 +- linopy/io.py | 15 +++ linopy/model.py | 3 + linopy/solvers.py | 277 ++++++++++++++++++++++++++++++++++++------ test/test_io.py | 16 +++ 5 files changed, 279 insertions(+), 35 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 9601e0a9..0293ac94 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -43,6 +43,7 @@ Most users should keep calling ``model.solve(...)``. If you want more control, y **Performance** * ~10× faster direct solver communication (``io_api="direct"``), thanks to the new CSR-based matrix construction. Conversion helpers like ``to_highspy`` benefit too. +* Xpress now supports ``io_api="direct"``: the linopy model is loaded via the native ``loadproblem`` array API instead of being serialised through an LP/MPS file, with SOS constraints attached in-place. Adds ``model.to_xpress()`` matching the existing ``to_gurobipy`` / ``to_highspy`` / ``to_mosek`` helpers. * Writing the solution back to the model after solving is faster: it no longer rebuilds the constraint matrix, and now uses positional (rather than label-based) indexing — roughly 2× faster overall. **Deprecations** @@ -57,7 +58,7 @@ Most users should keep calling ``model.solve(...)``. If you want more control, y **Internal** -* Each ``Solver`` subclass now overrides at most three hooks: ``_build_direct`` (build the native model), ``_run_direct`` (run it), and ``_run_file`` (run the solver on an LP/MPS file). File-only solvers (CBC, GLPK, CPLEX, SCIP, Xpress, Knitro, COPT, MindOpt) only override ``_run_file``. +* Each ``Solver`` subclass now overrides at most three hooks: ``_build_direct`` (build the native model), ``_run_direct`` (run it), and ``_run_file`` (run the solver on an LP/MPS file). File-only solvers (CBC, GLPK, CPLEX, SCIP, Knitro, COPT, MindOpt) only override ``_run_file``. * New ``ConstraintLabelIndex`` cached on ``Model.constraints`` (mirrors the existing ``Variables.label_index``); ``ConstraintBase`` gains ``active_labels()`` and a ``range`` property; ``CSRConstraint`` exposes ``coords``. * ``linopy.common`` gains ``values_to_lookup_array``; the legacy pandas-based helpers ``series_to_lookup_array`` and ``lookup_vals`` are removed. ``model.to_gurobipy()`` / ``model.to_highspy()`` / ``to_cupdlpx(model)`` (and similar) all return the underlying solver model as before; internally they now go through ``Solver.from_model(model, io_api="direct")``. No user-visible change. diff --git a/linopy/io.py b/linopy/io.py index 36d7abb3..54adee87 100644 --- a/linopy/io.py +++ b/linopy/io.py @@ -685,6 +685,21 @@ def to_highspy( return solver.solver_model +def to_xpress( + m: Model, + explicit_coordinate_names: bool = False, + set_names: bool = True, +) -> Any: + """Build the xpress.problem instance for `m`.""" + solver = solvers.Xpress.from_model( + m, + io_api="direct", + explicit_coordinate_names=explicit_coordinate_names, + set_names=set_names, + ) + return solver.solver_model + + def to_cupdlpx(m: Model) -> cupdlpxModel: """Build the cupdlpx.Model for `m`.""" solver = solvers.cuPDLPx.from_model(m, io_api="direct") diff --git a/linopy/model.py b/linopy/model.py index 1e4ae637..38b86c76 100644 --- a/linopy/model.py +++ b/linopy/model.py @@ -71,6 +71,7 @@ to_highspy, to_mosek, to_netcdf, + to_xpress, ) from linopy.matrices import MatrixAccessor from linopy.objective import Objective @@ -2113,4 +2114,6 @@ def reset_solution(self) -> None: to_cupdlpx = to_cupdlpx + to_xpress = to_xpress + to_block_files = to_block_files diff --git a/linopy/solvers.py b/linopy/solvers.py index 548db835..8370ed50 100644 --- a/linopy/solvers.py +++ b/linopy/solvers.py @@ -2013,10 +2013,12 @@ class Xpress(Solver[None]): { SolverFeature.INTEGER_VARIABLES, SolverFeature.QUADRATIC_OBJECTIVE, + SolverFeature.DIRECT_API, SolverFeature.LP_FILE_NAMES, SolverFeature.READ_MODEL_FROM_FILE, SolverFeature.SOLUTION_FILE_NOT_NEEDED, SolverFeature.IIS_COMPUTATION, + SolverFeature.SOS_CONSTRAINTS, } ) @@ -2025,12 +2027,190 @@ class Xpress(Solver[None]): def is_available(cls) -> bool: return _has_module("xpress") + def _build_direct( + self, + explicit_coordinate_names: bool = False, + set_names: bool = True, + **kwargs: Any, + ) -> None: + model = self.model + assert model is not None + problem = self._build_solver_model( + model, + explicit_coordinate_names=explicit_coordinate_names, + set_names=set_names, + ) + self.solver_model = problem + self.io_api = "direct" + self.sense = model.sense + self._cache_model_labels(model) + + @staticmethod + def _build_solver_model( + model: Model, + explicit_coordinate_names: bool = False, + set_names: bool = True, + ) -> xpress.problem: + """Build an ``xpress.problem`` that mirrors the linopy ``model`` via ``loadproblem``.""" + model.constraints.sanitize_missings() + problem = xpress.problem() + + M = model.matrices + A = M.A + Q = M.Q + + if A is not None and A.nnz: + if A.format != "csc": + A = A.tocsc() + start = A.indptr.astype(np.int64, copy=False) + rowind = A.indices.astype(np.int64, copy=False) + rowcoef = A.data.astype(float, copy=False) + else: + start = np.zeros(len(M.vlabels) + 1, dtype=np.int64) + rowind = np.empty(0, dtype=np.int64) + rowcoef = np.empty(0, dtype=float) + + lb = np.asarray(M.lb, dtype=float) + ub = np.asarray(M.ub, dtype=float) + np.place(lb, np.isneginf(lb), -xpress.infinity) + np.place(ub, np.isposinf(ub), xpress.infinity) + + if len(M.clabels): + sense = M.sense + rowtype = np.full(sense.shape, "E", dtype="U1") + rowtype[sense == "<"] = "L" + rowtype[sense == ">"] = "G" + rhs = np.asarray(M.b, dtype=float) + else: + rowtype = np.empty(0, dtype="U1") + rhs = np.empty(0, dtype=float) + + objqcol1: np.ndarray | None + objqcol2: np.ndarray | None + objqcoef: np.ndarray | None + if Q is not None and Q.nnz: + Qt = Q if Q.format == "coo" else triu(Q, format="coo") # codespell:ignore coo + mask = Qt.row <= Qt.col + objqcol1 = Qt.row[mask].astype(np.int64, copy=False) + objqcol2 = Qt.col[mask].astype(np.int64, copy=False) + objqcoef = Qt.data[mask].astype(float, copy=False) + else: + objqcol1 = None + objqcol2 = None + objqcoef = None + + vtypes = M.vtypes + integer_mask = (vtypes == "B") | (vtypes == "I") + if integer_mask.any(): + entind = np.flatnonzero(integer_mask).astype(np.int64, copy=False) + coltype = vtypes[entind] + else: + entind = None + coltype = None + + problem.loadproblem( + probname="", + rowtype=rowtype, + rhs=rhs, + rng=None, + objcoef=np.asarray(M.c, dtype=float), + start=start, + collen=None, + rowind=rowind, + rowcoef=rowcoef, + lb=lb, + ub=ub, + objqcol1=objqcol1, + objqcol2=objqcol2, + objqcoef=objqcoef, + qrowind=None, + nrowqcoefs=None, + rowqcol1=None, + rowqcol2=None, + rowqcoef=None, + coltype=coltype, + entind=entind, + limit=None, + settype=None, + setstart=None, + setind=None, + refval=None, + ) + + if model.objective.sense == "max": + try: + problem.chgObjSense(xpress.ObjSense.MAXIMIZE) + except AttributeError: + problem.chgobjsense(xpress.maximize) + + if set_names: + print_variable, print_constraint = linopy.io.get_printers_scalar( + model, explicit_coordinate_names=explicit_coordinate_names + ) + vnames = print_variable(M.vlabels) + if vnames: + try: + problem.addNames(xpress_Namespaces.COLUMN, vnames, 0, len(vnames) - 1) + except AttributeError: + problem.addnames(xpress_Namespaces.COLUMN, vnames, 0, len(vnames) - 1) + cnames = print_constraint(M.clabels) + if cnames: + try: + problem.addNames(xpress_Namespaces.ROW, cnames, 0, len(cnames) - 1) + except AttributeError: + problem.addnames(xpress_Namespaces.ROW, cnames, 0, len(cnames) - 1) + + if model.variables.sos: + for var_name in model.variables.sos: + var = model.variables.sos[var_name] + sos_type: int = var.attrs[SOS_TYPE_ATTR] # type: ignore[assignment] + sos_dim: str = var.attrs[SOS_DIM_ATTR] # type: ignore[assignment] + + def add_sos(s: xr.DataArray, sos_type: int, sos_dim: str) -> None: + s = s.squeeze() + labels = s.values.flatten() + mask = labels != -1 + if not mask.any(): + return + indices = labels[mask].tolist() + weights = s.coords[sos_dim].values[mask].tolist() + problem.addSOS(indices, weights, type=sos_type) + + others = [dim for dim in var.labels.dims if dim != sos_dim] + if not others: + add_sos(var.labels, sos_type, sos_dim) + else: + stacked = var.labels.stack(_sos_group=others) + for _, s in stacked.groupby("_sos_group"): + add_sos(s.unstack("_sos_group"), sos_type, sos_dim) + + return problem + @classmethod def runtime_features(cls) -> frozenset[SolverFeature]: if _installed_version_in("xpress", ">=9.8.0"): return frozenset({SolverFeature.GPU_ACCELERATION}) return frozenset() + def _run_direct( + self, + solution_fn: Path | None = None, + log_fn: Path | None = None, + warmstart_fn: Path | None = None, + basis_fn: Path | None = None, + env: None = None, + **kw: Any, + ) -> Result: + return self._solve( + self.solver_model, + solution_fn=solution_fn, + log_fn=log_fn, + warmstart_fn=warmstart_fn, + basis_fn=basis_fn, + io_api=self.io_api, + sense=self.sense, + ) + def _run_file( self, solution_fn: Path | None = None, @@ -2042,6 +2222,37 @@ def _run_file( ) -> Result: problem_fn = self._problem_fn assert problem_fn is not None + io_api = read_io_api_from_problem_file(problem_fn) + sense = read_sense_from_problem_file(problem_fn) + + m = xpress.problem() + try: + m.readProb(path_to_string(problem_fn)) + except AttributeError: + m.read(path_to_string(problem_fn)) + + return self._solve( + m, + solution_fn=solution_fn, + log_fn=log_fn, + warmstart_fn=warmstart_fn, + basis_fn=basis_fn, + io_api=io_api, + sense=sense, + from_file=True, + ) + + def _solve( + self, + m: xpress.problem, + solution_fn: Path | None, + log_fn: Path | None, + warmstart_fn: Path | None, + basis_fn: Path | None, + io_api: str | None, + sense: str | None, + from_file: bool = False, + ) -> Result: CONDITION_MAP = { xpress.SolStatus.NOTFOUND: "unknown", xpress.SolStatus.OPTIMAL: "optimal", @@ -2050,55 +2261,43 @@ def _run_file( xpress.SolStatus.UNBOUNDED: "unbounded", } - io_api = read_io_api_from_problem_file(problem_fn) - sense = read_sense_from_problem_file(problem_fn) - - m = xpress.problem() - - try: # Try new API first - m.readProb(path_to_string(problem_fn)) - except AttributeError: # Fallback to old API - m.read(path_to_string(problem_fn)) - - # Set solver options - new API uses setControl per option, old API accepts dict if self.solver_options is not None: m.setControl(self.solver_options) if log_fn is not None: - try: # Try new API first + try: m.setLogFile(path_to_string(log_fn)) - except AttributeError: # Fallback to old API + except AttributeError: m.setlogfile(path_to_string(log_fn)) if warmstart_fn is not None: - try: # Try new API first + try: m.readBasis(path_to_string(warmstart_fn)) - except AttributeError: # Fallback to old API + except AttributeError: m.readbasis(path_to_string(warmstart_fn)) m.optimize() - # if the solver is stopped (timelimit for example), postsolve the problem if m.attributes.solvestatus == xpress.enums.SolveStatus.STOPPED: - try: # Try new API first + try: m.postSolve() - except AttributeError: # Fallback to old API + except AttributeError: m.postsolve() if basis_fn is not None: try: - try: # Try new API first + try: m.writeBasis(path_to_string(basis_fn)) - except AttributeError: # Fallback to old API + except AttributeError: m.writebasis(path_to_string(basis_fn)) except (xpress.SolverError, xpress.ModelError) as err: logger.info("No model basis stored. Raised error: %s", err) if solution_fn is not None: try: - try: # Try new API first + try: m.writeBinSol(path_to_string(solution_fn)) - except AttributeError: # Fallback to old API + except AttributeError: m.writebinsol(path_to_string(solution_fn)) except (xpress.SolverError, xpress.ModelError) as err: logger.info("Unable to save solution file. Raised error: %s", err) @@ -2111,25 +2310,35 @@ def _run_file( def get_solver_solution() -> Solution: objective = m.attributes.objval - sol = _solution_from_names( - np.asarray(m.getSolution(), dtype=float), - [v.name for v in m.getVariable()], - self._n_vars, - ) + sol_values = np.asarray(m.getSolution(), dtype=float) + if from_file: + sol = _solution_from_names( + sol_values, + [v.name for v in m.getVariable()], + self._n_vars, + ) + else: + sol = _solution_from_labels(sol_values, self._vlabels, self._n_vars) try: if m.attributes.rows == 0: dual = np.array([], dtype=float) else: - try: # Try new API first + try: _dual = m.getDuals() - except AttributeError: # Fallback to old API + except AttributeError: _dual = m.getDual() - dual = _solution_from_names( - np.asarray(_dual, dtype=float), - [c.name for c in m.getConstraint()], - self._n_cons, - ) + dual_values = np.asarray(_dual, dtype=float) + if from_file: + dual = _solution_from_names( + dual_values, + [c.name for c in m.getConstraint()], + self._n_cons, + ) + else: + dual = _solution_from_labels( + dual_values, self._clabels, self._n_cons + ) except (xpress.SolverError, xpress.ModelError, SystemError): logger.warning("Dual values of MILP couldn't be parsed") dual = np.array([], dtype=float) diff --git a/test/test_io.py b/test/test_io.py index b049c0dc..fba65aab 100644 --- a/test/test_io.py +++ b/test/test_io.py @@ -307,6 +307,22 @@ def test_to_mosek(model: Model) -> None: assert task.getnumvar() > 0 +@pytest.mark.skipif("xpress" not in available_solvers, reason="Xpress not installed") +def test_to_xpress(model: Model) -> None: + p = model.to_xpress() + assert p.attributes.cols > 0 + assert p.attributes.rows > 0 + + +@pytest.mark.skipif("xpress" not in available_solvers, reason="Xpress not installed") +def test_to_xpress_no_names(model: Model) -> None: + p_with = model.to_xpress(set_names=True) + p_without = model.to_xpress(set_names=False) + names_with = [v.name for v in p_with.getVariable()] + names_without = [v.name for v in p_without.getVariable()] + assert names_with != names_without + + @pytest.mark.skipif("cupdlpx" not in available_solvers, reason="cuPDLPx not installed") def test_to_cupdlpx(model: Model) -> None: cu = model.to_cupdlpx() From c1e792b1a3df6b52bdcc7892d1aebc26e6d601ca Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 18 May 2026 08:31:10 +0000 Subject: [PATCH 02/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- linopy/solvers.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/linopy/solvers.py b/linopy/solvers.py index 8370ed50..e909ef4f 100644 --- a/linopy/solvers.py +++ b/linopy/solvers.py @@ -2089,7 +2089,9 @@ def _build_solver_model( objqcol2: np.ndarray | None objqcoef: np.ndarray | None if Q is not None and Q.nnz: - Qt = Q if Q.format == "coo" else triu(Q, format="coo") # codespell:ignore coo + Qt = ( + Q if Q.format == "coo" else triu(Q, format="coo") + ) # codespell:ignore coo mask = Qt.row <= Qt.col objqcol1 = Qt.row[mask].astype(np.int64, copy=False) objqcol2 = Qt.col[mask].astype(np.int64, copy=False) @@ -2150,9 +2152,13 @@ def _build_solver_model( vnames = print_variable(M.vlabels) if vnames: try: - problem.addNames(xpress_Namespaces.COLUMN, vnames, 0, len(vnames) - 1) + problem.addNames( + xpress_Namespaces.COLUMN, vnames, 0, len(vnames) - 1 + ) except AttributeError: - problem.addnames(xpress_Namespaces.COLUMN, vnames, 0, len(vnames) - 1) + problem.addnames( + xpress_Namespaces.COLUMN, vnames, 0, len(vnames) - 1 + ) cnames = print_constraint(M.clabels) if cnames: try: From 8e0c2942cd49f7f6a4e78a2fdd84b41b01e73a5f Mon Sep 17 00:00:00 2001 From: Fabian Date: Mon, 18 May 2026 10:34:56 +0200 Subject: [PATCH 03/18] fix: keep codespell:ignore inline with coo occurrences --- linopy/solvers.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/linopy/solvers.py b/linopy/solvers.py index e909ef4f..cb091945 100644 --- a/linopy/solvers.py +++ b/linopy/solvers.py @@ -2089,9 +2089,7 @@ def _build_solver_model( objqcol2: np.ndarray | None objqcoef: np.ndarray | None if Q is not None and Q.nnz: - Qt = ( - Q if Q.format == "coo" else triu(Q, format="coo") - ) # codespell:ignore coo + Qt = Q if Q.format == "coo" else triu(Q, format="coo") # codespell:ignore mask = Qt.row <= Qt.col objqcol1 = Qt.row[mask].astype(np.int64, copy=False) objqcol2 = Qt.col[mask].astype(np.int64, copy=False) From 761a99b6c858e8b4b8e92b5724c6301e7ae003b3 Mon Sep 17 00:00:00 2001 From: Fabian Date: Mon, 18 May 2026 10:40:29 +0200 Subject: [PATCH 04/18] fix(mypy): annotate Xpress rowtype/rhs to unify branches --- linopy/solvers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/linopy/solvers.py b/linopy/solvers.py index cb091945..85ad723d 100644 --- a/linopy/solvers.py +++ b/linopy/solvers.py @@ -2075,6 +2075,8 @@ def _build_solver_model( np.place(lb, np.isneginf(lb), -xpress.infinity) np.place(ub, np.isposinf(ub), xpress.infinity) + rowtype: np.ndarray + rhs: np.ndarray if len(M.clabels): sense = M.sense rowtype = np.full(sense.shape, "E", dtype="U1") From 3f35a137aece93de36ba5ccbd64e4c509f27253e Mon Sep 17 00:00:00 2001 From: Fabian Date: Mon, 18 May 2026 11:22:28 +0200 Subject: [PATCH 05/18] test(xpress): cover SOS via direct API (to_xpress + solve) --- test/test_sos_constraints.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test/test_sos_constraints.py b/test/test_sos_constraints.py index 5d94162e..f1762f37 100644 --- a/test/test_sos_constraints.py +++ b/test/test_sos_constraints.py @@ -137,6 +137,33 @@ def test_sos2_binary_maximize_different_coeffs() -> None: assert np.isclose(m.objective.value, 4) +@pytest.mark.skipif("xpress" not in available_solvers, reason="Xpress not installed") +def test_to_xpress_emits_sos_constraints() -> None: + m = Model() + segments = pd.Index([0.0, 0.5, 1.0], name="seg") + var = m.add_variables(coords=[segments], name="lambda") + m.add_sos_constraints(var, sos_type=1, sos_dim="seg") + m.add_objective(var.sum()) + + problem = m.to_xpress() + assert problem.attributes.sets == 1 + + +@pytest.mark.skipif("xpress" not in available_solvers, reason="Xpress not installed") +def test_sos2_xpress_direct() -> None: + m = Model() + locations = pd.Index([0, 1, 2], name="locations") + build = m.add_variables(coords=[locations], name="build", binary=True) + m.add_sos_constraints(build, sos_type=2, sos_dim="locations") + m.add_objective(build * np.array([1, 2, 3]), sense="max") + + m.solve(solver_name="xpress", io_api="direct") + + assert np.isclose(build.solution.values, [0, 1, 1]).all() + assert m.objective.value is not None + assert np.isclose(m.objective.value, 5) + + def test_unsupported_solver_raises_error() -> None: m = Model() locations = pd.Index([0, 1, 2], name="locations") From c5c7f20970efd0aa94897eef291b95914a06bedb Mon Sep 17 00:00:00 2001 From: Fabian Date: Mon, 18 May 2026 11:49:52 +0200 Subject: [PATCH 06/18] test(xpress): cover multi-dimensional SOS grouping --- test/test_sos_constraints.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/test_sos_constraints.py b/test/test_sos_constraints.py index f1762f37..b17f9267 100644 --- a/test/test_sos_constraints.py +++ b/test/test_sos_constraints.py @@ -149,6 +149,19 @@ def test_to_xpress_emits_sos_constraints() -> None: assert problem.attributes.sets == 1 +@pytest.mark.skipif("xpress" not in available_solvers, reason="Xpress not installed") +def test_to_xpress_emits_grouped_sos_constraints() -> None: + m = Model() + groups = pd.Index(["a", "b"], name="group") + segments = pd.Index([0.0, 0.5, 1.0], name="seg") + var = m.add_variables(coords=[groups, segments], name="lambda") + m.add_sos_constraints(var, sos_type=1, sos_dim="seg") + m.add_objective(var.sum()) + + problem = m.to_xpress() + assert problem.attributes.sets == len(groups) + + @pytest.mark.skipif("xpress" not in available_solvers, reason="Xpress not installed") def test_sos2_xpress_direct() -> None: m = Model() From db524bb2418e3ff18e36814d45669719e51ec324 Mon Sep 17 00:00:00 2001 From: Fabian Date: Mon, 18 May 2026 13:13:25 +0200 Subject: [PATCH 07/18] refactor(xpress): collapse new/old API dispatches to lowercase MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The lowercase Xpress API (addnames, chgobjsense, read, setlogfile, readbasis, postsolve, writebasis, writebinsol, getDual) exists in every released xpress version. Drop the try/except dispatch to the PascalCase aliases — they only added uncovered fallback branches without behavioural benefit on either 9.4 or 9.8+. --- linopy/solvers.py | 55 +++++++++-------------------------------------- 1 file changed, 10 insertions(+), 45 deletions(-) diff --git a/linopy/solvers.py b/linopy/solvers.py index 85ad723d..8cbb5aa8 100644 --- a/linopy/solvers.py +++ b/linopy/solvers.py @@ -2140,10 +2140,7 @@ def _build_solver_model( ) if model.objective.sense == "max": - try: - problem.chgObjSense(xpress.ObjSense.MAXIMIZE) - except AttributeError: - problem.chgobjsense(xpress.maximize) + problem.chgobjsense(xpress.maximize) if set_names: print_variable, print_constraint = linopy.io.get_printers_scalar( @@ -2151,20 +2148,10 @@ def _build_solver_model( ) vnames = print_variable(M.vlabels) if vnames: - try: - problem.addNames( - xpress_Namespaces.COLUMN, vnames, 0, len(vnames) - 1 - ) - except AttributeError: - problem.addnames( - xpress_Namespaces.COLUMN, vnames, 0, len(vnames) - 1 - ) + problem.addnames(xpress_Namespaces.COLUMN, vnames, 0, len(vnames) - 1) cnames = print_constraint(M.clabels) if cnames: - try: - problem.addNames(xpress_Namespaces.ROW, cnames, 0, len(cnames) - 1) - except AttributeError: - problem.addnames(xpress_Namespaces.ROW, cnames, 0, len(cnames) - 1) + problem.addnames(xpress_Namespaces.ROW, cnames, 0, len(cnames) - 1) if model.variables.sos: for var_name in model.variables.sos: @@ -2232,10 +2219,7 @@ def _run_file( sense = read_sense_from_problem_file(problem_fn) m = xpress.problem() - try: - m.readProb(path_to_string(problem_fn)) - except AttributeError: - m.read(path_to_string(problem_fn)) + m.read(path_to_string(problem_fn)) return self._solve( m, @@ -2271,40 +2255,25 @@ def _solve( m.setControl(self.solver_options) if log_fn is not None: - try: - m.setLogFile(path_to_string(log_fn)) - except AttributeError: - m.setlogfile(path_to_string(log_fn)) + m.setlogfile(path_to_string(log_fn)) if warmstart_fn is not None: - try: - m.readBasis(path_to_string(warmstart_fn)) - except AttributeError: - m.readbasis(path_to_string(warmstart_fn)) + m.readbasis(path_to_string(warmstart_fn)) m.optimize() if m.attributes.solvestatus == xpress.enums.SolveStatus.STOPPED: - try: - m.postSolve() - except AttributeError: - m.postsolve() + m.postsolve() if basis_fn is not None: try: - try: - m.writeBasis(path_to_string(basis_fn)) - except AttributeError: - m.writebasis(path_to_string(basis_fn)) + m.writebasis(path_to_string(basis_fn)) except (xpress.SolverError, xpress.ModelError) as err: logger.info("No model basis stored. Raised error: %s", err) if solution_fn is not None: try: - try: - m.writeBinSol(path_to_string(solution_fn)) - except AttributeError: - m.writebinsol(path_to_string(solution_fn)) + m.writebinsol(path_to_string(solution_fn)) except (xpress.SolverError, xpress.ModelError) as err: logger.info("Unable to save solution file. Raised error: %s", err) @@ -2330,11 +2299,7 @@ def get_solver_solution() -> Solution: if m.attributes.rows == 0: dual = np.array([], dtype=float) else: - try: - _dual = m.getDuals() - except AttributeError: - _dual = m.getDual() - dual_values = np.asarray(_dual, dtype=float) + dual_values = np.asarray(m.getDual(), dtype=float) if from_file: dual = _solution_from_names( dual_values, From cc4799e62129e8d29d13f970729505f7913bd5f4 Mon Sep 17 00:00:00 2001 From: Fabian Date: Mon, 18 May 2026 13:38:01 +0200 Subject: [PATCH 08/18] test(xpress): mark defensive exception handlers as no-cover --- linopy/solvers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/linopy/solvers.py b/linopy/solvers.py index 8cbb5aa8..ebe5eb27 100644 --- a/linopy/solvers.py +++ b/linopy/solvers.py @@ -2268,13 +2268,13 @@ def _solve( if basis_fn is not None: try: m.writebasis(path_to_string(basis_fn)) - except (xpress.SolverError, xpress.ModelError) as err: + except (xpress.SolverError, xpress.ModelError) as err: # pragma: no cover logger.info("No model basis stored. Raised error: %s", err) if solution_fn is not None: try: m.writebinsol(path_to_string(solution_fn)) - except (xpress.SolverError, xpress.ModelError) as err: + except (xpress.SolverError, xpress.ModelError) as err: # pragma: no cover logger.info("Unable to save solution file. Raised error: %s", err) condition = m.attributes.solstatus @@ -2310,7 +2310,7 @@ def get_solver_solution() -> Solution: dual = _solution_from_labels( dual_values, self._clabels, self._n_cons ) - except (xpress.SolverError, xpress.ModelError, SystemError): + except (xpress.SolverError, xpress.ModelError, SystemError): # pragma: no cover logger.warning("Dual values of MILP couldn't be parsed") dual = np.array([], dtype=float) From e302e4fcea1dcd0bc1b068f0a0fb9ace8bff92eb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 18 May 2026 11:38:10 +0000 Subject: [PATCH 09/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- linopy/solvers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/linopy/solvers.py b/linopy/solvers.py index ebe5eb27..cbff8a53 100644 --- a/linopy/solvers.py +++ b/linopy/solvers.py @@ -2310,7 +2310,11 @@ def get_solver_solution() -> Solution: dual = _solution_from_labels( dual_values, self._clabels, self._n_cons ) - except (xpress.SolverError, xpress.ModelError, SystemError): # pragma: no cover + except ( + xpress.SolverError, + xpress.ModelError, + SystemError, + ): # pragma: no cover logger.warning("Dual values of MILP couldn't be parsed") dual = np.array([], dtype=float) From 85708207f5568d4e4fcce6c0c9e1af4a4bf75155 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Mon, 18 May 2026 14:30:37 +0200 Subject: [PATCH 10/18] fix(sos): refuse masked SOS variables early; fix `reformulate_sos=True` on native SOS solvers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SOS plumbing (direct-API solver builds + LP-file writer) treats linopy variable labels as solver column indices. That assumption breaks as soon as any variable in the model is masked (label space becomes non-contiguous): - Gurobi direct: `IndexError` - Xpress direct (after #684): `?404 Invalid column number` - LP file: parse errors on Xpress/CPLEX, silent SOS-set corruption on Gurobi Until the proper fix (#688), refuse masked SOS up front with a clear `NotImplementedError` instead of producing solver-specific failures deeper down. - Add `linopy.io._raise_if_sos_has_masked(model)` and call it from `Solver._build` (covers both `Model.solve` and the 2-step `Solver.from_model().solve()` API) plus `sos_to_file` (covers standalone `m.to_file()`). - Fix the related bug where `reformulate_sos=True` silently no-op'd on solvers that support SOS natively (only a warning was emitted). `True` now means "always reformulate", as documented. - The combination of the two changes gives users a working workaround for masked SOS: pass `reformulate_sos=True` and the SOS gets converted to binary+linear constraints before reaching the masked-SOS guard. - Adjust the piecewise NaN-padding regression test: split the LP and SOS2 cases since the SOS2 path now errors (separate `test_sos2_per_entity_ nan_padding_errors` asserts that), and the LP case keeps its original regression coverage. Stacked on #684 — the Xpress changes are part of the affected surface. Refs #688. Co-Authored-By: Claude Opus 4.7 (1M context) --- linopy/io.py | 31 ++++++++++++++ linopy/model.py | 23 +++++------ linopy/solvers.py | 1 + test/test_piecewise_constraints.py | 57 ++++++++++++++++++-------- test/test_sos_constraints.py | 66 ++++++++++++++++++++++++++++++ 5 files changed, 147 insertions(+), 31 deletions(-) diff --git a/linopy/io.py b/linopy/io.py index 54adee87..3f273338 100644 --- a/linopy/io.py +++ b/linopy/io.py @@ -394,6 +394,35 @@ def integers_to_file( _format_and_write(df, columns, f) +def _raise_if_sos_has_masked(m: Model) -> None: + """ + Reject models whose SOS variables have masked entries. + + The SOS plumbing (both direct-API solvers and the LP file writer) treats + linopy variable labels as solver column indices / names, which breaks as + soon as a label is ``-1`` (linopy's ``FILL_VALUE["labels"]`` for masked + slots). The downstream symptoms are solver-specific — ``IndexError`` on + gurobipy, ``?404 Invalid column number`` on xpress, parse errors on + xpress/cplex LP readers, silent SOS-set corruption on gurobi's LP reader. + + Surface a single clear error until #688 lands the proper fix. + """ + if not m.variables.sos: + return + affected = [ + name + for name in m.variables.sos + if (m.variables[name].labels.values == -1).any() + ] + if affected: + raise NotImplementedError( + f"SOS constraints on masked variables are not yet supported " + f"(affected: {affected}; " + "see https://github.com/PyPSA/linopy/issues/688). " + "Pass reformulate_sos=True as a workaround." + ) + + def sos_to_file( m: Model, f: BufferedWriter, @@ -408,6 +437,8 @@ def sos_to_file( if not len(list(names)): return + _raise_if_sos_has_masked(m) + print_variable, _ = get_printers( m, explicit_coordinate_names=explicit_coordinate_names ) diff --git a/linopy/model.py b/linopy/model.py index ef31d7d0..24a75237 100644 --- a/linopy/model.py +++ b/linopy/model.py @@ -1594,12 +1594,10 @@ def solve( mock_solve : bool, optional Whether to run a mock solve. This will skip the actual solving. Variables will be set to have dummy values reformulate_sos : bool | Literal["auto"], optional - Whether to automatically reformulate SOS constraints as binary + linear - constraints for solvers that don't support them natively. - If True, always reformulates (warns if solver supports SOS natively). - If "auto", silently reformulates only when the solver lacks SOS support. - If False, raises if solver doesn't support SOS. - This uses the Big-M method and requires all SOS variables to have finite bounds. + Whether to reformulate SOS constraints as binary + linear constraints. + If True, always reformulates, even when the solver supports SOS natively. + If "auto", reformulates only when the solver lacks SOS support. + If False, raises if the solver doesn't support SOS. Default is False. **solver_options : kwargs Options passed to the solver. @@ -1715,18 +1713,17 @@ def solve( sos_reform_result = None if self.variables.sos: supports_sos = solver_class.supports(SolverFeature.SOS_CONSTRAINTS) - if reformulate_sos in (True, "auto") and not supports_sos: + should_reformulate = reformulate_sos is True or ( + reformulate_sos == "auto" and not supports_sos + ) + + if should_reformulate: logger.info(f"Reformulating SOS constraints for solver {solver_name}") sos_reform_result = reformulate_sos_constraints(self) - elif reformulate_sos is True and supports_sos: - logger.warning( - f"Solver {solver_name} supports SOS natively; " - "reformulate_sos=True is ignored." - ) elif reformulate_sos is False and not supports_sos: raise ValueError( f"Solver {solver_name} does not support SOS constraints. " - "Use reformulate_sos=True or 'auto', or a solver that supports SOS (gurobi, cplex)." + "Use reformulate_sos=True or 'auto', or a solver that supports SOS." ) if self.variables.semi_continuous: diff --git a/linopy/solvers.py b/linopy/solvers.py index cbff8a53..f0894bd4 100644 --- a/linopy/solvers.py +++ b/linopy/solvers.py @@ -507,6 +507,7 @@ def _build(self, **build_kwargs: Any) -> None: """Dispatch to direct or file build based on ``io_api``.""" if self.model is None: raise RuntimeError("Solver has no model attached; cannot build.") + linopy.io._raise_if_sos_has_masked(self.model) if self.io_api == "direct": self._build_direct(**build_kwargs) else: diff --git a/test/test_piecewise_constraints.py b/test/test_piecewise_constraints.py index 987336a4..baed1e9a 100644 --- a/test/test_piecewise_constraints.py +++ b/test/test_piecewise_constraints.py @@ -2286,30 +2286,51 @@ def test_lp_per_entity_nan_padding(self) -> None: Per-entity NaN-padded breakpoints with method='lp': padded segments must be masked out so they don't create spurious ``y ≤ 0`` constraints (bug-2 regression). + + ``method='sos2'`` would emit a masked SOS lambda variable, which the + native SOS path doesn't yet support (#688) — exercised separately in + :py:meth:`test_sos2_per_entity_nan_padding_errors`. """ from linopy.piecewise import breakpoints bp_y = pd.DataFrame([[0, 20, 30, 35], [0, 10, 15, np.nan]], index=["a", "b"]) bp_x = pd.DataFrame([[0, 10, 20, 30], [0, 5, 15, np.nan]], index=["a", "b"]) - results: dict[str, float] = {} - methods: list[Method] = ["lp", "sos2"] - for method in methods: - m = Model() - coord = pd.Index(["a", "b"], name="entity") - x = m.add_variables(lower=0, upper=20, coords=[coord], name="x") - y = m.add_variables(lower=0, upper=40, coords=[coord], name="y") - m.add_piecewise_formulation( - (y, breakpoints(bp_y, dim="entity"), "<="), - (x, breakpoints(bp_x, dim="entity")), - method=method, - ) - m.add_constraints(x.sel(entity="b") == 10) - m.add_objective(-y.sel(entity="b")) - m.solve() - results[method] = float(m.solution.sel({"entity": "b"})["y"]) + + m = Model() + coord = pd.Index(["a", "b"], name="entity") + x = m.add_variables(lower=0, upper=20, coords=[coord], name="x") + y = m.add_variables(lower=0, upper=40, coords=[coord], name="y") + m.add_piecewise_formulation( + (y, breakpoints(bp_y, dim="entity"), "<="), + (x, breakpoints(bp_x, dim="entity")), + method="lp", + ) + m.add_constraints(x.sel(entity="b") == 10) + m.add_objective(-y.sel(entity="b")) + m.solve() # f_b(10) on chord (5,10)→(15,15) is 12.5 - assert abs(results["lp"] - 12.5) < 1e-3 - assert abs(results["sos2"] - results["lp"]) < 1e-3 + assert abs(float(m.solution.sel({"entity": "b"})["y"]) - 12.5) < 1e-3 + + def test_sos2_per_entity_nan_padding_errors(self) -> None: + """Masked SOS lambdas hit the #688 guard at solve time.""" + from linopy.piecewise import breakpoints + + bp_y = pd.DataFrame([[0, 20, 30, 35], [0, 10, 15, np.nan]], index=["a", "b"]) + bp_x = pd.DataFrame([[0, 10, 20, 30], [0, 5, 15, np.nan]], index=["a", "b"]) + + m = Model() + coord = pd.Index(["a", "b"], name="entity") + x = m.add_variables(lower=0, upper=20, coords=[coord], name="x") + y = m.add_variables(lower=0, upper=40, coords=[coord], name="y") + m.add_piecewise_formulation( + (y, breakpoints(bp_y, dim="entity"), "<="), + (x, breakpoints(bp_x, dim="entity")), + method="sos2", + ) + m.add_constraints(x.sel(entity="b") == 10) + m.add_objective(-y.sel(entity="b")) + with pytest.raises(NotImplementedError, match="masked"): + m.solve() def test_lp_rejects_decreasing_x_concave_ge(self) -> None: """ diff --git a/test/test_sos_constraints.py b/test/test_sos_constraints.py index b17f9267..317598ac 100644 --- a/test/test_sos_constraints.py +++ b/test/test_sos_constraints.py @@ -177,6 +177,72 @@ def test_sos2_xpress_direct() -> None: assert np.isclose(m.objective.value, 5) +def _masked_sos_model(sos_type: int = 1) -> Model: + """Build a tiny model with a single masked SOS variable.""" + m = Model() + coords = pd.Index([0, 1, 2, 3], name="i") + mask = pd.Series([True, True, False, True], index=coords) + var = m.add_variables(lower=0, upper=1, coords=[coords], mask=mask, name="sos_var") + m.add_sos_constraints(var, sos_type=sos_type, sos_dim="i") + m.add_objective(-var.sum()) + return m + + +@pytest.mark.skipif("gurobi" not in available_solvers, reason="Gurobi not installed") +def test_gurobi_direct_raises_on_masked_sos() -> None: + m = _masked_sos_model() + with pytest.raises(NotImplementedError, match="masked"): + m.solve(solver_name="gurobi", io_api="direct") + + +@pytest.mark.skipif("xpress" not in available_solvers, reason="Xpress not installed") +def test_xpress_direct_raises_on_masked_sos() -> None: + m = _masked_sos_model() + with pytest.raises(NotImplementedError, match="masked"): + m.solve(solver_name="xpress", io_api="direct") + + +def test_lp_writer_raises_on_masked_sos(tmp_path: Path) -> None: + m = _masked_sos_model() + with pytest.raises(NotImplementedError, match="masked"): + m.to_file(tmp_path / "sos.lp", io_api="lp") + + +@pytest.mark.skipif("gurobi" not in available_solvers, reason="Gurobi not installed") +def test_reformulate_sos_true_solves_masked_sos() -> None: + """The documented workaround for the masked-SOS bug actually solves.""" + m = _masked_sos_model() + m.solve(solver_name="gurobi", reformulate_sos=True) + sol = m.variables["sos_var"].solution.values + # SOS1 over 3 unmasked entries, max sum, each in [0, 1]: + # one entry == 1, others == 0, masked stays NaN. + assert m.objective.value is not None + assert np.isclose(m.objective.value, -1.0) + assert np.isnan(sol[2]) + nonzero = np.flatnonzero(~np.isnan(sol) & (sol > 1e-6)) + assert len(nonzero) == 1 + assert np.isclose(sol[nonzero[0]], 1.0) + + +@pytest.mark.skipif("gurobi" not in available_solvers, reason="Gurobi not installed") +def test_reformulate_sos_true_reformulates_on_native_solver( + caplog: pytest.LogCaptureFixture, +) -> None: + """``reformulate_sos=True`` must reformulate even when the solver supports SOS.""" + import logging + + m = Model() + idx = pd.Index([0, 1, 2], name="i") + x = m.add_variables(lower=0, upper=1, coords=[idx], name="x") + m.add_sos_constraints(x, sos_type=1, sos_dim="i") + m.add_objective(x.sum()) + + with caplog.at_level(logging.INFO, logger="linopy.model"): + m.solve(solver_name="gurobi", reformulate_sos=True) + + assert any("Reformulating SOS" in msg for msg in caplog.messages) + + def test_unsupported_solver_raises_error() -> None: m = Model() locations = pd.Index([0, 1, 2], name="locations") From d005e044778f40721e98eea6684bef517a2c0a66 Mon Sep 17 00:00:00 2001 From: Fabian Hofmann Date: Mon, 18 May 2026 14:38:22 +0200 Subject: [PATCH 11/18] Apply suggestions from code review Co-authored-by: Felix <117816358+FBumann@users.noreply.github.com> --- linopy/solvers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linopy/solvers.py b/linopy/solvers.py index cbff8a53..d6298a45 100644 --- a/linopy/solvers.py +++ b/linopy/solvers.py @@ -2111,7 +2111,7 @@ def _build_solver_model( coltype = None problem.loadproblem( - probname="", + probname="linopy", rowtype=rowtype, rhs=rhs, rng=None, From e10937d5910c2bfeeb7e655cb647b8c4a570d8e1 Mon Sep 17 00:00:00 2001 From: Fabian Date: Mon, 18 May 2026 14:50:13 +0200 Subject: [PATCH 12/18] docs(xpress): document loadproblem; test(xpress): QP+SOS via direct API --- linopy/solvers.py | 10 +++++++++- test/test_sos_constraints.py | 19 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/linopy/solvers.py b/linopy/solvers.py index d6298a45..1b8aff78 100644 --- a/linopy/solvers.py +++ b/linopy/solvers.py @@ -2051,7 +2051,15 @@ def _build_solver_model( explicit_coordinate_names: bool = False, set_names: bool = True, ) -> xpress.problem: - """Build an ``xpress.problem`` that mirrors the linopy ``model`` via ``loadproblem``.""" + """ + Build an ``xpress.problem`` that mirrors the linopy ``model`` via ``loadproblem``. + + ``loadproblem`` is Xpress' universal native-array entry point loading LP/QP/MIQP + in a single call; see the parameter reference at + https://www.fico.com/fico-xpress-optimization/docs/latest/solver/optimizer/python/HTML/problem.loadproblem.html. + SOS arguments are left ``None`` and sets are added afterwards via ``addSOS`` so + multi-dim ``add_sos_constraints`` can be grouped natively. + """ model.constraints.sanitize_missings() problem = xpress.problem() diff --git a/test/test_sos_constraints.py b/test/test_sos_constraints.py index b17f9267..f340f768 100644 --- a/test/test_sos_constraints.py +++ b/test/test_sos_constraints.py @@ -5,6 +5,7 @@ import numpy as np import pandas as pd import pytest +import xarray as xr from linopy import Model, available_solvers @@ -177,6 +178,24 @@ def test_sos2_xpress_direct() -> None: assert np.isclose(m.objective.value, 5) +@pytest.mark.skipif("xpress" not in available_solvers, reason="Xpress not installed") +def test_qp_sos1_xpress_direct() -> None: + m = Model() + seg = pd.Index([0, 1, 2], name="seg") + x = m.add_variables(lower=0, upper=10, coords=[seg], name="x") + m.add_sos_constraints(x, sos_type=1, sos_dim="seg") + m.add_constraints(x.sum() >= 5) + + linear_coeffs = xr.DataArray([0.0, -10.0, 0.0], coords=[seg]) + m.add_objective((x * x).sum() + (linear_coeffs * x).sum(), sense="min") + + m.solve(solver_name="xpress", io_api="direct") + + assert np.isclose(x.solution.values, [0, 5, 0]).all() + assert m.objective.value is not None + assert np.isclose(m.objective.value, -25) + + def test_unsupported_solver_raises_error() -> None: m = Model() locations = pd.Index([0, 1, 2], name="locations") From 513334bee7c3b317058fab8d25866269497496b4 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Mon, 18 May 2026 15:06:41 +0200 Subject: [PATCH 13/18] refactor(sos): centralize masked-SOS guard on Model; widen test coverage Move the masked-SOS check to ``Model._check_sos_unmasked`` and call it from a single hoisted spot in ``to_file`` (covers LP and MPS) plus ``Solver._build``. Removes the now-unreachable masked filter from the Xpress direct ``add_sos`` helper. Restore Big-M / finite-bounds note to the ``reformulate_sos`` docstring. Replace the brittle log-string check in ``test_reformulate_sos_true_reformulates_on_native_solver`` with a behavioural assertion on the auxiliary artifacts the reformulation writes into the LP file, and parametrize the workaround test to also run on HiGHS. Co-Authored-By: Claude Opus 4.7 (1M context) --- linopy/io.py | 33 ++------------------- linopy/model.py | 31 +++++++++++++++++++- linopy/solvers.py | 10 ++----- test/test_sos_constraints.py | 57 ++++++++++++++++++++++++++++-------- 4 files changed, 79 insertions(+), 52 deletions(-) diff --git a/linopy/io.py b/linopy/io.py index 3f273338..4dc4dc02 100644 --- a/linopy/io.py +++ b/linopy/io.py @@ -394,35 +394,6 @@ def integers_to_file( _format_and_write(df, columns, f) -def _raise_if_sos_has_masked(m: Model) -> None: - """ - Reject models whose SOS variables have masked entries. - - The SOS plumbing (both direct-API solvers and the LP file writer) treats - linopy variable labels as solver column indices / names, which breaks as - soon as a label is ``-1`` (linopy's ``FILL_VALUE["labels"]`` for masked - slots). The downstream symptoms are solver-specific — ``IndexError`` on - gurobipy, ``?404 Invalid column number`` on xpress, parse errors on - xpress/cplex LP readers, silent SOS-set corruption on gurobi's LP reader. - - Surface a single clear error until #688 lands the proper fix. - """ - if not m.variables.sos: - return - affected = [ - name - for name in m.variables.sos - if (m.variables[name].labels.values == -1).any() - ] - if affected: - raise NotImplementedError( - f"SOS constraints on masked variables are not yet supported " - f"(affected: {affected}; " - "see https://github.com/PyPSA/linopy/issues/688). " - "Pass reformulate_sos=True as a workaround." - ) - - def sos_to_file( m: Model, f: BufferedWriter, @@ -437,8 +408,6 @@ def sos_to_file( if not len(list(names)): return - _raise_if_sos_has_masked(m) - print_variable, _ = get_printers( m, explicit_coordinate_names=explicit_coordinate_names ) @@ -622,6 +591,8 @@ def to_file( """ Write out a model to a lp or mps file. """ + m._check_sos_unmasked() + if fn is None: fn = Path(m.get_problem_file()) if isinstance(fn, str): diff --git a/linopy/model.py b/linopy/model.py index 24a75237..03fd9479 100644 --- a/linopy/model.py +++ b/linopy/model.py @@ -1221,6 +1221,34 @@ def remove_sos_constraints(self, variable: Variable) -> None: reformulate_sos_constraints = reformulate_sos_constraints + def _check_sos_unmasked(self) -> None: + """ + Reject the model if any SOS variable has masked entries. + + The SOS plumbing (both direct-API solvers and the LP file writer) treats + linopy variable labels as solver column indices / names, which breaks as + soon as a label is ``-1`` (linopy's ``FILL_VALUE["labels"]`` for masked + slots). The downstream symptoms are solver-specific — ``IndexError`` on + gurobipy, ``?404 Invalid column number`` on xpress, parse errors on + xpress/cplex LP readers, silent SOS-set corruption on gurobi's LP reader. + + Surface a single clear error until #688 lands the proper fix. + """ + if not self.variables.sos: + return + affected = [ + name + for name in self.variables.sos + if (self.variables[name].labels.values == -1).any() + ] + if affected: + raise NotImplementedError( + f"SOS constraints on masked variables are not yet supported " + f"(affected: {affected}; " + "see https://github.com/PyPSA/linopy/issues/688). " + "Pass reformulate_sos=True as a workaround." + ) + def remove_objective(self) -> None: """ Remove the objective's linear expression from the model. @@ -1598,7 +1626,8 @@ def solve( If True, always reformulates, even when the solver supports SOS natively. If "auto", reformulates only when the solver lacks SOS support. If False, raises if the solver doesn't support SOS. - Default is False. + Reformulation uses the Big-M method and requires all SOS variables + to have finite bounds. Default is False. **solver_options : kwargs Options passed to the solver. diff --git a/linopy/solvers.py b/linopy/solvers.py index f0894bd4..3349235d 100644 --- a/linopy/solvers.py +++ b/linopy/solvers.py @@ -507,7 +507,7 @@ def _build(self, **build_kwargs: Any) -> None: """Dispatch to direct or file build based on ``io_api``.""" if self.model is None: raise RuntimeError("Solver has no model attached; cannot build.") - linopy.io._raise_if_sos_has_masked(self.model) + self.model._check_sos_unmasked() if self.io_api == "direct": self._build_direct(**build_kwargs) else: @@ -2162,12 +2162,8 @@ def _build_solver_model( def add_sos(s: xr.DataArray, sos_type: int, sos_dim: str) -> None: s = s.squeeze() - labels = s.values.flatten() - mask = labels != -1 - if not mask.any(): - return - indices = labels[mask].tolist() - weights = s.coords[sos_dim].values[mask].tolist() + indices = s.values.flatten().tolist() + weights = s.coords[sos_dim].values.tolist() problem.addSOS(indices, weights, type=sos_type) others = [dim for dim in var.labels.dims if dim != sos_dim] diff --git a/test/test_sos_constraints.py b/test/test_sos_constraints.py index 317598ac..d7285272 100644 --- a/test/test_sos_constraints.py +++ b/test/test_sos_constraints.py @@ -208,11 +208,27 @@ def test_lp_writer_raises_on_masked_sos(tmp_path: Path) -> None: m.to_file(tmp_path / "sos.lp", io_api="lp") -@pytest.mark.skipif("gurobi" not in available_solvers, reason="Gurobi not installed") -def test_reformulate_sos_true_solves_masked_sos() -> None: +@pytest.mark.parametrize( + "solver_name", + [ + pytest.param( + "gurobi", + marks=pytest.mark.skipif( + "gurobi" not in available_solvers, reason="Gurobi not installed" + ), + ), + pytest.param( + "highs", + marks=pytest.mark.skipif( + "highs" not in available_solvers, reason="HiGHS not installed" + ), + ), + ], +) +def test_reformulate_sos_true_solves_masked_sos(solver_name: str) -> None: """The documented workaround for the masked-SOS bug actually solves.""" m = _masked_sos_model() - m.solve(solver_name="gurobi", reformulate_sos=True) + m.solve(solver_name=solver_name, reformulate_sos=True) sol = m.variables["sos_var"].solution.values # SOS1 over 3 unmasked entries, max sum, each in [0, 1]: # one entry == 1, others == 0, masked stays NaN. @@ -225,22 +241,37 @@ def test_reformulate_sos_true_solves_masked_sos() -> None: @pytest.mark.skipif("gurobi" not in available_solvers, reason="Gurobi not installed") -def test_reformulate_sos_true_reformulates_on_native_solver( - caplog: pytest.LogCaptureFixture, -) -> None: - """``reformulate_sos=True`` must reformulate even when the solver supports SOS.""" - import logging - +def test_reformulate_sos_true_reformulates_on_native_solver(tmp_path: Path) -> None: + """ + ``reformulate_sos=True`` must reformulate even when the solver supports SOS. + + Asserted against the artifacts ``reformulate_sos_constraints`` writes into + the LP file (the auxiliary binary + cardinality constraint, no ``sos`` + section). The reformulation is undone after solve, so the model itself + looks unchanged — the LP snapshot is the durable evidence. + """ m = Model() idx = pd.Index([0, 1, 2], name="i") x = m.add_variables(lower=0, upper=1, coords=[idx], name="x") m.add_sos_constraints(x, sos_type=1, sos_dim="i") m.add_objective(x.sum()) - with caplog.at_level(logging.INFO, logger="linopy.model"): - m.solve(solver_name="gurobi", reformulate_sos=True) - - assert any("Reformulating SOS" in msg for msg in caplog.messages) + problem_fn = tmp_path / "problem.lp" + m.solve( + solver_name="gurobi", + io_api="lp", + reformulate_sos=True, + problem_fn=problem_fn, + keep_files=True, + explicit_coordinate_names=True, + ) + + content = problem_fn.read_text() + # SOS got rewritten to binary + linear: no `sos` section, the auxiliary + # binary indicator and cardinality constraint appear instead. + assert "\nsos\n" not in content + assert "_sos_reform_x_y" in content + assert "_sos_reform_x_card" in content def test_unsupported_solver_raises_error() -> None: From 6d16358478a8b98951045a66a37faf95975f3bbd Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Mon, 18 May 2026 16:19:38 +0200 Subject: [PATCH 14/18] test(sos): convert masked-SOS helper to fixture, parametrize raise tests Addresses review feedback on #689: - _masked_sos_model() helper -> masked_sos_model pytest fixture - collapse three test_*_raises_on_masked_sos tests into one test_masked_sos_raises parametrized over (gurobi-direct, xpress-direct, lp-writer) Co-Authored-By: Claude Opus 4.7 (1M context) --- test/test_sos_constraints.py | 68 ++++++++++++++++++++++-------------- 1 file changed, 41 insertions(+), 27 deletions(-) diff --git a/test/test_sos_constraints.py b/test/test_sos_constraints.py index 34a93bff..a166bb30 100644 --- a/test/test_sos_constraints.py +++ b/test/test_sos_constraints.py @@ -1,7 +1,7 @@ from __future__ import annotations +from collections.abc import Callable from pathlib import Path -from typing import Literal import numpy as np import pandas as pd @@ -197,35 +197,48 @@ def test_qp_sos1_xpress_direct() -> None: assert np.isclose(m.objective.value, -25) -def _masked_sos_model(sos_type: Literal[1, 2] = 1) -> Model: - """Build a tiny model with a single masked SOS variable.""" +@pytest.fixture +def masked_sos_model() -> Model: + """Tiny model with a single masked SOS1 variable.""" m = Model() coords = pd.Index([0, 1, 2, 3], name="i") mask = pd.Series([True, True, False, True], index=coords) var = m.add_variables(lower=0, upper=1, coords=[coords], mask=mask, name="sos_var") - m.add_sos_constraints(var, sos_type=sos_type, sos_dim="i") + m.add_sos_constraints(var, sos_type=1, sos_dim="i") m.add_objective(-var.sum()) return m -@pytest.mark.skipif("gurobi" not in available_solvers, reason="Gurobi not installed") -def test_gurobi_direct_raises_on_masked_sos() -> None: - m = _masked_sos_model() - with pytest.raises(NotImplementedError, match="masked"): - m.solve(solver_name="gurobi", io_api="direct") - - -@pytest.mark.skipif("xpress" not in available_solvers, reason="Xpress not installed") -def test_xpress_direct_raises_on_masked_sos() -> None: - m = _masked_sos_model() - with pytest.raises(NotImplementedError, match="masked"): - m.solve(solver_name="xpress", io_api="direct") - - -def test_lp_writer_raises_on_masked_sos(tmp_path: Path) -> None: - m = _masked_sos_model() +@pytest.mark.parametrize( + "trigger", + [ + pytest.param( + lambda m, tmp_path: m.solve(solver_name="gurobi", io_api="direct"), + id="gurobi-direct", + marks=pytest.mark.skipif( + "gurobi" not in available_solvers, reason="Gurobi not installed" + ), + ), + pytest.param( + lambda m, tmp_path: m.solve(solver_name="xpress", io_api="direct"), + id="xpress-direct", + marks=pytest.mark.skipif( + "xpress" not in available_solvers, reason="Xpress not installed" + ), + ), + pytest.param( + lambda m, tmp_path: m.to_file(tmp_path / "sos.lp", io_api="lp"), + id="lp-writer", + ), + ], +) +def test_masked_sos_raises( + trigger: Callable[[Model, Path], object], + masked_sos_model: Model, + tmp_path: Path, +) -> None: with pytest.raises(NotImplementedError, match="masked"): - m.to_file(tmp_path / "sos.lp", io_api="lp") + trigger(masked_sos_model, tmp_path) @pytest.mark.parametrize( @@ -245,15 +258,16 @@ def test_lp_writer_raises_on_masked_sos(tmp_path: Path) -> None: ), ], ) -def test_reformulate_sos_true_solves_masked_sos(solver_name: str) -> None: +def test_reformulate_sos_true_solves_masked_sos( + solver_name: str, masked_sos_model: Model +) -> None: """The documented workaround for the masked-SOS bug actually solves.""" - m = _masked_sos_model() - m.solve(solver_name=solver_name, reformulate_sos=True) - sol = m.variables["sos_var"].solution.values + masked_sos_model.solve(solver_name=solver_name, reformulate_sos=True) + sol = masked_sos_model.variables["sos_var"].solution.values # SOS1 over 3 unmasked entries, max sum, each in [0, 1]: # one entry == 1, others == 0, masked stays NaN. - assert m.objective.value is not None - assert np.isclose(m.objective.value, -1.0) + assert masked_sos_model.objective.value is not None + assert np.isclose(masked_sos_model.objective.value, -1.0) assert np.isnan(sol[2]) nonzero = np.flatnonzero(~np.isnan(sol) & (sol > 1e-6)) assert len(nonzero) == 1 From 8e05eb73ba997f16e9b210a3abe4d8d825f58cc6 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Mon, 18 May 2026 16:22:09 +0200 Subject: [PATCH 15/18] test(sos): split masked-SOS raise tests by entry point Replace single parametrized test using lambda triggers with: - test_direct_api_raises_on_masked_sos: parametrized over solver_name - test_lp_writer_raises_on_masked_sos: dedicated for the to_file path The solve and to_file entry points hit distinct guards (Solver._build vs sos_to_file), so a single matrix obscured what was being exercised. Co-Authored-By: Claude Opus 4.7 (1M context) --- test/test_sos_constraints.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/test/test_sos_constraints.py b/test/test_sos_constraints.py index a166bb30..3d00474c 100644 --- a/test/test_sos_constraints.py +++ b/test/test_sos_constraints.py @@ -1,6 +1,5 @@ from __future__ import annotations -from collections.abc import Callable from pathlib import Path import numpy as np @@ -210,35 +209,34 @@ def masked_sos_model() -> Model: @pytest.mark.parametrize( - "trigger", + "solver_name", [ pytest.param( - lambda m, tmp_path: m.solve(solver_name="gurobi", io_api="direct"), - id="gurobi-direct", + "gurobi", marks=pytest.mark.skipif( "gurobi" not in available_solvers, reason="Gurobi not installed" ), ), pytest.param( - lambda m, tmp_path: m.solve(solver_name="xpress", io_api="direct"), - id="xpress-direct", + "xpress", marks=pytest.mark.skipif( "xpress" not in available_solvers, reason="Xpress not installed" ), ), - pytest.param( - lambda m, tmp_path: m.to_file(tmp_path / "sos.lp", io_api="lp"), - id="lp-writer", - ), ], ) -def test_masked_sos_raises( - trigger: Callable[[Model, Path], object], - masked_sos_model: Model, - tmp_path: Path, +def test_direct_api_raises_on_masked_sos( + solver_name: str, masked_sos_model: Model +) -> None: + with pytest.raises(NotImplementedError, match="masked"): + masked_sos_model.solve(solver_name=solver_name, io_api="direct") + + +def test_lp_writer_raises_on_masked_sos( + masked_sos_model: Model, tmp_path: Path ) -> None: with pytest.raises(NotImplementedError, match="masked"): - trigger(masked_sos_model, tmp_path) + masked_sos_model.to_file(tmp_path / "sos.lp", io_api="lp") @pytest.mark.parametrize( From 1c271dc4a8f38366596a23b9e1d168f949ccb0ba Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Mon, 18 May 2026 16:24:56 +0200 Subject: [PATCH 16/18] test(sos): derive direct-API SOS solvers from capabilities MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace hard-coded gurobi/xpress + skipif marks with a parametrize list built from SolverFeature.SOS_CONSTRAINTS ∩ DIRECT_API, matching the pattern already used in test_piecewise_constraints.py. Co-Authored-By: Claude Opus 4.7 (1M context) --- test/test_sos_constraints.py | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/test/test_sos_constraints.py b/test/test_sos_constraints.py index 3d00474c..30b2d767 100644 --- a/test/test_sos_constraints.py +++ b/test/test_sos_constraints.py @@ -8,6 +8,19 @@ import xarray as xr from linopy import Model, available_solvers +from linopy.solver_capabilities import ( + SolverFeature, + get_available_solvers_with_feature, + solver_supports, +) + +_direct_sos_solvers = [ + s + for s in get_available_solvers_with_feature( + SolverFeature.SOS_CONSTRAINTS, available_solvers + ) + if solver_supports(s, SolverFeature.DIRECT_API) +] def test_add_sos_constraints_registers_variable() -> None: @@ -208,23 +221,7 @@ def masked_sos_model() -> Model: return m -@pytest.mark.parametrize( - "solver_name", - [ - pytest.param( - "gurobi", - marks=pytest.mark.skipif( - "gurobi" not in available_solvers, reason="Gurobi not installed" - ), - ), - pytest.param( - "xpress", - marks=pytest.mark.skipif( - "xpress" not in available_solvers, reason="Xpress not installed" - ), - ), - ], -) +@pytest.mark.parametrize("solver_name", _direct_sos_solvers) def test_direct_api_raises_on_masked_sos( solver_name: str, masked_sos_model: Model ) -> None: From 78f01dabbeadcb68c8338ece0d2fd5c872038c1f Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Mon, 18 May 2026 16:38:24 +0200 Subject: [PATCH 17/18] test(pwl): factor NaN-padded model into a factory fixture Deduplicate the shared 10-line model construction between test_lp_per_entity_nan_padding and test_sos2_per_entity_nan_padding_errors into a module-level nan_padded_pwl_model factory fixture. Co-Authored-By: Claude Opus 4.7 (1M context) --- test/test_piecewise_constraints.py | 69 +++++++++++++++--------------- 1 file changed, 34 insertions(+), 35 deletions(-) diff --git a/test/test_piecewise_constraints.py b/test/test_piecewise_constraints.py index baed1e9a..3c91a88e 100644 --- a/test/test_piecewise_constraints.py +++ b/test/test_piecewise_constraints.py @@ -4,7 +4,7 @@ import logging import warnings -from collections.abc import Generator +from collections.abc import Callable, Generator from pathlib import Path from typing import TYPE_CHECKING, Any, Literal, TypeAlias @@ -2064,6 +2064,31 @@ def test_scalar_coord_dropped(self) -> None: # =========================================================================== +@pytest.fixture +def nan_padded_pwl_model() -> Callable[[Method], Model]: + """Factory: NaN-padded per-entity piecewise model parametrized by method.""" + from linopy.piecewise import breakpoints + + def _build(method: Method) -> Model: + bp_y = pd.DataFrame([[0, 20, 30, 35], [0, 10, 15, np.nan]], index=["a", "b"]) + bp_x = pd.DataFrame([[0, 10, 20, 30], [0, 5, 15, np.nan]], index=["a", "b"]) + + m = Model() + coord = pd.Index(["a", "b"], name="entity") + x = m.add_variables(lower=0, upper=20, coords=[coord], name="x") + y = m.add_variables(lower=0, upper=40, coords=[coord], name="y") + m.add_piecewise_formulation( + (y, breakpoints(bp_y, dim="entity"), "<="), + (x, breakpoints(bp_x, dim="entity")), + method=method, + ) + m.add_constraints(x.sel(entity="b") == 10) + m.add_objective(-y.sel(entity="b")) + return m + + return _build + + class TestSignParameter: """Tests for per-tuple sign on add_piecewise_formulation.""" @@ -2281,7 +2306,9 @@ def test_convexity_invariant_to_x_direction(self) -> None: assert f_asc.method != "lp" assert f_desc.method != "lp" - def test_lp_per_entity_nan_padding(self) -> None: + def test_lp_per_entity_nan_padding( + self, nan_padded_pwl_model: Callable[[Method], Model] + ) -> None: """ Per-entity NaN-padded breakpoints with method='lp': padded segments must be masked out so they don't create spurious @@ -2291,44 +2318,16 @@ def test_lp_per_entity_nan_padding(self) -> None: native SOS path doesn't yet support (#688) — exercised separately in :py:meth:`test_sos2_per_entity_nan_padding_errors`. """ - from linopy.piecewise import breakpoints - - bp_y = pd.DataFrame([[0, 20, 30, 35], [0, 10, 15, np.nan]], index=["a", "b"]) - bp_x = pd.DataFrame([[0, 10, 20, 30], [0, 5, 15, np.nan]], index=["a", "b"]) - - m = Model() - coord = pd.Index(["a", "b"], name="entity") - x = m.add_variables(lower=0, upper=20, coords=[coord], name="x") - y = m.add_variables(lower=0, upper=40, coords=[coord], name="y") - m.add_piecewise_formulation( - (y, breakpoints(bp_y, dim="entity"), "<="), - (x, breakpoints(bp_x, dim="entity")), - method="lp", - ) - m.add_constraints(x.sel(entity="b") == 10) - m.add_objective(-y.sel(entity="b")) + m = nan_padded_pwl_model("lp") m.solve() # f_b(10) on chord (5,10)→(15,15) is 12.5 assert abs(float(m.solution.sel({"entity": "b"})["y"]) - 12.5) < 1e-3 - def test_sos2_per_entity_nan_padding_errors(self) -> None: + def test_sos2_per_entity_nan_padding_errors( + self, nan_padded_pwl_model: Callable[[Method], Model] + ) -> None: """Masked SOS lambdas hit the #688 guard at solve time.""" - from linopy.piecewise import breakpoints - - bp_y = pd.DataFrame([[0, 20, 30, 35], [0, 10, 15, np.nan]], index=["a", "b"]) - bp_x = pd.DataFrame([[0, 10, 20, 30], [0, 5, 15, np.nan]], index=["a", "b"]) - - m = Model() - coord = pd.Index(["a", "b"], name="entity") - x = m.add_variables(lower=0, upper=20, coords=[coord], name="x") - y = m.add_variables(lower=0, upper=40, coords=[coord], name="y") - m.add_piecewise_formulation( - (y, breakpoints(bp_y, dim="entity"), "<="), - (x, breakpoints(bp_x, dim="entity")), - method="sos2", - ) - m.add_constraints(x.sel(entity="b") == 10) - m.add_objective(-y.sel(entity="b")) + m = nan_padded_pwl_model("sos2") with pytest.raises(NotImplementedError, match="masked"): m.solve() From 5725ab5b6b7ffbeda9aae8335ed61b80c2fa6dd1 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Mon, 18 May 2026 16:40:34 +0200 Subject: [PATCH 18/18] docs: add release notes for masked-SOS guard and reformulate_sos=True fix Co-Authored-By: Claude Opus 4.7 (1M context) --- doc/release_notes.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 0293ac94..e5b7033f 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -50,6 +50,11 @@ Most users should keep calling ``model.solve(...)``. If you want more control, y * ``Solver.solve_problem``, ``Solver.solve_problem_from_model``, and ``Solver.solve_problem_from_file`` still work but emit a ``DeprecationWarning``. Use ``Solver.from_name(...).solve()`` (or simply ``model.solve(...)``) instead. They will be removed in a future release. +**Bug Fixes** + +* SOS constraints on masked variables no longer cause solver-specific failures (Gurobi ``IndexError``, Xpress ``?404 Invalid column number``, LP parse errors, silent set corruption). ``Model.solve()`` and ``Model.to_file()`` now raise a clear ``NotImplementedError`` referring users to `#688 `__; pass ``reformulate_sos=True`` as a workaround. +* ``Model.solve(..., reformulate_sos=True)`` now actually reformulates SOS constraints even when the solver supports them natively. Previously it was silently ignored with a warning. + **Breaking Changes** * ``available_solvers`` now lists all *installed* solvers, even ones without a working license. If you used it to decide "can I actually solve with X?", switch to ``linopy.licensed_solvers`` or ``SolverClass.license_status()``.