From 3205daaa39d0e682606e7a3722d33ae16589f093 Mon Sep 17 00:00:00 2001
From: Savannah Ostrowski <savannah@python.org>
Date: Fri, 8 May 2026 11:38:52 -0700
Subject: [PATCH 1/4] Allow for configuration of timeout_minutes

---
 CHANGELOG.md                    | 14 ++++++++++++++
 README.md                       |  6 ++++++
 bench_runner/runners.py         |  4 ++++
 bench_runner/scripts/install.py |  3 +++
 tests/data/bench_runner.toml    |  1 +
 tests/test_config.py            |  8 ++++++++
 6 files changed, 36 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index fe37dfc3..01c8b8f8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,19 @@
 ## Unreleased
 
+### Configurable benchmark timeout per runner
+
+Runners can now set a `timeout_minutes` field in `bench_runner.toml` to
+override the maximum runtime for benchmark jobs. If unset, the per-OS default
+in the workflow template applies (Linux: 1440, others: GitHub default).
+
+```toml
+[runners.example]
+os = "linux"
+arch = "x86_64"
+hostname = "example"
+timeout_minutes = 210
+```
+
 ### Bugfixes
 
 #### Use PGO on weekly builds
diff --git a/README.md b/README.md
index 5d0a8174..25b804d6 100644
--- a/README.md
+++ b/README.md
@@ -98,6 +98,12 @@ You may limit the number of cores used to build Python with the `use_cores` opti
 use_cores = 2
 ```
 
+You may set a maximum runtime for benchmark jobs on a runner with `timeout_minutes`. If unset, the per-OS default in the workflow template applies (Linux: 1440, others: the GitHub default).
+
+```
+timeout_minutes = 210
+```
+
 ### Try a benchmarking run
 
 There are instructions for running a benchmarking action already in the `README.md` of your repo. Look there and give it a try!
diff --git a/bench_runner/runners.py b/bench_runner/runners.py
index 826a43fe..b218b0cb 100644
--- a/bench_runner/runners.py
+++ b/bench_runner/runners.py
@@ -46,6 +46,10 @@ class Runner:
     # The number of cores to use to compile CPython. If not provided, `make -j`
     # will be used.
     use_cores: int | None = None
+    # Maximum runtime for benchmark jobs on this runner, in minutes. If unset,
+    # the per-OS default in the workflow template applies (Linux: 1440, others:
+    # GitHub default).
+    timeout_minutes: int | None = None
 
     def __post_init__(self):
         if self.github_runner_name is None:
diff --git a/bench_runner/scripts/install.py b/bench_runner/scripts/install.py
index 3e78e7f7..80b68758 100644
--- a/bench_runner/scripts/install.py
+++ b/bench_runner/scripts/install.py
@@ -178,6 +178,9 @@ def generate__benchmark(src: Any) -> Any:
 
         runner_template["runs-on"].append(runner.github_runner_name)
 
+        if runner.timeout_minutes is not None:
+            runner_template["timeout-minutes"] = runner.timeout_minutes
+
         machine_clauses = [
             f"inputs.machine == '{runner.name}'",
             "inputs.machine == '__really_all'",
diff --git a/tests/data/bench_runner.toml b/tests/data/bench_runner.toml
index aeb00e49..0d6552a8 100644
--- a/tests/data/bench_runner.toml
+++ b/tests/data/bench_runner.toml
@@ -7,6 +7,7 @@ os = "linux"
 arch = "x86_64"
 hostname = "pyperf"
 plot = { name = "Linux", color = "C0" }
+timeout_minutes = 210
 
 [longitudinal_plot]
 subplots = [
diff --git a/tests/test_config.py b/tests/test_config.py
index 10cbfb9e..5011e1b4 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -16,3 +16,11 @@ def test_get_runner_for_hostname(monkeypatch):
     assert runner.os == "linux"
     assert runner.arch == "x86_64"
     assert runner.hostname == "pyperf"
+    assert runner.timeout_minutes == 210
+
+
+def test_runner_timeout_minutes_default():
+    runner = runners.Runner(
+        nickname="x", os="linux", arch="x86_64", hostname="x"
+    )
+    assert runner.timeout_minutes is None

From 8701ffe2c471950ec19ff1cd8bfbdbe2af47b1e0 Mon Sep 17 00:00:00 2001
From: Savannah Ostrowski <savannah@python.org>
Date: Fri, 8 May 2026 11:43:50 -0700
Subject: [PATCH 2/4] Ensure that timeout_minutes shows up after runs-on

---
 bench_runner/scripts/install.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/bench_runner/scripts/install.py b/bench_runner/scripts/install.py
index 80b68758..3b9a0b97 100644
--- a/bench_runner/scripts/install.py
+++ b/bench_runner/scripts/install.py
@@ -179,7 +179,13 @@ def generate__benchmark(src: Any) -> Any:
         runner_template["runs-on"].append(runner.github_runner_name)
 
         if runner.timeout_minutes is not None:
-            runner_template["timeout-minutes"] = runner.timeout_minutes
+            # Insert right after runs-on for readable YAML, regardless of
+            # whether the per-OS skeleton already had a timeout-minutes key.
+            runner_template.pop("timeout-minutes", None)
+            idx = list(runner_template.keys()).index("runs-on") + 1
+            runner_template.insert(
+                idx, "timeout-minutes", runner.timeout_minutes
+            )
 
         machine_clauses = [
             f"inputs.machine == '{runner.name}'",

From ab473322a3713d45150aa4deb902ea7494e28107 Mon Sep 17 00:00:00 2001
From: Savannah Ostrowski <savannah@python.org>
Date: Fri, 8 May 2026 11:50:46 -0700
Subject: [PATCH 3/4] Lint

---
 bench_runner/scripts/install.py | 5 +----
 tests/test_config.py            | 5 +----
 2 files changed, 2 insertions(+), 8 deletions(-)

diff --git a/bench_runner/scripts/install.py b/bench_runner/scripts/install.py
index 3b9a0b97..2eaf6fce 100644
--- a/bench_runner/scripts/install.py
+++ b/bench_runner/scripts/install.py
@@ -23,7 +23,6 @@
 from bench_runner import runners
 from bench_runner.util import PathLike
 
-
 ROOT_PATH = Path()
 TEMPLATE_PATH = Path(__file__).parents[1] / "templates"
 WORKFLOW_PATH = Path() / ".github" / "workflows"
@@ -183,9 +182,7 @@ def generate__benchmark(src: Any) -> Any:
             # whether the per-OS skeleton already had a timeout-minutes key.
             runner_template.pop("timeout-minutes", None)
             idx = list(runner_template.keys()).index("runs-on") + 1
-            runner_template.insert(
-                idx, "timeout-minutes", runner.timeout_minutes
-            )
+            runner_template.insert(idx, "timeout-minutes", runner.timeout_minutes)
 
         machine_clauses = [
             f"inputs.machine == '{runner.name}'",
diff --git a/tests/test_config.py b/tests/test_config.py
index 5011e1b4..5988b99b 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -4,7 +4,6 @@
 
 from bench_runner import runners
 
-
 DATA_PATH = Path(__file__).parent / "data"
 
 
@@ -20,7 +19,5 @@ def test_get_runner_for_hostname(monkeypatch):
 
 
 def test_runner_timeout_minutes_default():
-    runner = runners.Runner(
-        nickname="x", os="linux", arch="x86_64", hostname="x"
-    )
+    runner = runners.Runner(nickname="x", os="linux", arch="x86_64", hostname="x")
     assert runner.timeout_minutes is None

From bd7b6155aab5b8a5d08f4369a2bbc08bfb23a736 Mon Sep 17 00:00:00 2001
From: Savannah Ostrowski <savannah@python.org>
Date: Fri, 6 Nov 2026 13:02:38 -0800
Subject: [PATCH 4/4] Validate runner timeouts and test generated workflows

---
 CHANGELOG.md                    | 13 ++--------
 README.md                       |  2 +-
 bench_runner/runners.py         | 10 +++++---
 bench_runner/scripts/install.py |  6 ++---
 tests/test_config.py            | 34 +++++++++++++++++++++++---
 tests/test_install.py           | 43 +++++++++++++++++++++++++++++++++
 6 files changed, 87 insertions(+), 21 deletions(-)
 create mode 100644 tests/test_install.py

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 01c8b8f8..4aaced4f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,17 +2,8 @@
 
 ### Configurable benchmark timeout per runner
 
-Runners can now set a `timeout_minutes` field in `bench_runner.toml` to
-override the maximum runtime for benchmark jobs. If unset, the per-OS default
-in the workflow template applies (Linux: 1440, others: GitHub default).
-
-```toml
-[runners.example]
-os = "linux"
-arch = "x86_64"
-hostname = "example"
-timeout_minutes = 210
-```
+Set `timeout_minutes` in a runner's `bench_runner.toml` configuration to
+override its benchmark job timeout.
 
 ### Bugfixes
 
diff --git a/README.md b/README.md
index 25b804d6..9fffe79b 100644
--- a/README.md
+++ b/README.md
@@ -98,7 +98,7 @@ You may limit the number of cores used to build Python with the `use_cores` opti
 use_cores = 2
 ```
 
-You may set a maximum runtime for benchmark jobs on a runner with `timeout_minutes`. If unset, the per-OS default in the workflow template applies (Linux: 1440, others: the GitHub default).
+Set `timeout_minutes` to a positive integer to limit a runner's benchmark job runtime in minutes. If omitted, the workflow template's default applies (Linux: 1440; otherwise GitHub's default).
 
 ```
 timeout_minutes = 210
diff --git a/bench_runner/runners.py b/bench_runner/runners.py
index b218b0cb..c98e6cf0 100644
--- a/bench_runner/runners.py
+++ b/bench_runner/runners.py
@@ -46,12 +46,16 @@ class Runner:
     # The number of cores to use to compile CPython. If not provided, `make -j`
     # will be used.
     use_cores: int | None = None
-    # Maximum runtime for benchmark jobs on this runner, in minutes. If unset,
-    # the per-OS default in the workflow template applies (Linux: 1440, others:
-    # GitHub default).
+    # Benchmark job timeout in minutes. None uses the workflow template's default.
     timeout_minutes: int | None = None
 
     def __post_init__(self):
+        if self.timeout_minutes is not None and (
+            type(self.timeout_minutes) is not int or self.timeout_minutes <= 0
+        ):
+            raise ValueError(
+                f"Runner {self.nickname!r}: timeout_minutes must be a positive integer"
+            )
         if self.github_runner_name is None:
             self.github_runner_name = self.name
         if self.plot is None:
diff --git a/bench_runner/scripts/install.py b/bench_runner/scripts/install.py
index 2eaf6fce..24f72e41 100644
--- a/bench_runner/scripts/install.py
+++ b/bench_runner/scripts/install.py
@@ -23,6 +23,7 @@
 from bench_runner import runners
 from bench_runner.util import PathLike
 
+
 ROOT_PATH = Path()
 TEMPLATE_PATH = Path(__file__).parents[1] / "templates"
 WORKFLOW_PATH = Path() / ".github" / "workflows"
@@ -178,10 +179,9 @@ def generate__benchmark(src: Any) -> Any:
         runner_template["runs-on"].append(runner.github_runner_name)
 
         if runner.timeout_minutes is not None:
-            # Insert right after runs-on for readable YAML, regardless of
-            # whether the per-OS skeleton already had a timeout-minutes key.
+            # Keep the timeout immediately after runs-on in generated YAML.
             runner_template.pop("timeout-minutes", None)
-            idx = list(runner_template.keys()).index("runs-on") + 1
+            idx = list(runner_template).index("runs-on") + 1
             runner_template.insert(idx, "timeout-minutes", runner.timeout_minutes)
 
         machine_clauses = [
diff --git a/tests/test_config.py b/tests/test_config.py
index 5988b99b..f3673491 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -2,8 +2,13 @@
 import socket
 
 
+import pytest
+
+
+from bench_runner import config
 from bench_runner import runners
 
+
 DATA_PATH = Path(__file__).parent / "data"
 
 
@@ -18,6 +23,29 @@ def test_get_runner_for_hostname(monkeypatch):
     assert runner.timeout_minutes == 210
 
 
-def test_runner_timeout_minutes_default():
-    runner = runners.Runner(nickname="x", os="linux", arch="x86_64", hostname="x")
-    assert runner.timeout_minutes is None
+@pytest.mark.parametrize("timeout_minutes", [None, 1, 1440])
+def test_runner_timeout_minutes_valid(timeout_minutes):
+    runner = runners.Runner(
+        nickname="x",
+        os="linux",
+        arch="x86_64",
+        hostname="x",
+        timeout_minutes=timeout_minutes,
+    )
+    assert runner.timeout_minutes == timeout_minutes
+
+
+@pytest.mark.parametrize(
+    "timeout_minutes", ["0", "-1", "true", "false", "1.5", '"210"']
+)
+def test_runner_timeout_minutes_invalid(tmp_path, timeout_minutes):
+    cfgpath = tmp_path / "bench_runner.toml"
+    contents = (DATA_PATH / "bench_runner.toml").read_text()
+    cfgpath.write_text(
+        contents.replace(
+            "timeout_minutes = 210", f"timeout_minutes = {timeout_minutes}"
+        )
+    )
+
+    with pytest.raises(ValueError, match="timeout_minutes.*must be a positive integer"):
+        config.get_config(cfgpath)
diff --git a/tests/test_install.py b/tests/test_install.py
new file mode 100644
index 00000000..b2d024ce
--- /dev/null
+++ b/tests/test_install.py
@@ -0,0 +1,43 @@
+from pathlib import Path
+
+
+import pytest
+
+
+from bench_runner import config
+from bench_runner import runners
+from bench_runner.scripts import install
+
+
+DATA_PATH = Path(__file__).parent / "data"
+
+
+@pytest.mark.parametrize(
+    "os, default_timeout", [("linux", 1440), ("darwin", None), ("windows", None)]
+)
+def test_benchmark_timeout_minutes(tmp_path, monkeypatch, os, default_timeout):
+    cfg = config.get_config(DATA_PATH / "bench_runner.toml")
+    configured = runners.Runner(
+        nickname="configured",
+        os=os,
+        arch="x86_64",
+        hostname="configured",
+        timeout_minutes=210,
+    )
+    default = runners.Runner(
+        nickname="default", os=os, arch="x86_64", hostname="default"
+    )
+    monkeypatch.setattr(cfg, "runners", {"configured": configured, "default": default})
+    monkeypatch.setattr(config, "get_config", lambda: cfg)
+
+    source = install.load_yaml(install.TEMPLATE_PATH / "_benchmark.src.yml")
+    output = tmp_path / "_benchmark.yml"
+    install.write_yaml(output, install.generate__benchmark(source), check=False)
+    jobs = install.load_yaml(output)["jobs"]
+
+    assert jobs[f"benchmark-{configured.name}"]["timeout-minutes"] == 210
+    default_job = jobs[f"benchmark-{default.name}"]
+    if default_timeout is None:
+        assert "timeout-minutes" not in default_job
+    else:
+        assert default_job["timeout-minutes"] == default_timeout