From b0719e18d6fd789c1fe9460fb0780ef2fe067cd6 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 10 Apr 2026 15:03:52 +0200 Subject: [PATCH 1/7] Remove unused variable --- master/master.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/master/master.cfg b/master/master.cfg index 174a903f..94fa6f8d 100644 --- a/master/master.cfg +++ b/master/master.cfg @@ -244,7 +244,7 @@ mail_status_builders = [] # Regular builders -for branch_num, (git_url, branchname, git_branch) in enumerate(git_branches): +for git_url, branchname, git_branch in git_branches: buildernames = [] refleakbuildernames = [] for name, worker, buildfactory, stability, tier in BUILDERS: From 451bed4572da82edc7d7e325ad94310e9c5c1d5e Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 10 Apr 2026 15:16:37 +0200 Subject: [PATCH 2/7] Move downtime calculations to their own file --- master/custom/worker_downtime.py | 48 ++++++++++++++++++++++++++++++++ master/master.cfg | 47 ++----------------------------- 2 files changed, 50 insertions(+), 45 deletions(-) create mode 100644 master/custom/worker_downtime.py diff --git a/master/custom/worker_downtime.py b/master/custom/worker_downtime.py new file mode 100644 index 00000000..41f4cf48 --- /dev/null +++ b/master/custom/worker_downtime.py @@ -0,0 +1,48 @@ +from datetime import datetime, timedelta + + +def no_builds_between(start, end, *, day_of_week=None, tz=None): + """A function for "builder.canStartBuild" that implements planned downtime + + Avoid a build to be started between start and end time and delay such + builds until end time. + """ + + start = datetime.strptime(start, "%H:%M").time() + end = datetime.strptime(end, "%H:%M").time() + def canStartBuild(builder, wfb, request): + now_dt = datetime.now(tz=tz) + if day_of_week is not None and now_dt.weekday() != day_of_week: + return True + now = now_dt.time() + if is_within_time_range(now, start, end): + delay = get_delay(now, end) + # Adapted from: https://docs.buildbot.net/current/manual/customization.html#canstartbuild-functions + wfb.worker.quarantine_timeout = delay + wfb.worker.putInQuarantine() + # This does not take the worker out of quarantine, it only resets + # the timeout value to default (restarting the default + # exponential backoff) + wfb.worker.resetQuarantine() + return False + # Schedule the build now + return True + return canStartBuild + +def is_within_time_range(now, start, end): + if start <= end: + return start <= now <= end + else: + return now >= start or now <= end + + +def get_delay(now, end): + today = datetime.today() + now = datetime.combine(today, now) + end = datetime.combine(today, end) + + if now > end: + end += timedelta(days=1) + + difference = end - now + return difference.total_seconds() diff --git a/master/master.cfg b/master/master.cfg index 94fa6f8d..7a6621fe 100644 --- a/master/master.cfg +++ b/master/master.cfg @@ -14,7 +14,7 @@ import os import subprocess import sys -from datetime import datetime, timedelta +from datetime import timedelta from functools import partial from zoneinfo import ZoneInfo @@ -52,6 +52,7 @@ from custom.builders import ( # noqa: E402 STABLE, ONLY_MAIN_BRANCH, ) +from custom.worker_downtime import no_builds_between def set_up_sentry(): @@ -194,50 +195,6 @@ def is_important_change(change): return any(is_important_file(filename) for filename in change.files) -def is_within_time_range(now, start, end): - if start <= end: - return start <= now <= end - else: - return now >= start or now <= end - - -def get_delay(now, end): - today = datetime.today() - now = datetime.combine(today, now) - end = datetime.combine(today, end) - - if now > end: - end += timedelta(days=1) - - difference = end - now - return difference.total_seconds() - - -# Avoid a build to be started between start and end time and delay such build -# at end time -def no_builds_between(start, end, *, day_of_week=None, tz=None): - start = datetime.strptime(start, "%H:%M").time() - end = datetime.strptime(end, "%H:%M").time() - def canStartBuild(builder, wfb, request): - now_dt = datetime.now(tz=tz) - if day_of_week is not None and now_dt.weekday() != day_of_week: - return True - now = now_dt.time() - if is_within_time_range(now, start, end): - delay = get_delay(now, end) - # Adapted from: https://docs.buildbot.net/current/manual/customization.html#canstartbuild-functions - wfb.worker.quarantine_timeout = delay - wfb.worker.putInQuarantine() - # This does not take the worker out of quarantine, it only resets - # the timeout value to default (restarting the default - # exponential backoff) - wfb.worker.resetQuarantine() - return False - # Schedule the build now - return True - return canStartBuild - - github_status_builders = [] release_status_builders = [] mail_status_builders = [] From 69f7080af77cb068c6386c421d671c98383ecf0e Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 10 Apr 2026 15:29:27 +0200 Subject: [PATCH 3/7] Move worker downtime configuration to workers.py The `canStartBuild=no_builds_between(...)` configuration is used for regular maintenance windows (restarts/reprovisions) and quiet periods for benchmark runs. - Rename to `downtime`; canStartBuild & quarantining are implementation details. - Move configuration to workers.py. - Move implementation to new module, worker_downtime.py --- master/custom/workers.py | 22 ++++++++++++++++++++++ master/master.cfg | 34 ++++------------------------------ 2 files changed, 26 insertions(+), 30 deletions(-) diff --git a/master/custom/workers.py b/master/custom/workers.py index ebd7dd9b..b1b4ab07 100644 --- a/master/custom/workers.py +++ b/master/custom/workers.py @@ -3,10 +3,12 @@ # vim:set ts=8 sw=4 sts=4 et: from functools import partial +from zoneinfo import ZoneInfo from buildbot.plugins import worker as _worker from custom.factories import MAIN_BRANCH_NAME +from custom.worker_downtime import no_builds_between # By default, the buildmaster sends a simple, non-blocking message to each @@ -32,6 +34,7 @@ def __init__( parallel_tests=None, timeout_factor=1, exclude_test_resources=None, + downtime=None ): self.name = name self.tags = tags or set() @@ -41,6 +44,7 @@ def __init__( self.parallel_tests = parallel_tests self.timeout_factor = timeout_factor self.exclude_test_resources = exclude_test_resources or [] + self.downtime = downtime worker_settings = settings.workers[name] owner = name.split("-")[0] @@ -55,6 +59,12 @@ def __init__( notify_on_missing=emails, keepalive_interval=KEEPALIVE) +# Some of Itamar's workers are reprovisioned every Wednesday at 9am PT. +# Builds scheduled between 8am - 10am PT on Wednesdays will be delayed to +# 10am PT. +itamaro_downtime = no_builds_between( + "8:00", "10:00", day_of_week=2, tz=ZoneInfo("America/Los_Angeles") +) def get_workers(settings): cpw = partial(CPythonWorker, settings) @@ -160,6 +170,10 @@ def get_workers(settings): tags=['linux', 'unix', 'ubuntu', 'arm', 'arm64', 'aarch64', 'bigmem'], not_branches=['3.10', '3.11', '3.12', '3.13', '3.14'], parallel_tests=8, + # This worker runs pyperformance at 12am UTC. + # If a build is scheduled between 10pm UTC and 2am UTC, + # it will be delayed to 2am UTC. + downtime=no_builds_between("22:00", "2:00") ), cpw( name="cstratak-fedora-rawhide-s390x", @@ -294,6 +308,12 @@ def get_workers(settings): tags=['windows', 'win11', 'amd64', 'x86-64', 'bigmem'], not_branches=['3.10', '3.11', '3.12', '3.13', '3.14'], parallel_tests=4, + # This worker restarts every day at 9am UTC to work around issues + # stemming from failing bigmem tests trashing disk space and + # fragmenting RAM. + # Builds scheduled between 07:20am - 9:20am UTC will be delayed + # to 9:20am UTC. + downtime=no_builds_between("7:20", "9:20") ), cpw( name="itamaro-centos-aws", @@ -301,6 +321,7 @@ def get_workers(settings): not_branches=['3.10', '3.11', '3.12'], parallel_tests=10, parallel_builders=2, + downtime=itamaro_downtime, ), cpw( name="itamaro-win64-srv-22-aws", @@ -308,6 +329,7 @@ def get_workers(settings): not_branches=['3.10', '3.11', '3.12'], parallel_tests=10, parallel_builders=2, + downtime=itamaro_downtime, ), cpw( name="itamaro-macos-intel-aws", diff --git a/master/master.cfg b/master/master.cfg index 7a6621fe..3c03e74b 100644 --- a/master/master.cfg +++ b/master/master.cfg @@ -265,24 +265,8 @@ for git_url, branchname, git_branch in git_branches: locks=[cpulock.access("counting")], ) - # This worker runs pyperformance at 12am UTC. If a build is scheduled between - # 10pm UTC and 2am UTC, it will be delayed to 2am UTC. - if worker.name == "diegorusso-aarch64-bigmem": - builder.canStartBuild = no_builds_between("22:00", "2:00") - - # This worker restarts every day at 9am UTC to work around issues stemming from - # failing bigmem tests trashing disk space and fragmenting RAM. Builds scheduled - # between 07:20am - 9:20am UTC will be delayed to 9:20am UTC. - if worker.name == "ambv-bb-win11": - builder.canStartBuild = no_builds_between("7:20", "9:20") - - # These workers are reprovisioned every Wednesday at 9am PT. Builds - # scheduled between 8am - 10am PT on Wednesdays will be delayed to - # 10am PT. - if worker.name in ("itamaro-win64-srv-22-aws", "itamaro-centos-aws"): - builder.canStartBuild = no_builds_between( - "8:00", "10:00", day_of_week=2, tz=ZoneInfo("America/Los_Angeles") - ) + if worker.downtime: + builder.canStartBuild = worker.downtime c["builders"].append(builder) @@ -354,18 +338,8 @@ for name, worker, buildfactory, stability, tier in BUILDERS: locks=[cpulock.access("counting")], ) - # This worker runs pyperformance at 12am. If a build is scheduled between - # 10pm and 2am, it will be delayed at 2am. - if worker.name == "diegorusso-aarch64-bigmem": - builder.canStartBuild = no_builds_between("22:00", "2:00") - - # These workers are reprovisioned every Wednesday at 9am PT. Builds - # scheduled between 8am - 10am PT on Wednesdays will be delayed to - # 10am PT. - if worker.name in ("itamaro-win64-srv-22-aws", "itamaro-centos-aws"): - builder.canStartBuild = no_builds_between( - "8:00", "10:00", day_of_week=2, tz=ZoneInfo("America/Los_Angeles") - ) + if worker.downtime: + builder.canStartBuild = worker.downtime c["builders"].append(builder) From dadb464be4ac3fa38765f0870464ca9aa12f1859 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 10 Apr 2026 16:05:11 +0200 Subject: [PATCH 4/7] Whitespace --- master/custom/worker_downtime.py | 1 + 1 file changed, 1 insertion(+) diff --git a/master/custom/worker_downtime.py b/master/custom/worker_downtime.py index 41f4cf48..3f73936b 100644 --- a/master/custom/worker_downtime.py +++ b/master/custom/worker_downtime.py @@ -29,6 +29,7 @@ def canStartBuild(builder, wfb, request): return True return canStartBuild + def is_within_time_range(now, start, end): if start <= end: return start <= now <= end From 38be801c3c2123e147ffb70ddc6d33b47d5df521 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 13 Apr 2026 17:48:38 +0200 Subject: [PATCH 5/7] Address review comments --- master/custom/workers.py | 6 +++++- master/master.cfg | 1 - 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/master/custom/workers.py b/master/custom/workers.py index b1b4ab07..170bf3da 100644 --- a/master/custom/workers.py +++ b/master/custom/workers.py @@ -5,6 +5,8 @@ from functools import partial from zoneinfo import ZoneInfo +import calendar + from buildbot.plugins import worker as _worker from custom.factories import MAIN_BRANCH_NAME @@ -63,7 +65,9 @@ def __init__( # Builds scheduled between 8am - 10am PT on Wednesdays will be delayed to # 10am PT. itamaro_downtime = no_builds_between( - "8:00", "10:00", day_of_week=2, tz=ZoneInfo("America/Los_Angeles") + "8:00", "10:00", + day_of_week=calendar.WEDNESDAY, + tz=ZoneInfo("America/Los_Angeles"), ) def get_workers(settings): diff --git a/master/master.cfg b/master/master.cfg index 3c03e74b..bd6870b8 100644 --- a/master/master.cfg +++ b/master/master.cfg @@ -16,7 +16,6 @@ import sys from datetime import timedelta from functools import partial -from zoneinfo import ZoneInfo from buildbot.plugins import reporters, schedulers, util from buildbot import locks From 49e2b6fad24f5ff0a837612f25fa1b3854c2245a Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 16 Apr 2026 07:50:17 +0200 Subject: [PATCH 6/7] Remove phantom import --- master/master.cfg | 1 - 1 file changed, 1 deletion(-) diff --git a/master/master.cfg b/master/master.cfg index bd6870b8..9c56ed53 100644 --- a/master/master.cfg +++ b/master/master.cfg @@ -51,7 +51,6 @@ from custom.builders import ( # noqa: E402 STABLE, ONLY_MAIN_BRANCH, ) -from custom.worker_downtime import no_builds_between def set_up_sentry(): From aa978bfa654a20e39efdda8433a55784fdf49c6f Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 16 Apr 2026 07:51:41 +0200 Subject: [PATCH 7/7] Whitespace --- master/custom/workers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/master/custom/workers.py b/master/custom/workers.py index 170bf3da..15ed5333 100644 --- a/master/custom/workers.py +++ b/master/custom/workers.py @@ -4,7 +4,6 @@ from functools import partial from zoneinfo import ZoneInfo - import calendar from buildbot.plugins import worker as _worker