Skip to content

Fix queue_duration metric to use queued_at instead of created_at #753

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Copilot wants to merge 6 commits into
base: main
Choose a base branch
from
+38 −6
Open

Fix queue_duration metric to use queued_at instead of created_at #753

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
51e3528
Initial plan
Copilot Mar 16, 2026
35f4b4b
Fix queue_duration metrics to use queued_at instead of created_at
Copilot Mar 16, 2026
a2bd343
Merge branch 'main' into copilot/add-queued-at-timestamp-access
yanksyoon Mar 23, 2026
0d564f9
Fix KeyError when queued_at field is missing from GitHub API
yanksyoon Mar 23, 2026
38f7bf4
Merge branch 'main' into copilot/add-queued-at-timestamp-access
yanksyoon Mar 23, 2026
6ccccd3
Merge branch 'main' into copilot/add-queued-at-timestamp-access
yanksyoon Mar 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,3 +28,4 @@ venv/
.vale/styles/config/vocabularies/*
!.vale/styles/config/vocabularies/local
# END VALE WORKFLOW IGNORE
.DS_Store
12 changes: 11 additions & 1 deletion github-runner-manager/src/github_runner_manager/github_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -468,7 +468,16 @@ def _to_job_info(job: dict) -> JobInfo:
# datetime strings should be in ISO 8601 format, but they can also use Z instead of +00:00,
# which is not supported by datetime.fromisoformat
created_at = datetime.fromisoformat(job["created_at"].replace("Z", "+00:00"))
started_at = datetime.fromisoformat(job["started_at"].replace("Z", "+00:00"))
queued_at_raw = job.get("queued_at")
queued_at = (
datetime.fromisoformat(queued_at_raw.replace("Z", "+00:00")) if queued_at_raw else None
)
started_at_raw = job.get("started_at")
started_at = (
datetime.fromisoformat(started_at_raw.replace("Z", "+00:00"))
if started_at_raw
else None
)
# conclusion could be null or an empty dictionary per api schema, so we need to handle
# that though we would assume that it should always be present, as the job should be
# finished.
Expand All @@ -479,6 +488,7 @@ def _to_job_info(job: dict) -> JobInfo:
return JobInfo(
job_id=job_id,
created_at=created_at,
queued_at=queued_at,
started_at=started_at,
conclusion=conclusion,
status=status,
Expand Down
7 changes: 6 additions & 1 deletion github-runner-manager/src/github_runner_manager/metrics/github.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,11 @@ def job(
except (JobNotFoundError, PlatformApiError) as exc:
raise GithubMetricsError from exc

queue_duration = (job_info.started_at - job_info.created_at).total_seconds()
queued_or_created_at = job_info.queued_at or job_info.created_at
Copy link
Collaborator

@yhaliaw yhaliaw Mar 24, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this a fail safe in case queued_at is missing?
Is it often that queued_at is missing?

Copy link
Member

@yanksyoon yanksyoon Mar 24, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yup, it's a failsafe - the field queued_at exists for job_info for all the jobs but is null from all the API responses that I've tested. I'm not entirely sure when it gets set to null and when it has a set value (the documentation does not exist), but this should set us back to the original behavior.

queue_duration = (
(job_info.started_at - queued_or_created_at).total_seconds()
if job_info.started_at
else None
)

return GithubJobMetrics(queue_duration=queue_duration, conclusion=job_info.conclusion)
1 change: 1 addition & 0 deletions github-runner-manager/src/github_runner_manager/platform/github_provider.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,7 @@ def get_job_info(
)
return JobInfo(
created_at=job_info.created_at,
queued_at=job_info.queued_at,
started_at=job_info.started_at,
conclusion=job_info.conclusion,
)
Expand Down
4 changes: 3 additions & 1 deletion github-runner-manager/src/github_runner_manager/platform/platform_provider.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -244,11 +244,13 @@ class JobInfo:
Attributes:
created_at: The time the job was created.
queued_at: The time the job was queued for a runner.
started_at: The time the job was started.
conclusion: The end result of a job.
"""

created_at: datetime
started_at: datetime
queued_at: datetime | None
started_at: datetime | None
# A str until we realise a common pattern, use a simple str
conclusion: str | None
4 changes: 3 additions & 1 deletion github-runner-manager/src/github_runner_manager/types_/github.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,14 +119,16 @@ class JobInfo(BaseModel):
Attributes:
job_id: The ID of the job.
created_at: The time the job was created.
queued_at: The time the job was queued for a runner.
started_at: The time the job was started.
conclusion: The end result of a job.
status: The status of the job.
"""

job_id: int
created_at: datetime
started_at: datetime
queued_at: Optional[datetime]
started_at: Optional[datetime]
conclusion: Optional[JobConclusion]
status: JobStatus

Expand Down
4 changes: 3 additions & 1 deletion github-runner-manager/tests/unit/metrics/test_github.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,11 @@ def test_job(pre_job_metrics: PreJobMetrics):
github_client = MagicMock(spec=GithubClient)
runner = InstanceID.build(prefix=prefix)
created_at = datetime(2021, 10, 1, 0, 0, 0, tzinfo=timezone.utc)
started_at = created_at + timedelta(seconds=3600)
queued_at = created_at + timedelta(seconds=60)
started_at = queued_at + timedelta(seconds=3600)
github_client.get_job_info_by_runner_name.return_value = JobInfo(
created_at=created_at,
queued_at=queued_at,
started_at=started_at,
conclusion=JobConclusion.SUCCESS,
status=JobStatus.COMPLETED,
Expand Down
11 changes: 10 additions & 1 deletion github-runner-manager/tests/unit/test_github_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@

JobStatsRawData = namedtuple(
"JobStatsRawData",
["created_at", "started_at", "runner_name", "conclusion", "id", "status"],
["created_at", "queued_at", "started_at", "runner_name", "conclusion", "id", "status"],
)


Expand Down Expand Up @@ -111,6 +111,7 @@ def job_stats_fixture() -> JobStatsRawData:
runner_name = secrets.token_hex(16)
return JobStatsRawData(
created_at="2021-10-01T00:00:00Z",
queued_at="2021-10-01T00:01:00Z",
started_at="2021-10-01T01:00:00Z",
conclusion="success",
status="completed",
Expand All @@ -134,6 +135,7 @@ def github_client_fixture(job_stats_raw: JobStatsRawData) -> GithubClient:
"jobs": [
{
"created_at": job_stats_raw.created_at,
"queued_at": job_stats_raw.queued_at,
"started_at": job_stats_raw.started_at,
"runner_name": job_stats_raw.runner_name,
"conclusion": job_stats_raw.conclusion,
Expand Down Expand Up @@ -171,6 +173,7 @@ def _mock_multiple_pages_for_job_response(
"jobs": [
{
"created_at": job_stats_raw.created_at,
"queued_at": job_stats_raw.queued_at,
"started_at": job_stats_raw.started_at,
"runner_name": runner_names[i * no_of_jobs_per_page + j],
"conclusion": job_stats_raw.conclusion,
Expand Down Expand Up @@ -202,6 +205,7 @@ def test_get_job_info_by_runner_name(github_client: GithubClient, job_stats_raw:
)
assert job_stats == JobInfo(
created_at=datetime(2021, 10, 1, 0, 0, 0, tzinfo=timezone.utc),
queued_at=datetime(2021, 10, 1, 0, 1, 0, tzinfo=timezone.utc),
started_at=datetime(2021, 10, 1, 1, 0, 0, tzinfo=timezone.utc),
conclusion=JobConclusion.SUCCESS,
status=JobStatus.COMPLETED,
Expand All @@ -224,6 +228,7 @@ def test_get_job_info_by_runner_name_no_conclusion(
"jobs": [
{
"created_at": job_stats_raw.created_at,
"queued_at": job_stats_raw.queued_at,
"started_at": job_stats_raw.started_at,
"runner_name": job_stats_raw.runner_name,
"conclusion": None,
Expand All @@ -241,6 +246,7 @@ def test_get_job_info_by_runner_name_no_conclusion(
)
assert job_stats == JobInfo(
created_at=datetime(2021, 10, 1, 0, 0, 0, tzinfo=timezone.utc),
queued_at=datetime(2021, 10, 1, 0, 1, 0, tzinfo=timezone.utc),
started_at=datetime(2021, 10, 1, 1, 0, 0, tzinfo=timezone.utc),
conclusion=None,
status=JobStatus.COMPLETED,
Expand All @@ -258,6 +264,7 @@ def test_get_job_info(github_client: GithubClient, job_stats_raw: JobStatsRawDat
{},
{
"created_at": job_stats_raw.created_at,
"queued_at": job_stats_raw.queued_at,
"started_at": job_stats_raw.started_at,
"runner_name": job_stats_raw.runner_name,
"conclusion": job_stats_raw.conclusion,
Expand All @@ -269,6 +276,7 @@ def test_get_job_info(github_client: GithubClient, job_stats_raw: JobStatsRawDat
job_stats = github_client.get_job_info(path=github_repo, job_id=job_stats_raw.id)
assert job_stats == JobInfo(
created_at=datetime(2021, 10, 1, 0, 0, 0, tzinfo=timezone.utc),
queued_at=datetime(2021, 10, 1, 0, 1, 0, tzinfo=timezone.utc),
started_at=datetime(2021, 10, 1, 1, 0, 0, tzinfo=timezone.utc),
conclusion=JobConclusion.SUCCESS,
status=JobStatus.COMPLETED,
Expand Down Expand Up @@ -297,6 +305,7 @@ def test_github_api_pagination_multiple_pages(
)
assert job_stats == JobInfo(
created_at=datetime(2021, 10, 1, 0, 0, 0, tzinfo=timezone.utc),
queued_at=datetime(2021, 10, 1, 0, 1, 0, tzinfo=timezone.utc),
started_at=datetime(2021, 10, 1, 1, 0, 0, tzinfo=timezone.utc),
conclusion=JobConclusion.SUCCESS,
status=JobStatus.COMPLETED,
Expand Down
Loading