From db524e423e3426eb1ec4dbf01eeb297f903a94b8 Mon Sep 17 00:00:00 2001 From: Will Godbe Date: Wed, 5 Aug 2026 09:01:17 -0700 Subject: [PATCH 1/4] Enable Helix Job Monitor Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .config/dotnet-tools.json | 12 ++++++++++++ azure-pipelines-internal-tests.yml | 3 +++ azure-pipelines-public.yml | 1 + eng/Version.Details.props | 2 ++ eng/Version.Details.xml | 4 ++++ eng/helix.proj | 1 + 6 files changed, 23 insertions(+) create mode 100644 .config/dotnet-tools.json diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json new file mode 100644 index 00000000000..d104beb316e --- /dev/null +++ b/.config/dotnet-tools.json @@ -0,0 +1,12 @@ +{ + "version": 1, + "isRoot": true, + "tools": { + "microsoft.dotnet.helix.jobmonitor": { + "version": "11.0.0-beta.26401.101", + "commands": [ + "dotnet-helix-job-monitor" + ] + } + } +} diff --git a/azure-pipelines-internal-tests.yml b/azure-pipelines-internal-tests.yml index da4325a67e7..67b72f81415 100644 --- a/azure-pipelines-internal-tests.yml +++ b/azure-pipelines-internal-tests.yml @@ -645,6 +645,9 @@ extends: HelixAccessToken: $(_HelixAccessToken) SYSTEM_ACCESSTOKEN: $(System.AccessToken) DotNetBuildsInternalReadSasToken: $(dotnetbuilds-internal-container-read-token) + - template: /eng/common/core-templates/job/helix-job-monitor.yml@self + parameters: + helixAccessToken: $(HelixApiAccessToken) - stage: validate displayName: Validate dependsOn: build diff --git a/azure-pipelines-public.yml b/azure-pipelines-public.yml index fd68cf6c873..e1e953a7b5d 100644 --- a/azure-pipelines-public.yml +++ b/azure-pipelines-public.yml @@ -509,6 +509,7 @@ stages: env: HelixAccessToken: $(_HelixAccessToken) SYSTEM_ACCESSTOKEN: $(System.AccessToken) + - template: /eng/common/core-templates/job/helix-job-monitor.yml - stage: validate displayName: Validate dependsOn: build diff --git a/eng/Version.Details.props b/eng/Version.Details.props index 142cdd226f9..2dd071adb12 100644 --- a/eng/Version.Details.props +++ b/eng/Version.Details.props @@ -8,6 +8,7 @@ This file should be imported by eng/Versions.props 11.0.0-beta.26401.101 11.0.0-beta.26401.101 + 11.0.0-beta.26401.101 11.0.0-beta.26401.101 11.0.0-rc.1.26401.101 11.0.0-rc.1.26401.101 @@ -29,6 +30,7 @@ This file should be imported by eng/Versions.props $(MicrosoftDotNetArcadeSdkPackageVersion) $(MicrosoftDotNetBuildTasksTemplatingPackageVersion) + $(MicrosoftDotNetHelixJobMonitorPackageVersion) $(MicrosoftDotNetHelixSdkPackageVersion) $(MicrosoftExtensionsCachingMemoryPackageVersion) $(MicrosoftExtensionsConfigurationPackageVersion) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 5fa787159b9..a822ba32a54 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -68,6 +68,10 @@ https://github.com/dotnet/dotnet 5da2de22f7c6e2ebe7ed4c16c614b00183145d3c + + https://github.com/dotnet/dotnet + 5da2de22f7c6e2ebe7ed4c16c614b00183145d3c + https://github.com/dotnet/dotnet 5da2de22f7c6e2ebe7ed4c16c614b00183145d3c diff --git a/eng/helix.proj b/eng/helix.proj index 3dfca72e27d..8d341f9724e 100644 --- a/eng/helix.proj +++ b/eng/helix.proj @@ -11,6 +11,7 @@ $(NETCoreSdkVersion) true + true true $(RepoRoot)/test/EFCore.SqlServer.FunctionalTests/*.csproj;$(RepoRoot)/test/EFCore.SqlServer.HierarchyId.Tests/*.csproj;$(RepoRoot)/test/EFCore.CrossStore.FunctionalTests/*.csproj;$(RepoRoot)/test/EFCore.OData.FunctionalTests/*.csproj;$(RepoRoot)/test/EFCore.AspNet.SqlServer.FunctionalTests/*.csproj;$(RepoRoot)/test/EFCore.VisualBasic.FunctionalTests/*.vbproj;$(RepoRoot)/test/EFCore.FSharp.FunctionalTests/*.fsproj $(RepoRoot)/test/EFCore.Cosmos.FunctionalTests/*.csproj From 34bf52554862a3d21addcb1645a38b9cf0b4e63e Mon Sep 17 00:00:00 2001 From: Will Godbe Date: Wed, 5 Aug 2026 10:34:07 -0700 Subject: [PATCH 2/4] Validate Helix monitor result Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- azure-pipelines-internal-tests.yml | 10 ++++++++++ azure-pipelines-public.yml | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/azure-pipelines-internal-tests.yml b/azure-pipelines-internal-tests.yml index 67b72f81415..d09e30640eb 100644 --- a/azure-pipelines-internal-tests.yml +++ b/azure-pipelines-internal-tests.yml @@ -673,8 +673,18 @@ extends: HelixMacOsArm64Result: $[ stageDependencies.build.Helix_macOS_ARM64.result ] HelixWindowsArm64Result: $[ stageDependencies.build.Helix_Windows_Arm64.result ] HelixUbuntuCosmosResult: $[ stageDependencies.build.Helix_Ubuntu_Cosmos.result ] + HelixJobMonitorResult: $[ stageDependencies.build.HelixJobMonitor.result ] steps: - pwsh: | + $helixJobMonitorResult = "$(HelixJobMonitorResult)" + Write-Host "Helix Job Monitor result: $helixJobMonitorResult" + + if ($helixJobMonitorResult -ne 'Succeeded') + { + Write-Error "Helix Job Monitor did not succeed: $helixJobMonitorResult." + exit 1 + } + $groupResults = @{ Windows = @("$(WindowsResult)", "$(HelixWindowsResult)") Linux = @("$(LinuxResult)", "$(HelixUbuntuResult)") diff --git a/azure-pipelines-public.yml b/azure-pipelines-public.yml index e1e953a7b5d..bc7715b8c2d 100644 --- a/azure-pipelines-public.yml +++ b/azure-pipelines-public.yml @@ -534,8 +534,18 @@ stages: HelixMacOsArm64Result: $[ stageDependencies.build.Helix_macOS_ARM64.result ] HelixWindowsArm64Result: $[ stageDependencies.build.Helix_Windows_Arm64.result ] HelixUbuntuCosmosResult: $[ stageDependencies.build.Helix_Ubuntu_Cosmos.result ] + HelixJobMonitorResult: $[ stageDependencies.build.HelixJobMonitor.result ] steps: - pwsh: | + $helixJobMonitorResult = "$(HelixJobMonitorResult)" + Write-Host "Helix Job Monitor result: $helixJobMonitorResult" + + if ($helixJobMonitorResult -ne 'Succeeded') + { + Write-Error "Helix Job Monitor did not succeed: $helixJobMonitorResult." + exit 1 + } + $groupResults = @{ Windows = @("$(WindowsResult)", "$(HelixWindowsResult)") Linux = @("$(LinuxResult)", "$(HelixUbuntuResult)") From 4abe9f1e1e1c2f2a229b6bb6af5b399e17dad20c Mon Sep 17 00:00:00 2001 From: Will Godbe Date: Tue, 15 Sep 2026 15:15:57 -0700 Subject: [PATCH 3/4] Skip Helix monitor when no Helix jobs are selected Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- azure-pipelines-internal-tests.yml | 7 ++++--- azure-pipelines-public.yml | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/azure-pipelines-internal-tests.yml b/azure-pipelines-internal-tests.yml index 9763b800657..e406b52eba2 100644 --- a/azure-pipelines-internal-tests.yml +++ b/azure-pipelines-internal-tests.yml @@ -663,9 +663,10 @@ extends: HelixAccessToken: $(_HelixAccessToken) SYSTEM_ACCESSTOKEN: $(System.AccessToken) DotNetBuildsInternalReadSasToken: $(dotnetbuilds-internal-container-read-token) - - template: /eng/common/core-templates/job/helix-job-monitor.yml@self - parameters: - helixAccessToken: $(HelixApiAccessToken) + - ${{ if or(eq(parameters.jobs, '[]'), contains(parameters.jobs, '"Helix_')) }}: + - template: /eng/common/core-templates/job/helix-job-monitor.yml@self + parameters: + helixAccessToken: $(HelixApiAccessToken) - stage: validate displayName: Validate dependsOn: build diff --git a/azure-pipelines-public.yml b/azure-pipelines-public.yml index f8cacc25cd9..89078385670 100644 --- a/azure-pipelines-public.yml +++ b/azure-pipelines-public.yml @@ -527,7 +527,8 @@ stages: env: HelixAccessToken: $(_HelixAccessToken) SYSTEM_ACCESSTOKEN: $(System.AccessToken) - - template: /eng/common/core-templates/job/helix-job-monitor.yml + - ${{ if or(eq(parameters.jobs, '[]'), contains(parameters.jobs, '"Helix_')) }}: + - template: /eng/common/core-templates/job/helix-job-monitor.yml - stage: validate displayName: Validate dependsOn: build From 3677be42534efa1c1df6252a93895bf33cdf4f85 Mon Sep 17 00:00:00 2001 From: Will Godbe Date: Thu, 17 Sep 2026 15:59:55 -0700 Subject: [PATCH 4/4] Preserve grouped Helix validation Use the Job Monitor's per-queue failed-work-item attachments so grouped validation and selective retries retain their existing semantics. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- azure-pipelines-internal-tests.yml | 11 +- azure-pipelines-public.yml | 12 +- eng/aggregate-azdo-tests.ps1 | 190 +++++++++++++++++++++++++++++ 3 files changed, 193 insertions(+), 20 deletions(-) diff --git a/azure-pipelines-internal-tests.yml b/azure-pipelines-internal-tests.yml index e406b52eba2..718e5c5d3ad 100644 --- a/azure-pipelines-internal-tests.yml +++ b/azure-pipelines-internal-tests.yml @@ -667,6 +667,7 @@ extends: - template: /eng/common/core-templates/job/helix-job-monitor.yml@self parameters: helixAccessToken: $(HelixApiAccessToken) + continueOnError: true - stage: validate displayName: Validate dependsOn: build @@ -693,18 +694,8 @@ extends: HelixMacOsArm64Result: $[ stageDependencies.build.Helix_macOS_ARM64.result ] HelixWindowsArm64Result: $[ stageDependencies.build.Helix_Windows_Arm64.result ] HelixUbuntuCosmosResult: $[ stageDependencies.build.Helix_Ubuntu_Cosmos.result ] - HelixJobMonitorResult: $[ stageDependencies.build.HelixJobMonitor.result ] steps: - pwsh: | - $helixJobMonitorResult = "$(HelixJobMonitorResult)" - Write-Host "Helix Job Monitor result: $helixJobMonitorResult" - - if ($helixJobMonitorResult -ne 'Succeeded') - { - Write-Error "Helix Job Monitor did not succeed: $helixJobMonitorResult." - exit 1 - } - $jobResults = @{ Windows = "$(WindowsResult)" Helix_Windows = "$(HelixWindowsResult)" diff --git a/azure-pipelines-public.yml b/azure-pipelines-public.yml index 89078385670..d64df895e7e 100644 --- a/azure-pipelines-public.yml +++ b/azure-pipelines-public.yml @@ -529,6 +529,8 @@ stages: SYSTEM_ACCESSTOKEN: $(System.AccessToken) - ${{ if or(eq(parameters.jobs, '[]'), contains(parameters.jobs, '"Helix_')) }}: - template: /eng/common/core-templates/job/helix-job-monitor.yml + parameters: + continueOnError: true - stage: validate displayName: Validate dependsOn: build @@ -554,18 +556,8 @@ stages: HelixMacOsArm64Result: $[ stageDependencies.build.Helix_macOS_ARM64.result ] HelixWindowsArm64Result: $[ stageDependencies.build.Helix_Windows_Arm64.result ] HelixUbuntuCosmosResult: $[ stageDependencies.build.Helix_Ubuntu_Cosmos.result ] - HelixJobMonitorResult: $[ stageDependencies.build.HelixJobMonitor.result ] steps: - pwsh: | - $helixJobMonitorResult = "$(HelixJobMonitorResult)" - Write-Host "Helix Job Monitor result: $helixJobMonitorResult" - - if ($helixJobMonitorResult -ne 'Succeeded') - { - Write-Error "Helix Job Monitor did not succeed: $helixJobMonitorResult." - exit 1 - } - $jobResults = @{ Windows = "$(WindowsResult)" Helix_Windows = "$(HelixWindowsResult)" diff --git a/eng/aggregate-azdo-tests.ps1 b/eng/aggregate-azdo-tests.ps1 index 884b92f46f2..2255a670858 100644 --- a/eng/aggregate-azdo-tests.ps1 +++ b/eng/aggregate-azdo-tests.ps1 @@ -20,6 +20,21 @@ $jobDisplayNames = @{ Helix_Ubuntu_SqlServer = 'Helix Ubuntu SQL Server' Helix_Ubuntu_Cosmos = 'Helix Ubuntu Cosmos' Helix_Ubuntu = 'Helix Ubuntu' + HelixJobMonitor = 'Monitor Helix Jobs' +} + +# Test runs uploaded by the Helix Job Monitor are named after their target queue. +# Public queue names add ".Open"; normalize that suffix so the same map works for internal builds. +$helixQueueNames = @{ + Helix_Windows = 'Windows.10.Amd64' + Helix_Windows_SqlServer = 'Windows.11.Amd64.Client' + Helix_Windows_Arm64 = 'Windows.11.Arm64' + Helix_Windows_Cosmos = 'Windows.Server2025.Amd64' + Helix_macOS_x64 = 'OSX.15.Amd64' + Helix_macOS_ARM64 = 'OSX.15.ARM64' + Helix_Ubuntu_SqlServer = 'Ubuntu.2204.Amd64.XL@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-helix-sqlserver-amd64' + Helix_Ubuntu_Cosmos = 'Ubuntu.2204.Amd64.XL' + Helix_Ubuntu = 'Ubuntu.2204.Amd64' } # A group succeeds when at least one of its jobs succeeds. Jobs may participate in multiple groups. @@ -71,10 +86,173 @@ function Get-FailedGroups([hashtable]$resultsByJob) return $failed } +function Get-AzureDevOpsApiContext +{ + if ([string]::IsNullOrEmpty($env:SYSTEM_ACCESSTOKEN)) + { + throw 'SYSTEM_ACCESSTOKEN is required to get Helix test results.' + } + + if ([string]::IsNullOrEmpty($env:SYSTEM_COLLECTIONURI) -or + [string]::IsNullOrEmpty($env:SYSTEM_TEAMPROJECT)) + { + throw 'SYSTEM_COLLECTIONURI and SYSTEM_TEAMPROJECT are required to get Helix test results.' + } + + $project = [Uri]::EscapeDataString($env:SYSTEM_TEAMPROJECT) + @{ + ApiBaseUri = "$($env:SYSTEM_COLLECTIONURI.TrimEnd('/'))/$project/_apis" + Headers = @{ Authorization = "Bearer $env:SYSTEM_ACCESSTOKEN" } + } +} + +function Get-NormalizedHelixQueueName([string]$queueName) +{ + return $queueName -replace '\.Open(?=@|$)', '' +} + +function Invoke-AzureDevOpsRestMethod([string]$uri, [hashtable]$headers) +{ + $maxAttempts = 3 + + for ($attempt = 1; $attempt -le $maxAttempts; $attempt++) + { + try + { + return Invoke-RestMethod ` + -Uri $uri ` + -Headers $headers ` + -ErrorAction Stop + } + catch + { + $statusCode = 0 + if ($null -ne $_.Exception.Response) + { + $statusCode = [int]$_.Exception.Response.StatusCode + } + + if ($attempt -eq $maxAttempts -or $statusCode -notin @(408, 429, 500, 502, 503, 504)) + { + throw + } + + $delay = [Math]::Pow(2, $attempt) + Write-Warning "Azure DevOps request failed with HTTP $statusCode. Retrying in $delay seconds." + Start-Sleep -Seconds $delay + } + } +} + +function Get-AzureDevOpsTestRuns([int]$buildId, [hashtable]$apiContext) +{ + $buildUri = [Uri]::EscapeDataString("vstfs:///Build/Build/$buildId") + $pageSize = 1000 + $skip = 0 + $testRuns = @() + + do + { + $uri = "$($apiContext.ApiBaseUri)/test/runs?buildUri=$buildUri&%24top=$pageSize&%24skip=$skip&api-version=7.1" + $response = Invoke-AzureDevOpsRestMethod $uri $apiContext.Headers + $page = @($response.value) + $testRuns += $page + $skip += $page.Count + } + while ($page.Count -eq $pageSize) + + return $testRuns +} + +function Set-HelixJobResults( + [hashtable]$resultsByJob, + [string[]]$jobNames, + [int]$buildId) +{ + $helixJobNames = @($jobNames | Where-Object { $_ -like 'Helix_*' }) + if ($helixJobNames.Count -eq 0) + { + return + } + + foreach ($jobName in $helixJobNames) + { + if (-not $resultsByJob.ContainsKey($jobName)) + { + throw "Missing result for Helix job '$jobName'." + } + + if (-not $helixQueueNames.ContainsKey($jobName)) + { + throw "Missing Helix queue mapping for job '$jobName'." + } + } + + $apiContext = Get-AzureDevOpsApiContext + $testRuns = @(Get-AzureDevOpsTestRuns $buildId $apiContext) + + foreach ($jobName in $helixJobNames) + { + if ($resultsByJob[$jobName] -eq 'Skipped') + { + continue + } + + if ($resultsByJob[$jobName] -notin @('Succeeded', 'SucceededWithIssues')) + { + Write-Warning "Helix submission job '$jobName' did not succeed: $($resultsByJob[$jobName])." + $resultsByJob[$jobName] = 'Failed' + continue + } + + $queueName = $helixQueueNames[$jobName] + # Job Monitor retries create a new run with the same queue name containing only the retried work items. + $run = @($testRuns + | Where-Object { (Get-NormalizedHelixQueueName $_.name) -eq $queueName } + | Sort-Object id -Descending)[0] + + if ($null -eq $run) + { + Write-Warning "No completed Helix test run was found for job '$jobName' and queue '$queueName'." + $resultsByJob[$jobName] = 'Failed' + continue + } + + if ($run.state -ne 'Completed') + { + throw "Helix test run $($run.id) for job '$jobName' did not complete; its state is '$($run.state)'." + } + + $attachmentsResponse = Invoke-AzureDevOpsRestMethod ` + "$($apiContext.ApiBaseUri)/test/runs/$($run.id)/attachments?api-version=7.1" ` + $apiContext.Headers + $failedWorkItemsAttachment = @($attachmentsResponse.value + | Where-Object { $_.fileName -eq 'helix-failed-workitems.json' }) + + if ($failedWorkItemsAttachment.Count -gt 0) + { + $resultsByJob[$jobName] = 'Failed' + } + else + { + $resultsByJob[$jobName] = 'Succeeded' + } + + Write-Host " $jobName ($($run.name)): $($resultsByJob[$jobName])" + } +} + $jobAttempt = 1 [void][int]::TryParse($env:SYSTEM_JOBATTEMPT, [ref]$jobAttempt) $stageAttempt = 1 [void][int]::TryParse($env:SYSTEM_STAGEATTEMPT, [ref]$stageAttempt) +$buildId = 0 +if (-not [int]::TryParse($env:BUILD_BUILDID, [ref]$buildId) -or $buildId -le 0) +{ + throw "BUILD_BUILDID must contain a valid build ID; received '$env:BUILD_BUILDID'." +} + +Set-HelixJobResults $JobResults @($helixQueueNames.Keys) $buildId $failedGroups = @(Get-FailedGroups $JobResults) # Retrying validation queues a child build containing the distinct jobs needed by all failed groups. @@ -146,6 +324,18 @@ if (($jobAttempt -gt 1 -or $stageAttempt -gt 1) -and $failedGroups.Count -gt 0) $JobResults[$jobName] = $record.result } + $helixJobsToRetry = @($jobsToRetry | Where-Object { $_ -like 'Helix_*' }) + if ($helixJobsToRetry.Count -gt 0) + { + $monitorRecord = Get-JobRecord $timeline 'HelixJobMonitor' + if ($null -eq $monitorRecord) + { + throw 'Could not find the Helix Job Monitor timeline record for the retry build.' + } + + Set-HelixJobResults $JobResults $helixJobsToRetry $retryBuild.id + } + $failedGroups = @(Get-FailedGroups $JobResults) }