diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json new file mode 100644 index 00000000000..6f804df1b4c --- /dev/null +++ b/.config/dotnet-tools.json @@ -0,0 +1,12 @@ +{ + "version": 1, + "isRoot": true, + "tools": { + "microsoft.dotnet.helix.jobmonitor": { + "version": "12.0.0-beta.26461.114", + "commands": [ + "dotnet-helix-job-monitor" + ] + } + } +} diff --git a/azure-pipelines-internal-tests.yml b/azure-pipelines-internal-tests.yml index 3256c547d2e..718e5c5d3ad 100644 --- a/azure-pipelines-internal-tests.yml +++ b/azure-pipelines-internal-tests.yml @@ -663,6 +663,11 @@ extends: HelixAccessToken: $(_HelixAccessToken) SYSTEM_ACCESSTOKEN: $(System.AccessToken) DotNetBuildsInternalReadSasToken: $(dotnetbuilds-internal-container-read-token) + - ${{ if or(eq(parameters.jobs, '[]'), contains(parameters.jobs, '"Helix_')) }}: + - template: /eng/common/core-templates/job/helix-job-monitor.yml@self + parameters: + helixAccessToken: $(HelixApiAccessToken) + continueOnError: true - stage: validate displayName: Validate dependsOn: build diff --git a/azure-pipelines-public.yml b/azure-pipelines-public.yml index 232ad0555a6..d64df895e7e 100644 --- a/azure-pipelines-public.yml +++ b/azure-pipelines-public.yml @@ -527,6 +527,10 @@ stages: env: HelixAccessToken: $(_HelixAccessToken) SYSTEM_ACCESSTOKEN: $(System.AccessToken) + - ${{ if or(eq(parameters.jobs, '[]'), contains(parameters.jobs, '"Helix_')) }}: + - template: /eng/common/core-templates/job/helix-job-monitor.yml + parameters: + continueOnError: true - stage: validate displayName: Validate dependsOn: build diff --git a/eng/Version.Details.props b/eng/Version.Details.props index 7d8c8b9db66..21cbd6f4497 100644 --- a/eng/Version.Details.props +++ b/eng/Version.Details.props @@ -8,6 +8,7 @@ This file should be imported by eng/Versions.props 12.0.0-beta.26461.114 12.0.0-beta.26461.114 + 12.0.0-beta.26461.114 12.0.0-beta.26461.114 12.0.0-alpha.1.26461.114 12.0.0-alpha.1.26461.114 @@ -29,6 +30,7 @@ This file should be imported by eng/Versions.props $(MicrosoftDotNetArcadeSdkPackageVersion) $(MicrosoftDotNetBuildTasksTemplatingPackageVersion) + $(MicrosoftDotNetHelixJobMonitorPackageVersion) $(MicrosoftDotNetHelixSdkPackageVersion) $(MicrosoftExtensionsCachingMemoryPackageVersion) $(MicrosoftExtensionsConfigurationPackageVersion) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 385ef522b98..54d89139033 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -68,6 +68,10 @@ https://github.com/dotnet/dotnet 7cfa7a8277e6c63b36482018162a9c2dd8922a96 + + https://github.com/dotnet/dotnet + 7cfa7a8277e6c63b36482018162a9c2dd8922a96 + https://github.com/dotnet/dotnet 7cfa7a8277e6c63b36482018162a9c2dd8922a96 diff --git a/eng/aggregate-azdo-tests.ps1 b/eng/aggregate-azdo-tests.ps1 index 884b92f46f2..2255a670858 100644 --- a/eng/aggregate-azdo-tests.ps1 +++ b/eng/aggregate-azdo-tests.ps1 @@ -20,6 +20,21 @@ $jobDisplayNames = @{ Helix_Ubuntu_SqlServer = 'Helix Ubuntu SQL Server' Helix_Ubuntu_Cosmos = 'Helix Ubuntu Cosmos' Helix_Ubuntu = 'Helix Ubuntu' + HelixJobMonitor = 'Monitor Helix Jobs' +} + +# Test runs uploaded by the Helix Job Monitor are named after their target queue. +# Public queue names add ".Open"; normalize that suffix so the same map works for internal builds. +$helixQueueNames = @{ + Helix_Windows = 'Windows.10.Amd64' + Helix_Windows_SqlServer = 'Windows.11.Amd64.Client' + Helix_Windows_Arm64 = 'Windows.11.Arm64' + Helix_Windows_Cosmos = 'Windows.Server2025.Amd64' + Helix_macOS_x64 = 'OSX.15.Amd64' + Helix_macOS_ARM64 = 'OSX.15.ARM64' + Helix_Ubuntu_SqlServer = 'Ubuntu.2204.Amd64.XL@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-helix-sqlserver-amd64' + Helix_Ubuntu_Cosmos = 'Ubuntu.2204.Amd64.XL' + Helix_Ubuntu = 'Ubuntu.2204.Amd64' } # A group succeeds when at least one of its jobs succeeds. Jobs may participate in multiple groups. @@ -71,10 +86,173 @@ function Get-FailedGroups([hashtable]$resultsByJob) return $failed } +function Get-AzureDevOpsApiContext +{ + if ([string]::IsNullOrEmpty($env:SYSTEM_ACCESSTOKEN)) + { + throw 'SYSTEM_ACCESSTOKEN is required to get Helix test results.' + } + + if ([string]::IsNullOrEmpty($env:SYSTEM_COLLECTIONURI) -or + [string]::IsNullOrEmpty($env:SYSTEM_TEAMPROJECT)) + { + throw 'SYSTEM_COLLECTIONURI and SYSTEM_TEAMPROJECT are required to get Helix test results.' + } + + $project = [Uri]::EscapeDataString($env:SYSTEM_TEAMPROJECT) + @{ + ApiBaseUri = "$($env:SYSTEM_COLLECTIONURI.TrimEnd('/'))/$project/_apis" + Headers = @{ Authorization = "Bearer $env:SYSTEM_ACCESSTOKEN" } + } +} + +function Get-NormalizedHelixQueueName([string]$queueName) +{ + return $queueName -replace '\.Open(?=@|$)', '' +} + +function Invoke-AzureDevOpsRestMethod([string]$uri, [hashtable]$headers) +{ + $maxAttempts = 3 + + for ($attempt = 1; $attempt -le $maxAttempts; $attempt++) + { + try + { + return Invoke-RestMethod ` + -Uri $uri ` + -Headers $headers ` + -ErrorAction Stop + } + catch + { + $statusCode = 0 + if ($null -ne $_.Exception.Response) + { + $statusCode = [int]$_.Exception.Response.StatusCode + } + + if ($attempt -eq $maxAttempts -or $statusCode -notin @(408, 429, 500, 502, 503, 504)) + { + throw + } + + $delay = [Math]::Pow(2, $attempt) + Write-Warning "Azure DevOps request failed with HTTP $statusCode. Retrying in $delay seconds." + Start-Sleep -Seconds $delay + } + } +} + +function Get-AzureDevOpsTestRuns([int]$buildId, [hashtable]$apiContext) +{ + $buildUri = [Uri]::EscapeDataString("vstfs:///Build/Build/$buildId") + $pageSize = 1000 + $skip = 0 + $testRuns = @() + + do + { + $uri = "$($apiContext.ApiBaseUri)/test/runs?buildUri=$buildUri&%24top=$pageSize&%24skip=$skip&api-version=7.1" + $response = Invoke-AzureDevOpsRestMethod $uri $apiContext.Headers + $page = @($response.value) + $testRuns += $page + $skip += $page.Count + } + while ($page.Count -eq $pageSize) + + return $testRuns +} + +function Set-HelixJobResults( + [hashtable]$resultsByJob, + [string[]]$jobNames, + [int]$buildId) +{ + $helixJobNames = @($jobNames | Where-Object { $_ -like 'Helix_*' }) + if ($helixJobNames.Count -eq 0) + { + return + } + + foreach ($jobName in $helixJobNames) + { + if (-not $resultsByJob.ContainsKey($jobName)) + { + throw "Missing result for Helix job '$jobName'." + } + + if (-not $helixQueueNames.ContainsKey($jobName)) + { + throw "Missing Helix queue mapping for job '$jobName'." + } + } + + $apiContext = Get-AzureDevOpsApiContext + $testRuns = @(Get-AzureDevOpsTestRuns $buildId $apiContext) + + foreach ($jobName in $helixJobNames) + { + if ($resultsByJob[$jobName] -eq 'Skipped') + { + continue + } + + if ($resultsByJob[$jobName] -notin @('Succeeded', 'SucceededWithIssues')) + { + Write-Warning "Helix submission job '$jobName' did not succeed: $($resultsByJob[$jobName])." + $resultsByJob[$jobName] = 'Failed' + continue + } + + $queueName = $helixQueueNames[$jobName] + # Job Monitor retries create a new run with the same queue name containing only the retried work items. + $run = @($testRuns + | Where-Object { (Get-NormalizedHelixQueueName $_.name) -eq $queueName } + | Sort-Object id -Descending)[0] + + if ($null -eq $run) + { + Write-Warning "No completed Helix test run was found for job '$jobName' and queue '$queueName'." + $resultsByJob[$jobName] = 'Failed' + continue + } + + if ($run.state -ne 'Completed') + { + throw "Helix test run $($run.id) for job '$jobName' did not complete; its state is '$($run.state)'." + } + + $attachmentsResponse = Invoke-AzureDevOpsRestMethod ` + "$($apiContext.ApiBaseUri)/test/runs/$($run.id)/attachments?api-version=7.1" ` + $apiContext.Headers + $failedWorkItemsAttachment = @($attachmentsResponse.value + | Where-Object { $_.fileName -eq 'helix-failed-workitems.json' }) + + if ($failedWorkItemsAttachment.Count -gt 0) + { + $resultsByJob[$jobName] = 'Failed' + } + else + { + $resultsByJob[$jobName] = 'Succeeded' + } + + Write-Host " $jobName ($($run.name)): $($resultsByJob[$jobName])" + } +} + $jobAttempt = 1 [void][int]::TryParse($env:SYSTEM_JOBATTEMPT, [ref]$jobAttempt) $stageAttempt = 1 [void][int]::TryParse($env:SYSTEM_STAGEATTEMPT, [ref]$stageAttempt) +$buildId = 0 +if (-not [int]::TryParse($env:BUILD_BUILDID, [ref]$buildId) -or $buildId -le 0) +{ + throw "BUILD_BUILDID must contain a valid build ID; received '$env:BUILD_BUILDID'." +} + +Set-HelixJobResults $JobResults @($helixQueueNames.Keys) $buildId $failedGroups = @(Get-FailedGroups $JobResults) # Retrying validation queues a child build containing the distinct jobs needed by all failed groups. @@ -146,6 +324,18 @@ if (($jobAttempt -gt 1 -or $stageAttempt -gt 1) -and $failedGroups.Count -gt 0) $JobResults[$jobName] = $record.result } + $helixJobsToRetry = @($jobsToRetry | Where-Object { $_ -like 'Helix_*' }) + if ($helixJobsToRetry.Count -gt 0) + { + $monitorRecord = Get-JobRecord $timeline 'HelixJobMonitor' + if ($null -eq $monitorRecord) + { + throw 'Could not find the Helix Job Monitor timeline record for the retry build.' + } + + Set-HelixJobResults $JobResults $helixJobsToRetry $retryBuild.id + } + $failedGroups = @(Get-FailedGroups $JobResults) } diff --git a/eng/helix.proj b/eng/helix.proj index 3dfca72e27d..8d341f9724e 100644 --- a/eng/helix.proj +++ b/eng/helix.proj @@ -11,6 +11,7 @@ $(NETCoreSdkVersion) true + true true $(RepoRoot)/test/EFCore.SqlServer.FunctionalTests/*.csproj;$(RepoRoot)/test/EFCore.SqlServer.HierarchyId.Tests/*.csproj;$(RepoRoot)/test/EFCore.CrossStore.FunctionalTests/*.csproj;$(RepoRoot)/test/EFCore.OData.FunctionalTests/*.csproj;$(RepoRoot)/test/EFCore.AspNet.SqlServer.FunctionalTests/*.csproj;$(RepoRoot)/test/EFCore.VisualBasic.FunctionalTests/*.vbproj;$(RepoRoot)/test/EFCore.FSharp.FunctionalTests/*.fsproj $(RepoRoot)/test/EFCore.Cosmos.FunctionalTests/*.csproj