From d93ea451422b9e6082faeb58036a94253e50fff9 Mon Sep 17 00:00:00 2001 From: Simon Rozsival Date: Tue, 8 Sep 2026 10:53:44 +0200 Subject: [PATCH 1/4] [flaky-ci] Bound hosted dependency acquisition Fall back to verified source downloads when the Azure archive cache is unavailable, and bound workload-pack acquisition with Retry-After-aware NuGet limits, clean retry state, and a per-attempt timeout. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../yaml-templates/cache-android-archives.yaml | 3 +++ .../yaml-templates/setup-test-environment-steps.yaml | 9 ++++++++- build-tools/create-packs/Directory.Build.targets | 5 +++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/build-tools/automation/yaml-templates/cache-android-archives.yaml b/build-tools/automation/yaml-templates/cache-android-archives.yaml index 5ca9729e2db..e2f4c68f92f 100644 --- a/build-tools/automation/yaml-templates/cache-android-archives.yaml +++ b/build-tools/automation/yaml-templates/cache-android-archives.yaml @@ -38,6 +38,9 @@ steps: - task: Cache@2 displayName: cache Android toolchain archives condition: ${{ parameters.condition }} + # The cache service is an optimization. If it is unavailable or rate-limited, + # continue with the SHA-verified source downloads instead of failing the job. + continueOnError: true inputs: key: '"android-archives" | "v1" | "$(Agent.OS)" | ${{ parameters.xaSourcePath }}/Configuration.props | ${{ parameters.xaSourcePath }}/src/androidsdk/androidsdk.targets | ${{ parameters.xaSourcePath }}/src/openjdk/openjdk.targets | ${{ parameters.xaSourcePath }}/src/binutils/binutils.targets | ${{ parameters.xaSourcePath }}/src/aapt2/aapt2.targets | ${{ parameters.xaSourcePath }}/src/bundletool/bundletool.targets' restoreKeys: | diff --git a/build-tools/automation/yaml-templates/setup-test-environment-steps.yaml b/build-tools/automation/yaml-templates/setup-test-environment-steps.yaml index b0afbc6556c..55de20b2f33 100644 --- a/build-tools/automation/yaml-templates/setup-test-environment-steps.yaml +++ b/build-tools/automation/yaml-templates/setup-test-environment-steps.yaml @@ -110,7 +110,14 @@ steps: project: ${{ parameters.xaSourcePath }}/build-tools/create-packs/Microsoft.Android.Sdk.proj arguments: -t:ExtractWorkloadPacks -c ${{ parameters.configuration }} -v:n -bl:${{ parameters.xaSourcePath }}/bin/Test${{ parameters.configuration }}/extract-workloads.binlog continueOnError: false - retryCountOnTaskFailure: 2 + retryCountOnTaskFailure: 1 + taskTimeoutInMinutes: 20 + env: + NUGET_RETRY_HTTP_429: 'true' + NUGET_OBSERVE_RETRY_AFTER: 'true' + NUGET_MAX_RETRY_AFTER_DELAY_SECONDS: '60' + NUGET_ENHANCED_MAX_NETWORK_TRY_COUNT: '4' + NUGET_ENHANCED_NETWORK_RETRY_DELAY_MILLISECONDS: '2000' - ${{ if eq(parameters.installApkDiff, true) }}: - template: /build-tools/automation/yaml-templates/install-dotnet-tool.yaml diff --git a/build-tools/create-packs/Directory.Build.targets b/build-tools/create-packs/Directory.Build.targets index f7f03c0076a..2d12c397f65 100644 --- a/build-tools/create-packs/Directory.Build.targets +++ b/build-tools/create-packs/Directory.Build.targets @@ -90,6 +90,10 @@ + + <_StaleWorkloadTempDirectories Include="$([System.IO.Directory]::GetDirectories('$(IntermediateOutputPath)', '.xa-workload-temp-*'))" /> + + <_WLManifest Include="$(XamarinAndroidSourcePath)bin\Build$(Configuration)\nuget-unsigned\Microsoft.NET.Sdk.Android.Manifest-*.nupkg" /> @@ -142,6 +146,7 @@ <_InstallArguments Include="--skip-manifest-update" /> <_InstallArguments Include="--skip-sign-check" /> <_InstallArguments Include="--verbosity diag" /> + <_InstallArguments Include="--no-http-cache" /> <_InstallArguments Include="--source "%(_NuGetSources.Identity)"" /> <_InstallArguments Include="--temp-dir "$(_TempDirectory)"" /> From 7c73e526c2ee868da5a50af3d46ae0e7882b5e9a Mon Sep 17 00:00:00 2001 From: Simon Rozsival Date: Tue, 8 Sep 2026 13:51:36 +0200 Subject: [PATCH 2/4] Harden hosted dependency recovery Classify only dynamically recovered optional cache failures from a complete Azure timeline, and bound workload installation retries inside the child process. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../automation/azure-pipelines-internal.yaml | 2 + .../automation/azure-pipelines-public.yaml | 2 + build-tools/automation/azure-pipelines.yaml | 2 + .../scripts/FailOnUnexpectedIssues.Tests.ps1 | 363 ++++++++++++++++++ .../scripts/FailOnUnexpectedIssues.ps1 | 232 +++++++++++ .../scripts/Invoke-ProcessWithRetry.Tests.ps1 | 52 +++ .../scripts/Invoke-ProcessWithRetry.ps1 | 83 ++++ .../RecoverAndroidArchiveCache.Tests.ps1 | 53 +++ .../scripts/RecoverAndroidArchiveCache.ps1 | 38 ++ .../cache-android-archives.yaml | 7 + .../yaml-templates/fail-on-issue.yaml | 9 +- .../yaml-templates/run-dotnet-preview.yaml | 18 +- .../setup-test-environment-steps.yaml | 6 +- 13 files changed, 860 insertions(+), 7 deletions(-) create mode 100644 build-tools/automation/scripts/FailOnUnexpectedIssues.Tests.ps1 create mode 100644 build-tools/automation/scripts/FailOnUnexpectedIssues.ps1 create mode 100644 build-tools/automation/scripts/Invoke-ProcessWithRetry.Tests.ps1 create mode 100644 build-tools/automation/scripts/Invoke-ProcessWithRetry.ps1 create mode 100644 build-tools/automation/scripts/RecoverAndroidArchiveCache.Tests.ps1 create mode 100644 build-tools/automation/scripts/RecoverAndroidArchiveCache.ps1 diff --git a/build-tools/automation/azure-pipelines-internal.yaml b/build-tools/automation/azure-pipelines-internal.yaml index 256e9e7f57a..1c37bd62976 100644 --- a/build-tools/automation/azure-pipelines-internal.yaml +++ b/build-tools/automation/azure-pipelines-internal.yaml @@ -1014,3 +1014,5 @@ extends: use1ESTemplate: true - template: /build-tools/automation/yaml-templates/fail-on-issue.yaml@self + parameters: + xaSourcePath: $(Build.SourcesDirectory)/android diff --git a/build-tools/automation/azure-pipelines-public.yaml b/build-tools/automation/azure-pipelines-public.yaml index 52c6c79ea1f..42ddad9d424 100644 --- a/build-tools/automation/azure-pipelines-public.yaml +++ b/build-tools/automation/azure-pipelines-public.yaml @@ -663,3 +663,5 @@ stages: use1ESTemplate: false - template: /build-tools/automation/yaml-templates/fail-on-issue.yaml + parameters: + xaSourcePath: $(Build.SourcesDirectory)/android diff --git a/build-tools/automation/azure-pipelines.yaml b/build-tools/automation/azure-pipelines.yaml index e2c14fc0b03..be11cf2a359 100644 --- a/build-tools/automation/azure-pipelines.yaml +++ b/build-tools/automation/azure-pipelines.yaml @@ -280,6 +280,8 @@ extends: condition: or(ne(variables['Agent.JobStatus'], 'Succeeded'), eq(variables['XA.PublishAllLogs'], 'true')) - template: /build-tools/automation/yaml-templates/fail-on-issue.yaml@self + parameters: + xaSourcePath: $(Build.SourcesDirectory)/android - stage: dotnet_prepare_release diff --git a/build-tools/automation/scripts/FailOnUnexpectedIssues.Tests.ps1 b/build-tools/automation/scripts/FailOnUnexpectedIssues.Tests.ps1 new file mode 100644 index 00000000000..6591003cff3 --- /dev/null +++ b/build-tools/automation/scripts/FailOnUnexpectedIssues.Tests.ps1 @@ -0,0 +1,363 @@ +$ErrorActionPreference = 'Stop' +$gateScript = Join-Path $PSScriptRoot 'FailOnUnexpectedIssues.ps1' +$tempRoot = Join-Path ([IO.Path]::GetTempPath()) "xa-job-issue-tests-$([Guid]::NewGuid())" +$jobId = 'test-job' + +function Assert-True ([bool] $condition, [string] $message) { + if (-not $condition) { + throw $message + } +} + +function Invoke-Gate ([object[]] $records, [string] $recoveredTasks, [string] $jobStatus = 'SucceededWithIssues') { + $timelinePath = Join-Path $tempRoot "$([Guid]::NewGuid()).json" + New-Timeline $records | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath $timelinePath -Encoding ASCII + return & $gateScript ` + -JobStatus $jobStatus ` + -RecoveredOptionalTaskRefs $recoveredTasks ` + -JobId $jobId ` + -TimelinePath $timelinePath ` + -TimelinePollTimeoutSeconds 1 6>&1 +} + +function New-Task ([string] $refName, [string] $result, [string] $parentId = $jobId) { + return @{ + name = $refName + parentId = $parentId + refName = $refName + result = $result + state = 'completed' + type = 'Task' + } +} + +function New-Timeline ([object[]] $records) { + $timelineRecords = [System.Collections.Generic.List[object]]::new() + $nextOrder = 1 + foreach ($record in $records) { + if ($record.parentId -eq $jobId) { + if ($null -eq $record.order) { + $record.order = $nextOrder + } + $nextOrder = [Math]::Max($nextOrder, [int] $record.order + 1) + } + $timelineRecords.Add($record) + } + $timelineRecords.Add(@{ + name = 'fail if any issues occurred' + order = $nextOrder + parentId = $jobId + refName = 'failOnUnexpectedIssues' + result = $null + state = 'inProgress' + type = 'Task' + }) + return @{ + records = $timelineRecords.ToArray() + } +} + +function Invoke-GateSequence ([object[]] $timelines, [string] $recoveredTasks) { + $timelinePaths = [System.Collections.Generic.List[string]]::new() + foreach ($timeline in $timelines) { + $timelinePath = Join-Path $tempRoot "$([Guid]::NewGuid()).json" + $timeline | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath $timelinePath -Encoding ASCII + $timelinePaths.Add($timelinePath) + } + return & $gateScript ` + -JobStatus 'SucceededWithIssues' ` + -RecoveredOptionalTaskRefs $recoveredTasks ` + -JobId $jobId ` + -TimelinePath $timelinePaths.ToArray() ` + -TimelinePollTimeoutSeconds $timelines.Count ` + -TimelinePollIntervalSeconds 0 6>&1 +} + +function Get-FreeTcpPort { + $listener = [Net.Sockets.TcpListener]::new([Net.IPAddress]::Loopback, 0) + try { + $listener.Start() + return ([Net.IPEndPoint] $listener.LocalEndpoint).Port + } finally { + $listener.Stop() + } +} + +function Start-TestHttpServer ([string] $responseBody, [int] $bodyDelayMilliseconds = 0) { + $port = Get-FreeTcpPort + $readyPath = Join-Path $tempRoot "$([Guid]::NewGuid()).ready" + $requestPath = Join-Path $tempRoot "$([Guid]::NewGuid()).request" + $job = Start-Job -ScriptBlock { + param ($port, $readyPath, $requestPath, $responseBody, $bodyDelayMilliseconds) + $ErrorActionPreference = 'Stop' + $listener = [Net.Sockets.TcpListener]::new([Net.IPAddress]::Loopback, $port) + try { + $listener.Start() + Set-Content -LiteralPath $readyPath -Value 'ready' -Encoding ASCII + $client = $listener.AcceptTcpClient() + try { + $stream = $client.GetStream() + $reader = [IO.StreamReader]::new($stream, [Text.Encoding]::ASCII, $false, 1024, $true) + $requestLines = [System.Collections.Generic.List[string]]::new() + while ($true) { + $line = $reader.ReadLine() + if ([string]::IsNullOrEmpty($line)) { + break + } + $requestLines.Add($line) + } + $requestLines | Set-Content -LiteralPath $requestPath -Encoding ASCII + + $bodyBytes = [Text.Encoding]::UTF8.GetBytes($responseBody) + $headers = "HTTP/1.1 200 OK`r`nContent-Type: application/json`r`nContent-Length: $($bodyBytes.Length)`r`nConnection: close`r`n`r`n" + $headerBytes = [Text.Encoding]::ASCII.GetBytes($headers) + $stream.Write($headerBytes, 0, $headerBytes.Length) + $stream.Flush() + if ($bodyDelayMilliseconds -gt 0) { + Start-Sleep -Milliseconds $bodyDelayMilliseconds + } + $stream.Write($bodyBytes, 0, $bodyBytes.Length) + $stream.Flush() + } finally { + if ($null -ne $reader) { + $reader.Dispose() + } + if ($null -ne $client) { + $client.Dispose() + } + } + } finally { + $listener.Stop() + } + } -ArgumentList $port, $readyPath, $requestPath, $responseBody, $bodyDelayMilliseconds + + $readyDeadline = [DateTime]::UtcNow.AddSeconds(10) + while (-not (Test-Path -LiteralPath $readyPath) -and [DateTime]::UtcNow -lt $readyDeadline) { + if ($job.State -in @('Failed', 'Stopped', 'Completed')) { + break + } + Start-Sleep -Milliseconds 50 + } + if (-not (Test-Path -LiteralPath $readyPath)) { + $jobOutput = Receive-Job -Job $job 2>&1 + Remove-Job -Job $job -Force + throw "The test HTTP server did not start: $($jobOutput -join "`n")" + } + + return @{ + Job = $job + Port = $port + RequestPath = $requestPath + } +} + +function Stop-TestHttpServer ($server) { + $serverFailed = $server.Job.State -eq 'Failed' + if ($server.Job.State -notin @('Completed', 'Failed', 'Stopped')) { + Stop-Job -Job $server.Job + } + $serverOutput = Receive-Job -Job $server.Job 2>&1 + Remove-Job -Job $server.Job -Force + if ($serverFailed) { + throw "The test HTTP server failed: $($serverOutput -join "`n")" + } +} + +try { + New-Item -ItemType Directory -Path $tempRoot -Force | Out-Null + + $output = Invoke-Gate @( + (New-Task 'androidArchivesCache' 'failed') + ) 'androidArchivesCache' + Assert-True (-not ($output -join "`n").Contains('result=Failed')) 'A recovered optional cache failure remained gating.' + + $timelineJson = New-Timeline @( + (New-Task 'androidArchivesCache' 'failed') + ) | ConvertTo-Json -Depth 5 -Compress + $server = Start-TestHttpServer $timelineJson + try { + $env:SYSTEM_ACCESSTOKEN = 'test-token' + $env:SYSTEM_COLLECTIONURI = "http://127.0.0.1:$($server.Port)/" + $env:SYSTEM_TEAMPROJECT = 'project' + $env:BUILD_BUILDID = '42' + $env:SYSTEM_JOBID = $jobId + $output = & $gateScript ` + -JobStatus 'SucceededWithIssues' ` + -RecoveredOptionalTaskRefs 'androidArchivesCache' ` + -TimelinePollTimeoutSeconds 5 6>&1 + $requestLines = Get-Content -LiteralPath $server.RequestPath + } finally { + Stop-TestHttpServer $server + } + Assert-True (-not ($output -join "`n").Contains('result=Failed')) 'A recovered cache failure from the timeline API remained gating.' + Assert-True ($requestLines[0] -eq 'GET /project/_apis/build/builds/42/timeline?api-version=7.1 HTTP/1.1') 'The timeline API URI was incorrect.' + Assert-True ($requestLines -contains 'Authorization: Bearer test-token') 'The timeline API authorization header was incorrect.' + Remove-Item Env:SYSTEM_ACCESSTOKEN, Env:SYSTEM_JOBID + + $output = Invoke-Gate @( + @{ + name = 'androidArchivesCache' + parentId = $jobId + refName = $null + result = 'failed' + state = 'completed' + type = 'Task' + } + ) 'androidArchivesCache' + Assert-True (-not ($output -join "`n").Contains('result=Failed')) 'A recovered task without a timeline refName remained gating.' + + $output = Invoke-Gate @( + (New-Task 'androidArchivesCache' 'failed'), + (New-Task 'realFailure' 'succeededWithIssues') + ) 'androidArchivesCache' + Assert-True (($output -join "`n").Contains('result=Failed')) 'A later genuine issue was masked by cache recovery.' + Assert-True (($output -join "`n").Contains('realFailure')) 'The unexpected issue task was not diagnosed.' + + $output = Invoke-Gate @( + (New-Task 'androidArchivesCache' 'failed'), + (New-Task 'unknownResult' 'futureResult') + ) 'androidArchivesCache' + Assert-True (($output -join "`n").Contains('result=Failed')) 'An unknown completed task result did not fail closed.' + + $output = Invoke-GateSequence @( + (New-Timeline @( + (New-Task 'androidArchivesCache' 'failed'), + @{ + name = 'realFailure' + parentId = $jobId + refName = 'realFailure' + result = $null + state = 'inProgress' + type = 'Task' + } + )), + (New-Timeline @( + (New-Task 'androidArchivesCache' 'failed'), + (New-Task 'realFailure' 'succeededWithIssues') + )) + ) 'androidArchivesCache' + Assert-True (($output -join "`n").Contains('result=Failed')) 'An issue published after the first timeline read was masked.' + Assert-True (($output -join "`n").Contains('realFailure')) 'The later published issue was not diagnosed.' + + $output = Invoke-GateSequence @( + (New-Timeline @( + (New-Task 'androidArchivesCache' 'failed'), + @{ + name = 'eventualSuccess' + parentId = $jobId + refName = 'eventualSuccess' + result = $null + state = 'inProgress' + type = 'Task' + } + )), + (New-Timeline @( + (New-Task 'androidArchivesCache' 'failed'), + (New-Task 'eventualSuccess' 'succeeded') + )) + ) 'androidArchivesCache' + Assert-True (-not ($output -join "`n").Contains('result=Failed')) 'A preceding task that became successful remained gating.' + + $cacheTask = New-Task 'androidArchivesCache' 'failed' + $cacheTask.order = 1 + $output = Invoke-GateSequence @( + @{ + records = @($cacheTask) + }, + (New-Timeline @( + (New-Task 'androidArchivesCache' 'failed') + )) + ) 'androidArchivesCache' + Assert-True (-not ($output -join "`n").Contains('result=Failed')) 'The gate did not wait for its own timeline record to become visible.' + + $incompleteTimeline = New-Timeline @( + (New-Task 'androidArchivesCache' 'failed'), + @{ + name = 'neverCompletes' + parentId = $jobId + refName = 'neverCompletes' + result = $null + state = 'inProgress' + type = 'Task' + } + ) + $output = Invoke-GateSequence @( + $incompleteTimeline, + $incompleteTimeline, + $incompleteTimeline + ) 'androidArchivesCache' + Assert-True (($output -join "`n").Contains('result=Failed')) 'A non-terminal preceding task did not fail closed after the poll bound.' + Assert-True (($output -join "`n").Contains('Timeline completeness was not established')) 'The timeline poll timeout was not diagnosed.' + + $server = Start-TestHttpServer $timelineJson 3000 + try { + $env:SYSTEM_ACCESSTOKEN = 'test-token' + $env:SYSTEM_COLLECTIONURI = "http://127.0.0.1:$($server.Port)/" + $env:SYSTEM_TEAMPROJECT = 'project' + $env:BUILD_BUILDID = '42' + $env:SYSTEM_JOBID = $jobId + $deadlineStopwatch = [Diagnostics.Stopwatch]::StartNew() + $output = & $gateScript ` + -JobStatus 'SucceededWithIssues' ` + -RecoveredOptionalTaskRefs 'androidArchivesCache' ` + -TimelinePollTimeoutSeconds 1 ` + -TimelinePollIntervalSeconds 0 6>&1 + $deadlineStopwatch.Stop() + } finally { + Stop-TestHttpServer $server + } + Assert-True (($output -join "`n").Contains('result=Failed')) 'A response body completed after the deadline did not fail closed.' + Assert-True ($deadlineStopwatch.Elapsed.TotalSeconds -lt 2.5) 'The response-body read exceeded the timeline polling deadline tolerance.' + Remove-Item Env:SYSTEM_ACCESSTOKEN, Env:SYSTEM_JOBID + + $output = Invoke-Gate @( + (New-Task 'realFailure' 'succeededWithIssues') + ) '' + Assert-True (($output -join "`n").Contains('result=Failed')) 'A job with no recovered-task allowlist did not fail.' + + $output = Invoke-Gate @( + (New-Task 'unrelatedJobFailure' 'failed' 'other-job') + ) 'androidArchivesCache' + Assert-True (($output -join "`n").Contains('result=Failed')) 'A missing recovered task record did not fail safe.' + + $missingTimeline = Join-Path $tempRoot 'missing.json' + $output = & $gateScript ` + -JobStatus 'SucceededWithIssues' ` + -RecoveredOptionalTaskRefs 'androidArchivesCache' ` + -JobId $jobId ` + -TimelinePath $missingTimeline ` + -TimelinePollTimeoutSeconds 0 6>&1 + Assert-True (($output -join "`n").Contains('result=Failed')) 'A timeline read failure did not fail closed.' + + Remove-Item Env:SYSTEM_ACCESSTOKEN -ErrorAction SilentlyContinue + $env:BUILD_BUILDID = '123' + $env:SYSTEM_COLLECTIONURI = 'https://example.invalid/' + $env:SYSTEM_TEAMPROJECT = 'test-project' + $output = & $gateScript ` + -JobStatus 'SucceededWithIssues' ` + -RecoveredOptionalTaskRefs 'androidArchivesCache' ` + -JobId $jobId 6>&1 + Assert-True (($output -join "`n").Contains('result=Failed')) 'A missing Azure access token did not fail closed.' + + $env:SYSTEM_ACCESSTOKEN = 'test-token' + $env:SYSTEM_COLLECTIONURI = "http://127.0.0.1:$(Get-FreeTcpPort)/" + $env:SYSTEM_TEAMPROJECT = 'project' + $env:BUILD_BUILDID = '42' + $env:SYSTEM_JOBID = $jobId + $output = & $gateScript ` + -JobStatus 'SucceededWithIssues' ` + -RecoveredOptionalTaskRefs 'androidArchivesCache' ` + -TimelinePollTimeoutSeconds 1 6>&1 + Assert-True (($output -join "`n").Contains('result=Failed')) 'A timeline API failure did not fail closed.' + Remove-Item Env:SYSTEM_ACCESSTOKEN, Env:SYSTEM_JOBID + + $output = Invoke-Gate @() '' 'Succeeded' + Assert-True (-not ($output -join "`n").Contains('result=Failed')) 'A successful job was failed.' +} finally { + Remove-Item Env:BUILD_BUILDID -ErrorAction SilentlyContinue + Remove-Item Env:SYSTEM_COLLECTIONURI -ErrorAction SilentlyContinue + Remove-Item Env:SYSTEM_TEAMPROJECT -ErrorAction SilentlyContinue + Remove-Item Env:SYSTEM_JOBID -ErrorAction SilentlyContinue + Remove-Item Env:SYSTEM_ACCESSTOKEN -ErrorAction SilentlyContinue + Remove-Item -LiteralPath $tempRoot -Recurse -Force -ErrorAction SilentlyContinue +} diff --git a/build-tools/automation/scripts/FailOnUnexpectedIssues.ps1 b/build-tools/automation/scripts/FailOnUnexpectedIssues.ps1 new file mode 100644 index 00000000000..18df3c6da97 --- /dev/null +++ b/build-tools/automation/scripts/FailOnUnexpectedIssues.ps1 @@ -0,0 +1,232 @@ +param ( + [string] $JobStatus = $env:AGENT_JOBSTATUS, + [string] $RecoveredOptionalTaskRefs = $env:RECOVERED_OPTIONAL_TASK_REFS, + [string] $JobId = $env:SYSTEM_JOBID, + [string[]] $TimelinePath, + [ValidateRange(0, 300)] + [int] $TimelinePollTimeoutSeconds = 20, + [ValidateRange(0, 60)] + [int] $TimelinePollIntervalSeconds = 2 +) + +$ErrorActionPreference = 'Stop' +$gateTaskReferenceName = 'failOnUnexpectedIssues' +$validTaskResults = @('succeeded', 'succeededWithIssues', 'failed', 'canceled', 'skipped', 'abandoned') +Write-Host "Current job status is: $JobStatus" + +function Complete-AsFailed ([string] $message) { + Write-Host $message + Write-Host '##vso[task.complete result=Failed;]DONE' +} + +function Get-TaskReferenceName ($task) { + if (-not [string]::IsNullOrWhiteSpace($task.refName)) { + return $task.refName + } + return $task.name +} + +function Get-TimelineFromApi ([string] $uri, [string] $accessToken) { + $client = [Net.Http.HttpClient]::new() + $request = [Net.Http.HttpRequestMessage]::new([Net.Http.HttpMethod]::Get, $uri) + $cancellation = [Threading.CancellationTokenSource]::new() + $response = $null + try { + $client.Timeout = [Threading.Timeout]::InfiniteTimeSpan + $request.Headers.Authorization = [Net.Http.Headers.AuthenticationHeaderValue]::new('Bearer', $accessToken) + $remainingMilliseconds = [Math]::Floor( + ($TimelinePollTimeoutSeconds - $timelineStopwatch.Elapsed.TotalSeconds) * 1000 + ) + if ($remainingMilliseconds -le 0) { + throw [TimeoutException]::new('The timeline polling deadline elapsed before the request started.') + } + $cancellation.CancelAfter([int] $remainingMilliseconds) + + $response = $client.SendAsync( + $request, + [Net.Http.HttpCompletionOption]::ResponseContentRead, + $cancellation.Token + ).GetAwaiter().GetResult() + if (-not $response.IsSuccessStatusCode) { + throw "The timeline request failed with HTTP status $([int] $response.StatusCode) ($($response.ReasonPhrase))." + } + + $content = $response.Content.ReadAsStringAsync().GetAwaiter().GetResult() + return $content | ConvertFrom-Json + } finally { + if ($null -ne $response) { + $response.Dispose() + } + $cancellation.Dispose() + $request.Dispose() + $client.Dispose() + } +} + +if ($JobStatus -ne 'SucceededWithIssues') { + return +} + +$allowedTasks = @($RecoveredOptionalTaskRefs -split ';' | Where-Object { -not [string]::IsNullOrWhiteSpace($_) }) +if ($allowedTasks.Count -eq 0) { + Complete-AsFailed 'No recovered optional tasks were recorded.' + return +} + +$hasTimelineFixture = $null -ne $TimelinePath -and $TimelinePath.Count -ne 0 +try { + if (-not $hasTimelineFixture) { + $requiredEnvironmentVariables = @{ + BUILD_BUILDID = $env:BUILD_BUILDID + SYSTEM_COLLECTIONURI = $env:SYSTEM_COLLECTIONURI + SYSTEM_ACCESSTOKEN = $env:SYSTEM_ACCESSTOKEN + SYSTEM_JOBID = $JobId + SYSTEM_TEAMPROJECT = $env:SYSTEM_TEAMPROJECT + } + foreach ($entry in $requiredEnvironmentVariables.GetEnumerator()) { + if ([string]::IsNullOrWhiteSpace($entry.Value)) { + throw "Required Azure Pipelines environment variable '$($entry.Key)' is not set." + } + } + + $project = [Uri]::EscapeDataString($env:SYSTEM_TEAMPROJECT) + $timelineUri = "$($env:SYSTEM_COLLECTIONURI)$project/_apis/build/builds/$($env:BUILD_BUILDID)/timeline?api-version=7.1" + Add-Type -AssemblyName System.Net.Http + } +} catch { + Complete-AsFailed "Could not prepare the Azure Pipelines timeline request: $($_.Exception.Message)" + return +} + +$maxTimelineAttempts = if ($TimelinePollIntervalSeconds -eq 0) { + $TimelinePollTimeoutSeconds + 1 +} else { + [Math]::Floor($TimelinePollTimeoutSeconds / $TimelinePollIntervalSeconds) + 1 +} +$precedingTasks = $null +$timelineStatus = 'The gate task was not present in the current-job timeline.' +$timelineStopwatch = [Diagnostics.Stopwatch]::StartNew() + +for ($attempt = 1; $attempt -le $maxTimelineAttempts; $attempt++) { + $remainingMilliseconds = [Math]::Floor( + ($TimelinePollTimeoutSeconds - $timelineStopwatch.Elapsed.TotalSeconds) * 1000 + ) + if ($remainingMilliseconds -le 0) { + break + } + + try { + if ($hasTimelineFixture) { + $timelinePathIndex = [Math]::Min($attempt - 1, $TimelinePath.Count - 1) + $timeline = Get-Content -LiteralPath $TimelinePath[$timelinePathIndex] -Raw | ConvertFrom-Json + } else { + $timeline = Get-TimelineFromApi $timelineUri $env:SYSTEM_ACCESSTOKEN + } + + if ($null -eq $timeline -or $null -eq $timeline.records) { + throw 'The timeline response does not contain records.' + } + + $currentJobTasks = @($timeline.records | Where-Object { + $_.parentId -eq $JobId -and $_.type -eq 'Task' + }) + $gateTasks = @($currentJobTasks | Where-Object { + (Get-TaskReferenceName $_) -eq $gateTaskReferenceName + }) + if ($gateTasks.Count -gt 1) { + throw "The current-job timeline contains multiple '$gateTaskReferenceName' task records." + } + if ($gateTasks.Count -eq 1) { + $gateOrder = 0 + if (-not [int]::TryParse([string] $gateTasks[0].order, [ref] $gateOrder) -or $gateOrder -lt 1) { + throw "The '$gateTaskReferenceName' task has an invalid timeline order." + } + + $taskOrders = [System.Collections.Generic.List[int]]::new() + foreach ($task in $currentJobTasks) { + $taskOrder = 0 + if (-not [int]::TryParse([string] $task.order, [ref] $taskOrder) -or $taskOrder -lt 1) { + throw "Task '$(Get-TaskReferenceName $task)' has an invalid timeline order." + } + $taskOrders.Add($taskOrder) + } + if (@($taskOrders | Sort-Object -Unique).Count -ne $taskOrders.Count) { + throw 'The current-job timeline contains duplicate task order values.' + } + + $visiblePrecedingTasks = @($currentJobTasks | Where-Object { + [int] $_.order -lt $gateOrder + }) + if ($visiblePrecedingTasks.Count -ne $gateOrder - 1) { + $timelineStatus = "Only $($visiblePrecedingTasks.Count) of $($gateOrder - 1) preceding task records are visible." + } else { + $incompleteTasks = @($visiblePrecedingTasks | Where-Object { + $_.state -ne 'completed' -or [string]::IsNullOrWhiteSpace($_.result) + }) + if ($incompleteTasks.Count -eq 0) { + $invalidResultTasks = @($visiblePrecedingTasks | Where-Object { + $_.result -notin $validTaskResults + }) + if ($invalidResultTasks.Count -ne 0) { + $invalidTask = $invalidResultTasks[0] + throw "Task '$(Get-TaskReferenceName $invalidTask)' has unsupported result '$($invalidTask.result)'." + } + if ($timelineStopwatch.Elapsed.TotalSeconds -le $TimelinePollTimeoutSeconds) { + $precedingTasks = $visiblePrecedingTasks + break + } + $timelineStatus = 'The complete timeline was returned after the polling timeout.' + } + + if ($incompleteTasks.Count -ne 0) { + $incompleteNames = @($incompleteTasks | ForEach-Object { + "'$(Get-TaskReferenceName $_)' ($($_.state), $($_.result))" + }) + $timelineStatus = "Preceding task records are not terminal: $($incompleteNames -join ', ')." + } + } + } + } catch { + $timelineStatus = "Could not inspect the Azure Pipelines timeline: $($_.Exception.Message)" + } + + if ($attempt -lt $maxTimelineAttempts) { + Write-Host "$timelineStatus Retrying timeline inspection ($attempt/$maxTimelineAttempts)." + $remainingMilliseconds = [Math]::Floor( + ($TimelinePollTimeoutSeconds - $timelineStopwatch.Elapsed.TotalSeconds) * 1000 + ) + if ($remainingMilliseconds -le 0) { + break + } + $sleepMilliseconds = [Math]::Min($TimelinePollIntervalSeconds * 1000, $remainingMilliseconds) + if ($sleepMilliseconds -gt 0) { + Start-Sleep -Milliseconds $sleepMilliseconds + } + } +} + +if ($null -eq $precedingTasks) { + Complete-AsFailed "$timelineStatus Timeline completeness was not established within $TimelinePollTimeoutSeconds seconds." + return +} + +$issueTasks = @($precedingTasks | Where-Object { + $_.result -in @('failed', 'succeededWithIssues') +}) +$unexpectedTasks = @($issueTasks | Where-Object { + (Get-TaskReferenceName $_) -notin $allowedTasks +}) + +if ($issueTasks.Count -eq 0 -or $unexpectedTasks.Count -ne 0) { + foreach ($task in $unexpectedTasks) { + $taskReferenceName = Get-TaskReferenceName $task + Write-Host "Unexpected issue task: $($task.name) ($taskReferenceName, result $($task.result))" + } + Complete-AsFailed 'The current job contains an unexpected issue.' + return +} + +foreach ($task in $issueTasks) { + $taskReferenceName = Get-TaskReferenceName $task + Write-Host "Ignoring recovered optional task: $($task.name) ($taskReferenceName)" +} diff --git a/build-tools/automation/scripts/Invoke-ProcessWithRetry.Tests.ps1 b/build-tools/automation/scripts/Invoke-ProcessWithRetry.Tests.ps1 new file mode 100644 index 00000000000..720ffc74c7c --- /dev/null +++ b/build-tools/automation/scripts/Invoke-ProcessWithRetry.Tests.ps1 @@ -0,0 +1,52 @@ +$ErrorActionPreference = 'Stop' +$scriptUnderTest = Join-Path $PSScriptRoot 'Invoke-ProcessWithRetry.ps1' +$powerShellExe = (Get-Process -Id $PID).Path +if (-not $powerShellExe) { + $powerShellExe = 'powershell.exe' +} + +$testDirectory = Join-Path ([IO.Path]::GetTempPath()) ([IO.Path]::GetRandomFileName()) +$childScript = Join-Path $testDirectory 'Hang.ps1' +$attemptFile = Join-Path $testDirectory 'attempts.txt' + +try { + New-Item -ItemType Directory -Path $testDirectory | Out-Null + @' +param ( + [string] $AttemptFile +) + +$attempt = 0 +if (Test-Path -LiteralPath $AttemptFile) { + $attempt = [int] (Get-Content -LiteralPath $AttemptFile -Raw) +} +Set-Content -LiteralPath $AttemptFile -Value ($attempt + 1) -Encoding ASCII +Start-Sleep -Seconds 30 +'@ | Set-Content -LiteralPath $childScript -Encoding ASCII + + $childArguments = "-NoLogo -NoProfile -File `"$childScript`" -AttemptFile `"$attemptFile`"" + $stopwatch = [Diagnostics.Stopwatch]::StartNew() + & $powerShellExe -NoLogo -NoProfile -File $scriptUnderTest ` + -FilePath $powerShellExe ` + -Arguments $childArguments ` + -TimeoutSeconds 1 ` + -RetryCount 1 ` + -RetryDelaySeconds 0 + $exitCode = $LASTEXITCODE + $stopwatch.Stop() + + if ($exitCode -ne 124) { + throw "Expected timeout exit code 124, got $exitCode." + } + + $attempts = [int] (Get-Content -LiteralPath $attemptFile -Raw) + if ($attempts -ne 2) { + throw "Expected two process attempts, got $attempts." + } + + if ($stopwatch.Elapsed.TotalSeconds -lt 1.5 -or $stopwatch.Elapsed.TotalSeconds -gt 10) { + throw "Expected two bounded one-second attempts, elapsed time was $($stopwatch.Elapsed)." + } +} finally { + Remove-Item -LiteralPath $testDirectory -Recurse -Force -ErrorAction Ignore +} diff --git a/build-tools/automation/scripts/Invoke-ProcessWithRetry.ps1 b/build-tools/automation/scripts/Invoke-ProcessWithRetry.ps1 new file mode 100644 index 00000000000..bc6b8d7e049 --- /dev/null +++ b/build-tools/automation/scripts/Invoke-ProcessWithRetry.ps1 @@ -0,0 +1,83 @@ +param ( + [Parameter(Mandatory = $true)] + [string] $FilePath, + + [string] $Arguments = "", + + [Parameter(Mandatory = $true)] + [ValidateRange(1, 2147483)] + [int] $TimeoutSeconds, + + [ValidateRange(0, 10)] + [int] $RetryCount = 0, + + [ValidateRange(0, 300)] + [int] $RetryDelaySeconds = 1 +) + +$ErrorActionPreference = 'Stop' + +function Stop-ProcessTree { + param ( + [Diagnostics.Process] $Process + ) + + if ($Process.HasExited) { + return + } + + if ($env:OS -eq 'Windows_NT') { + & taskkill.exe /PID $Process.Id /T /F + if ($LASTEXITCODE -ne 0 -and -not $Process.HasExited) { + throw "Failed to stop process tree $($Process.Id)." + } + } else { + $Process.Kill($true) + } + + $Process.WaitForExit() +} + +$attemptCount = $RetryCount + 1 +for ($attempt = 1; $attempt -le $attemptCount; $attempt++) { + Write-Host "Starting process attempt $attempt of $attemptCount with a $TimeoutSeconds-second timeout." + + $startInfo = [Diagnostics.ProcessStartInfo]::new() + $startInfo.FileName = $FilePath + $startInfo.Arguments = $Arguments + $startInfo.UseShellExecute = $false + + $process = [Diagnostics.Process]::new() + $process.StartInfo = $startInfo + try { + if (-not $process.Start()) { + throw "Failed to start '$FilePath'." + } + + if (-not $process.WaitForExit($TimeoutSeconds * 1000)) { + Stop-ProcessTree $process + if ($attempt -eq $attemptCount) { + Write-Error "Process '$FilePath' timed out after $TimeoutSeconds seconds on attempt $attempt of $attemptCount." -ErrorAction Continue + exit 124 + } + + Write-Warning "Process '$FilePath' timed out after $TimeoutSeconds seconds on attempt $attempt of $attemptCount; retrying." + } else { + $exitCode = $process.ExitCode + if ($exitCode -eq 0) { + exit 0 + } + if ($attempt -eq $attemptCount) { + exit $exitCode + } + + Write-Warning "Process '$FilePath' exited with code $exitCode on attempt $attempt of $attemptCount; retrying." + } + } finally { + $process.Dispose() + } + + if ($RetryDelaySeconds -gt 0) { + Start-Sleep -Seconds $RetryDelaySeconds + } +} diff --git a/build-tools/automation/scripts/RecoverAndroidArchiveCache.Tests.ps1 b/build-tools/automation/scripts/RecoverAndroidArchiveCache.Tests.ps1 new file mode 100644 index 00000000000..741790be3de --- /dev/null +++ b/build-tools/automation/scripts/RecoverAndroidArchiveCache.Tests.ps1 @@ -0,0 +1,53 @@ +$ErrorActionPreference = 'Stop' +$recoveryScript = Join-Path $PSScriptRoot 'RecoverAndroidArchiveCache.ps1' +$tempRoot = Join-Path ([IO.Path]::GetTempPath()) "xa-android-cache-tests-$([Guid]::NewGuid())" + +function Assert-True ([bool] $condition, [string] $message) { + if (-not $condition) { + throw $message + } +} + +function New-CacheDirectory ([string] $name) { + $path = Join-Path $tempRoot $name + New-Item -ItemType Directory -Path $path -Force | Out-Null + Set-Content -LiteralPath (Join-Path $path 'existing') -Value 'cached' -Encoding ASCII + return $path +} + +try { + $cache = New-CacheDirectory 'valid' + $output = & $recoveryScript -CacheState 'false' -CachePath $cache 6>&1 + Assert-True (Test-Path (Join-Path $cache 'existing')) 'A valid cache miss was cleared.' + Assert-True (-not ($output -join "`n").Contains('androidArchivesCache')) 'A valid cache state was marked as recovered.' + + $cache = New-CacheDirectory 'unset' + $env:ANDROID_ARCHIVES_CACHE_RESTORED = $null + $env:ANDROID_ARCHIVES_DIR = $cache + $output = & $recoveryScript 6>&1 + Assert-True (-not (Test-Path (Join-Path $cache 'existing'))) 'An unset cache state was not reset.' + Assert-True (($output -join "`n").Contains('variable=RECOVERED_OPTIONAL_TASK_REFS]androidArchivesCache')) 'The recovered cache task was not recorded.' + + $cache = New-CacheDirectory 'append' + $output = & $recoveryScript ` + -CacheState $null ` + -CachePath $cache ` + -RecoveredOptionalTaskRefs 'gradleDependenciesCache' 6>&1 + Assert-True (($output -join "`n").Contains('variable=RECOVERED_OPTIONAL_TASK_REFS]gradleDependenciesCache;androidArchivesCache')) 'The existing recovered-task allowlist was not preserved.' + + $blockingFile = Join-Path $tempRoot 'blocking-file' + Set-Content -LiteralPath $blockingFile -Value 'not a directory' -Encoding ASCII + $recoveryFailed = $false + try { + & $recoveryScript ` + -CacheState $null ` + -CachePath (Join-Path $blockingFile 'cache') 6>&1 | Out-Null + } catch { + $recoveryFailed = $true + } + Assert-True $recoveryFailed 'A cache reset failure was incorrectly reported as recovered.' +} finally { + Remove-Item Env:ANDROID_ARCHIVES_CACHE_RESTORED -ErrorAction SilentlyContinue + Remove-Item Env:ANDROID_ARCHIVES_DIR -ErrorAction SilentlyContinue + Remove-Item -LiteralPath $tempRoot -Recurse -Force -ErrorAction SilentlyContinue +} diff --git a/build-tools/automation/scripts/RecoverAndroidArchiveCache.ps1 b/build-tools/automation/scripts/RecoverAndroidArchiveCache.ps1 new file mode 100644 index 00000000000..c11b2980492 --- /dev/null +++ b/build-tools/automation/scripts/RecoverAndroidArchiveCache.ps1 @@ -0,0 +1,38 @@ +param ( + [AllowNull()] + [string] $CacheState = $env:ANDROID_ARCHIVES_CACHE_RESTORED, + [string] $CachePath = $env:ANDROID_ARCHIVES_DIR, + [string] $RecoveredOptionalTaskRefs = $env:RECOVERED_OPTIONAL_TASK_REFS +) + +$ErrorActionPreference = 'Stop' +$validCacheStates = @('true', 'inexact', 'false') +$recoveredTasks = [System.Collections.Generic.List[string]]::new() +foreach ($taskReferenceName in $RecoveredOptionalTaskRefs -split ';') { + if (-not [string]::IsNullOrWhiteSpace($taskReferenceName)) { + $recoveredTasks.Add($taskReferenceName) + } +} + +if ($validCacheStates -contains $CacheState) { + Write-Host "Android archive cache state: $CacheState" +} else { + if ([string]::IsNullOrWhiteSpace($CachePath)) { + throw 'Android archive cache path is not set.' + } + + Write-Host 'Android archive cache restore did not complete; using an empty cache directory for verified source downloads.' + if (Test-Path -LiteralPath $CachePath) { + Remove-Item -LiteralPath $CachePath -Recurse -Force + } + New-Item -ItemType Directory -Path $CachePath -Force | Out-Null + if (-not (Test-Path -LiteralPath $CachePath -PathType Container) -or + @(Get-ChildItem -LiteralPath $CachePath -Force).Count -ne 0) { + throw 'Android archive cache path could not be reset.' + } + if (-not $recoveredTasks.Contains('androidArchivesCache')) { + $recoveredTasks.Add('androidArchivesCache') + } +} + +Write-Host "##vso[task.setvariable variable=RECOVERED_OPTIONAL_TASK_REFS]$($recoveredTasks -join ';')" diff --git a/build-tools/automation/yaml-templates/cache-android-archives.yaml b/build-tools/automation/yaml-templates/cache-android-archives.yaml index e2f4c68f92f..d92ca166b97 100644 --- a/build-tools/automation/yaml-templates/cache-android-archives.yaml +++ b/build-tools/automation/yaml-templates/cache-android-archives.yaml @@ -36,6 +36,7 @@ steps: # `restoreKeys` fallback still recovers most of the previous cache when only # a subset changes (e.g. a single component bump). - task: Cache@2 + name: androidArchivesCache displayName: cache Android toolchain archives condition: ${{ parameters.condition }} # The cache service is an optimization. If it is unavailable or rate-limited, @@ -46,3 +47,9 @@ steps: restoreKeys: | "android-archives" | "v1" | "$(Agent.OS)" path: $(ANDROID_ARCHIVES_DIR) + cacheHitVar: ANDROID_ARCHIVES_CACHE_RESTORED + +- pwsh: | + & '${{ parameters.xaSourcePath }}/build-tools/automation/scripts/RecoverAndroidArchiveCache.ps1' + displayName: recover Android archive cache failure + condition: ${{ parameters.condition }} diff --git a/build-tools/automation/yaml-templates/fail-on-issue.yaml b/build-tools/automation/yaml-templates/fail-on-issue.yaml index 0aa33f1f32a..8159797fac1 100644 --- a/build-tools/automation/yaml-templates/fail-on-issue.yaml +++ b/build-tools/automation/yaml-templates/fail-on-issue.yaml @@ -1,11 +1,12 @@ parameters: condition: succeeded() + xaSourcePath: $(System.DefaultWorkingDirectory) steps: - powershell: | - Write-Host "Current job status is: $env:AGENT_JOBSTATUS" - if ($env:AGENT_JOBSTATUS -eq "SucceededWithIssues") { - Write-Host "##vso[task.complete result=Failed;]DONE" - } + & '${{ parameters.xaSourcePath }}/build-tools/automation/scripts/FailOnUnexpectedIssues.ps1' + name: failOnUnexpectedIssues displayName: fail if any issues occurred condition: ${{ parameters.condition }} + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/build-tools/automation/yaml-templates/run-dotnet-preview.yaml b/build-tools/automation/yaml-templates/run-dotnet-preview.yaml index 68489576ca3..520a3980ac7 100644 --- a/build-tools/automation/yaml-templates/run-dotnet-preview.yaml +++ b/build-tools/automation/yaml-templates/run-dotnet-preview.yaml @@ -10,6 +10,9 @@ parameters: continueOnError: true retryCountOnTaskFailure: 0 taskTimeoutInMinutes: 0 + processTimeoutInMinutes: 0 + processRetryCount: 0 + processRetryDelaySeconds: 1 env: {} steps: @@ -23,7 +26,20 @@ steps: $env:PATH = "${DOTNET_ROOT};$env:PATH" $dotnetPath = "${DOTNET_ROOT}\dotnet.exe" } - & $dotnetPath ${{ parameters.command }} ${{ parameters.project }} ${{ parameters.arguments }} + $processTimeoutInMinutes = [int] "${{ parameters.processTimeoutInMinutes }}" + if ($processTimeoutInMinutes -gt 0) { + $dotnetArguments = @' + ${{ parameters.command }} ${{ parameters.project }} ${{ parameters.arguments }} + '@.Trim() + & "${{ parameters.xaSourcePath }}/build-tools/automation/scripts/Invoke-ProcessWithRetry.ps1" ` + -FilePath $dotnetPath ` + -Arguments $dotnetArguments ` + -TimeoutSeconds ($processTimeoutInMinutes * 60) ` + -RetryCount ${{ parameters.processRetryCount }} ` + -RetryDelaySeconds ${{ parameters.processRetryDelaySeconds }} + } else { + & $dotnetPath ${{ parameters.command }} ${{ parameters.project }} ${{ parameters.arguments }} + } if ([System.Convert]::ToBoolean("${{ parameters.useExitCodeForErrors }}") -and $LASTEXITCODE -ne 0) { Write-Host "##vso[task.logissue type=error]Test suite had $LASTEXITCODE failure(s)." Write-Host "##vso[task.complete result=Failed;]" diff --git a/build-tools/automation/yaml-templates/setup-test-environment-steps.yaml b/build-tools/automation/yaml-templates/setup-test-environment-steps.yaml index 55de20b2f33..7d47678130d 100644 --- a/build-tools/automation/yaml-templates/setup-test-environment-steps.yaml +++ b/build-tools/automation/yaml-templates/setup-test-environment-steps.yaml @@ -110,8 +110,10 @@ steps: project: ${{ parameters.xaSourcePath }}/build-tools/create-packs/Microsoft.Android.Sdk.proj arguments: -t:ExtractWorkloadPacks -c ${{ parameters.configuration }} -v:n -bl:${{ parameters.xaSourcePath }}/bin/Test${{ parameters.configuration }}/extract-workloads.binlog continueOnError: false - retryCountOnTaskFailure: 1 - taskTimeoutInMinutes: 20 + taskTimeoutInMinutes: 45 + processTimeoutInMinutes: 20 + processRetryCount: 1 + processRetryDelaySeconds: 5 env: NUGET_RETRY_HTTP_429: 'true' NUGET_OBSERVE_RETRY_AFTER: 'true' From 5892ff802aac92fad9c86d4696a70ca9c949dc6d Mon Sep 17 00:00:00 2001 From: Simon Rozsival Date: Tue, 8 Sep 2026 21:47:34 +0200 Subject: [PATCH 3/4] Test nonzero process retry recovery Cover the retry path where the first process exits nonzero and the second attempt succeeds. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../scripts/Invoke-ProcessWithRetry.Tests.ps1 | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/build-tools/automation/scripts/Invoke-ProcessWithRetry.Tests.ps1 b/build-tools/automation/scripts/Invoke-ProcessWithRetry.Tests.ps1 index 720ffc74c7c..5fdf6dfb84d 100644 --- a/build-tools/automation/scripts/Invoke-ProcessWithRetry.Tests.ps1 +++ b/build-tools/automation/scripts/Invoke-ProcessWithRetry.Tests.ps1 @@ -8,6 +8,8 @@ if (-not $powerShellExe) { $testDirectory = Join-Path ([IO.Path]::GetTempPath()) ([IO.Path]::GetRandomFileName()) $childScript = Join-Path $testDirectory 'Hang.ps1' $attemptFile = Join-Path $testDirectory 'attempts.txt' +$retryChildScript = Join-Path $testDirectory 'FailThenSucceed.ps1' +$retryAttemptFile = Join-Path $testDirectory 'retry-attempts.txt' try { New-Item -ItemType Directory -Path $testDirectory | Out-Null @@ -47,6 +49,40 @@ Start-Sleep -Seconds 30 if ($stopwatch.Elapsed.TotalSeconds -lt 1.5 -or $stopwatch.Elapsed.TotalSeconds -gt 10) { throw "Expected two bounded one-second attempts, elapsed time was $($stopwatch.Elapsed)." } + + @' +param ( + [string] $AttemptFile +) + +$attempt = 0 +if (Test-Path -LiteralPath $AttemptFile) { + $attempt = [int] (Get-Content -LiteralPath $AttemptFile -Raw) +} +$attempt++ +Set-Content -LiteralPath $AttemptFile -Value $attempt -Encoding ASCII +if ($attempt -eq 1) { + exit 42 +} +'@ | Set-Content -LiteralPath $retryChildScript -Encoding ASCII + + $retryArguments = "-NoLogo -NoProfile -File `"$retryChildScript`" -AttemptFile `"$retryAttemptFile`"" + & $powerShellExe -NoLogo -NoProfile -File $scriptUnderTest ` + -FilePath $powerShellExe ` + -Arguments $retryArguments ` + -TimeoutSeconds 10 ` + -RetryCount 1 ` + -RetryDelaySeconds 0 + $retryExitCode = $LASTEXITCODE + + if ($retryExitCode -ne 0) { + throw "Expected successful retry exit code 0, got $retryExitCode." + } + + $retryAttempts = [int] (Get-Content -LiteralPath $retryAttemptFile -Raw) + if ($retryAttempts -ne 2) { + throw "Expected two nonzero-exit process attempts, got $retryAttempts." + } } finally { Remove-Item -LiteralPath $testDirectory -Recurse -Force -ErrorAction Ignore } From 993124e4f3d9369b2753cbb0a779818e47cb540f Mon Sep 17 00:00:00 2001 From: Simon Rozsival Date: Thu, 10 Sep 2026 15:07:52 +0200 Subject: [PATCH 4/4] Test retry helper caller continuation Cover in-process call-operator invocation for success, failure, and timeout. Record results from a separate caller to detect premature exits without changing the working template behavior. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../scripts/Invoke-ProcessWithRetry.Tests.ps1 | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/build-tools/automation/scripts/Invoke-ProcessWithRetry.Tests.ps1 b/build-tools/automation/scripts/Invoke-ProcessWithRetry.Tests.ps1 index 5fdf6dfb84d..049193571c8 100644 --- a/build-tools/automation/scripts/Invoke-ProcessWithRetry.Tests.ps1 +++ b/build-tools/automation/scripts/Invoke-ProcessWithRetry.Tests.ps1 @@ -10,6 +10,8 @@ $childScript = Join-Path $testDirectory 'Hang.ps1' $attemptFile = Join-Path $testDirectory 'attempts.txt' $retryChildScript = Join-Path $testDirectory 'FailThenSucceed.ps1' $retryAttemptFile = Join-Path $testDirectory 'retry-attempts.txt' +$callerScript = Join-Path $testDirectory 'Caller.ps1' +$callerResultFile = Join-Path $testDirectory 'caller-results.txt' try { New-Item -ItemType Directory -Path $testDirectory | Out-Null @@ -83,6 +85,46 @@ if ($attempt -eq 1) { if ($retryAttempts -ne 2) { throw "Expected two nonzero-exit process attempts, got $retryAttempts." } + + @' +param ( + [string] $RetryScript, + [string] $PowerShellExe, + [string] $ResultFile +) + +$ErrorActionPreference = 'Stop' +$cases = @( + @{ Command = 'exit 0'; TimeoutSeconds = 10 } + @{ Command = 'exit 42'; TimeoutSeconds = 10 } + @{ Command = 'Start-Sleep -Seconds 30'; TimeoutSeconds = 1 } +) +$exitCodes = @() +foreach ($case in $cases) { + & $RetryScript ` + -FilePath $PowerShellExe ` + -Arguments "-NoLogo -NoProfile -Command $($case.Command)" ` + -TimeoutSeconds $case.TimeoutSeconds + $exitCodes += $LASTEXITCODE +} +Set-Content -LiteralPath $ResultFile -Value ($exitCodes -join ',') -Encoding ASCII +'@ | Set-Content -LiteralPath $callerScript -Encoding ASCII + + # A separate caller and result file also detect a premature successful exit. + & $powerShellExe -NoLogo -NoProfile -File $callerScript ` + -RetryScript $scriptUnderTest ` + -PowerShellExe $powerShellExe ` + -ResultFile $callerResultFile + if ($LASTEXITCODE -ne 0) { + throw "Expected the calling script to complete successfully, got $LASTEXITCODE." + } + if (-not (Test-Path -LiteralPath $callerResultFile)) { + throw 'The retry helper exited the calling script before it could record the exit codes.' + } + $callerExitCodes = (Get-Content -LiteralPath $callerResultFile -Raw).Trim() + if ($callerExitCodes -ne '0,42,124') { + throw "Expected the caller to observe exit codes 0,42,124, got '$callerExitCodes'." + } } finally { Remove-Item -LiteralPath $testDirectory -Recurse -Force -ErrorAction Ignore }