[main] Update dependencies from dotnet/arcade

eng/Version.Details.props

@@ -6,7 +6,7 @@ This file should be imported by eng/Versions.props
 <Project>
   <PropertyGroup>
     <!-- dotnet-arcade dependencies -->
-    <MicrosoftDotNetArcadeSdkPackageVersion>11.0.0-beta.26456.1</MicrosoftDotNetArcadeSdkPackageVersion>
+    <MicrosoftDotNetArcadeSdkPackageVersion>11.0.0-beta.26461.5</MicrosoftDotNetArcadeSdkPackageVersion>
     <!-- dotnet-msbuild dependencies -->
     <MicrosoftBuildPackageVersion>18.12.0-1.26454.5</MicrosoftBuildPackageVersion>
     <MicrosoftBuildFrameworkPackageVersion>18.12.0-1.26454.5</MicrosoftBuildFrameworkPackageVersion>

eng/Version.Details.xml

@@ -82,9 +82,9 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="11.0.0-beta.26456.1">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="11.0.0-beta.26461.5">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>66b75e61d883abd9e3af86a811c3809a8d9142ca</Sha>
+      <Sha>1574a0ce35761b7ce5e783074cc2f9567d278396</Sha>
     </Dependency>
     <Dependency Name="optimization.windows_nt-x64.MIBC.Runtime" Version="1.0.0-prerelease.26451.1">
       <Uri>https://dev.azure.com/dnceng/internal/_git/dotnet-optimization</Uri>

eng/common/Get-GitHubAppToken.ps1

@@ -1,13 +1,11 @@
 # Mints a short-lived GitHub App installation access token by signing a JWT
-# with a private key stored in Azure Key Vault (RSA, RS256). The signed JWT is
-# exchanged with the GitHub API for a token scoped to a single installation.
+# with an RSA private key (RS256). The signed JWT is exchanged with the GitHub
+# API for a token scoped to a single installation.
 #
 # Requirements:
-#   - A GitHub App whose private key has been uploaded into Key Vault as an RSA
-#     key (the PEM converted to a Key Vault *key*, NOT stored as a secret).
-#   - The caller (the federated Azure service connection used to run this script)
-#     must have the `Key Vault Crypto User` role (or at minimum the `Sign`
-#     action) on that key.
+#   - A GitHub App ID and PEM private key stored as Azure Key Vault secrets.
+#   - The federated Azure service connection running this script must have
+#     `Get` access to those two secrets.
 #   - The App must be installed on the target organization/account
 #     (`InstallationOwner`) with the permissions/repositories it needs.
 #
@@ -16,17 +14,17 @@
 
 [CmdletBinding()]
 param(
-    # Name of the Key Vault that holds the GitHub App's RSA signing key.
+    # Name of the Key Vault holding the GitHub App credentials.
     [Parameter(Mandatory = $true)]
     [string] $KeyVaultName,
 
-    # Name of the RSA key inside the Key Vault (the App's private key).
+    # Secret Manager projection containing the GitHub App ID.
     [Parameter(Mandatory = $true)]
-    [string] $KeyName,
+    [string] $AppIdSecretName,
 
-    # The GitHub App's Client ID (the value to put in the `iss` JWT claim).
+    # Secret Manager projection containing the PEM private key.
     [Parameter(Mandatory = $true)]
-    [string] $AppClientId,
+    [string] $AppPrivateKeySecretName,
 
     # Login of the organization or user account whose installation we should
     # mint the token for (e.g. `dotnet`, `microsoft`).
@@ -39,16 +37,69 @@ param(
     [Parameter(Mandatory = $false)]
     [string] $OutputVariableName
 )
-
 $ErrorActionPreference = 'Stop'
 $PSNativeCommandUseErrorActionPreference = $true
 
 . $PSScriptRoot\pipeline-logging-functions.ps1
 
+if ($KeyVaultName -notmatch '^[A-Za-z][A-Za-z0-9-]{1,22}[A-Za-z0-9]$' -or $KeyVaultName.Contains('--')) {
+    Write-PipelineTelemetryError -Category 'Build' -Message "KeyVaultName '$KeyVaultName' is not a valid Azure Key Vault name."
+    exit 1
+}
+
 function ConvertTo-Base64Url([byte[]] $bytes) {
     return [Convert]::ToBase64String($bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_')
 }
 
+$previousNativeCommandErrorPreference = $PSNativeCommandUseErrorActionPreference
+try {
+    # Azure CLI can emit non-fatal Python warnings to stderr.
+    $PSNativeCommandUseErrorActionPreference = $false
+    $keyVaultAccessToken = az account get-access-token `
+        --resource https://vault.azure.net `
+        --query accessToken `
+        --output tsv `
+        --only-show-errors
+    $tokenExitCode = $LASTEXITCODE
+}
+catch {
+    Write-PipelineTelemetryError -Category 'Build' -Message "Failed to acquire an Azure Key Vault access token: $_"
+    exit 1
+}
+finally {
+    $PSNativeCommandUseErrorActionPreference = $previousNativeCommandErrorPreference
+}
+if ($tokenExitCode -ne 0 -or [string]::IsNullOrWhiteSpace($keyVaultAccessToken)) {
+    Write-PipelineTelemetryError -Category 'Build' -Message "'az account get-access-token' exited with code $tokenExitCode while acquiring an Azure Key Vault access token."
+    exit 1
+}
+
+function Get-KeyVaultSecret([string] $SecretName) {
+    # Use the data-plane REST API because `az keyvault secret show` can fail
+    # with Errno 22 on hosted Windows agents when reading these projections.
+    $escapedSecretName = [Uri]::EscapeDataString($SecretName)
+    $secretUri = "https://$KeyVaultName.vault.azure.net/secrets/$escapedSecretName`?api-version=7.4"
+    try {
+        $response = Invoke-RestMethod `
+            -Uri $secretUri `
+            -Headers @{ Authorization = "Bearer $keyVaultAccessToken" } `
+            -Method Get
+    }
+    catch {
+        Write-PipelineTelemetryError -Category 'Build' -Message "Failed to read secret '$SecretName' from vault '$KeyVaultName': $_. Verify the secret exists and the service connection has 'Key Vault Secrets User' access to it."
+        exit 1
+    }
+    if ([string]::IsNullOrWhiteSpace($response.value)) {
+        Write-PipelineTelemetryError -Category 'Build' -Message "Secret '$SecretName' in vault '$KeyVaultName' is empty."
+        exit 1
+    }
+    return [string] $response.value
+}
+
+Write-Host "Reading GitHub App credentials from vault '$KeyVaultName'..."
+$appId = Get-KeyVaultSecret $AppIdSecretName
+$privateKey = Get-KeyVaultSecret $AppPrivateKeySecretName
+
 # Build JWT header and payload. Use [ordered] hashtables so JSON
 # serialization is deterministic.
 $jwtHeader = [ordered]@{
@@ -59,46 +110,38 @@ $now = [System.DateTimeOffset]::UtcNow
 $jwtPayload = [ordered]@{
     iat = $now.AddMinutes(-1).ToUnixTimeSeconds()
     exp = $now.AddMinutes(5).ToUnixTimeSeconds()
-    iss = $AppClientId
+    iss = $appId
 }
 
 $headerEncoded  = ConvertTo-Base64Url ([System.Text.Encoding]::UTF8.GetBytes(($jwtHeader  | ConvertTo-Json -Compress)))
 $payloadEncoded = ConvertTo-Base64Url ([System.Text.Encoding]::UTF8.GetBytes(($jwtPayload | ConvertTo-Json -Compress)))
 $signingInput   = "$headerEncoded.$payloadEncoded"
 
-# Key Vault `sign` expects the *digest* (base64), not the raw bytes.
-$sha256       = [System.Security.Cryptography.SHA256]::Create()
-$digestBytes  = $sha256.ComputeHash([System.Text.Encoding]::UTF8.GetBytes($signingInput))
-$digestBase64 = [Convert]::ToBase64String($digestBytes)
+$sha256 = [System.Security.Cryptography.SHA256]::Create()
+try {
+    $digestBytes = $sha256.ComputeHash([System.Text.Encoding]::UTF8.GetBytes($signingInput))
+}
+finally {
+    $sha256.Dispose()
+}
 
-Write-Host "Signing JWT with key '$KeyName' in vault '$KeyVaultName'..."
-$previousNativeCommandErrorPreference = $PSNativeCommandUseErrorActionPreference
+Write-Host 'Signing JWT with the GitHub App private key...'
+$rsa = [System.Security.Cryptography.RSA]::Create()
 try {
-    # Azure CLI can emit non-fatal Python warnings to stderr even when signing succeeds.
-    # Use the exit code to determine success for this invocation.
-    $PSNativeCommandUseErrorActionPreference = $false
-    $signatureBase64 = az keyvault key sign `
-        --vault-name $KeyVaultName `
-        --name $KeyName `
-        --algorithm RS256 `
-        --digest $digestBase64 `
-        --query signature `
-        --output tsv `
-        --only-show-errors
-    $signExitCode = $LASTEXITCODE
+    $rsa.ImportFromPem($privateKey)
+    $signatureBytes = $rsa.SignHash(
+        $digestBytes,
+        [System.Security.Cryptography.HashAlgorithmName]::SHA256,
+        [System.Security.Cryptography.RSASignaturePadding]::Pkcs1)
+    $signatureUrl = ConvertTo-Base64Url $signatureBytes
 }
 catch {
-    Write-PipelineTelemetryError -Category 'Build' -Message "Failed to sign the JWT via Key Vault (key '$KeyName', vault '$KeyVaultName'): $_. Verify the service connection identity has the 'Key Vault Crypto User' role (Sign action) on the key."
+    Write-PipelineTelemetryError -Category 'Build' -Message "Failed to sign the GitHub App JWT with the supplied private key: $_"
     exit 1
 }
 finally {
-    $PSNativeCommandUseErrorActionPreference = $previousNativeCommandErrorPreference
-}
-if ($signExitCode -ne 0 -or [string]::IsNullOrWhiteSpace($signatureBase64)) {
-    Write-PipelineTelemetryError -Category 'Build' -Message "'az keyvault key sign' exited with code $signExitCode for key '$KeyName' in vault '$KeyVaultName'. Verify the service connection identity has the 'Key Vault Crypto User' role (Sign action) on the key."
-    exit 1
+    $rsa.Dispose()
 }
-$signatureUrl = $signatureBase64.Trim().TrimEnd('=').Replace('+', '-').Replace('/', '_')
 $jwt = "$signingInput.$signatureUrl"
 
 $headers = @{
@@ -126,7 +169,7 @@ try {
     } while ($pageInstallationCount -eq 100)
 }
 catch {
-    Write-PipelineTelemetryError -Category 'Build' -Message "Failed to list GitHub App installations: $_. The signed JWT may be invalid or the App's Client ID ('$AppClientId') may be incorrect."
+    Write-PipelineTelemetryError -Category 'Build' -Message "Failed to list GitHub App installations: $_. The signed JWT may be invalid or the App ID may be incorrect."
     exit 1
 }
 $matchingInstallations = @($installations | Where-Object { $_.account.login -ieq $InstallationOwner })

eng/common/build.sh

@@ -254,7 +254,7 @@ function Build {
     properties+=("/p:Projects=$projects")
   fi
 
-  local bl=""
+  local bl=()
   if [[ "$binary_log" == true ]]; then
     local binary_log_path=""
     if [[ -z "$binary_log_name" ]]; then
@@ -266,16 +266,16 @@ function Build {
     fi
 
     mkdir -p "$(dirname "$binary_log_path")"
-    bl="/bl:\"$binary_log_path\""
+    bl=("/bl:$binary_log_path")
   fi
 
   local check=""
   if [[ "$build_check" == true ]]; then
     check="/check"
   fi
 
-  MSBuild $_InitializeToolset \
-    $bl \
+  MSBuild "$_InitializeToolset" \
+    ${bl[@]+"${bl[@]}"} \
     $check \
     /p:Configuration=$configuration \
     /p:RepoRoot="$repo_root" \
@@ -299,7 +299,7 @@ function Build {
 
 if [[ "$clean" == true ]]; then
   if [ -d "$artifacts_dir" ]; then
-    rm -rf $artifacts_dir
+    rm -rf "$artifacts_dir"
     echo "Artifacts directory deleted."
   fi
   exit 0

eng/common/core-templates/job/job.yml

@@ -28,6 +28,7 @@ parameters:
   enablePublishTestResults: false
   enablePublishing: false
   enableBuildRetry: false
+  enableAstred: false
   mergeTestResults: false
   testRunTitle: ''
   testResultsFormat: ''
@@ -119,6 +120,12 @@ jobs:
         - name: ${{ pair.key }}
           value: ${{ pair.value }}
 
+  - ${{ if and(eq(parameters.enableAstred, true), eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+    - name: MSBUILDDEBUGENGINE
+      value: 1
+    - name: MSBUILDDEBUGPATH
+      value: $(Build.ArtifactStagingDirectory)/AstredCapture/binlogs
+
   # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds
   - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
     - group: DotNet-HelixApi-Access
@@ -236,3 +243,8 @@ jobs:
       condition: always()
   - ${{ each step in parameters.artifactPublishSteps }}:
     - ${{ step }}
+
+  - ${{ if and(eq(parameters.enableAstred, true), eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+    - template: /eng/common/core-templates/steps/astred-artifacts.yml
+      parameters:
+        binlogDir: $(MSBUILDDEBUGPATH)

eng/common/core-templates/job/onelocbuild.yml

@@ -5,23 +5,14 @@ parameters:
   # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool
   pool: ''
 
-  CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex
-  GithubPat: $(BotAccount-dotnet-bot-repo-PAT)
-
-  # Service connection for WIF-based Entra authentication to ceapex feeds (replaces CeapexPat).
-  # dnceng/internal and DevDiv/DevDiv have same-named, project-scoped connections. Other projects,
-  # and any pipeline that sets this to '', fall back to PAT-based auth via the CeapexPat parameter.
+  # Project-scoped WIF service connection for Ceapex feed authentication.
   CeapexServiceConnection: 'dnceng-onelocbuild-ceapex'
 
   # GitHub App authentication for the OneLoc check-in PR.
-  # dnceng/internal and DevDiv/DevDiv are enabled by default with their project-scoped service
-  # connections. Other projects must explicitly opt in after provisioning equivalent infrastructure.
-  UseGitHubAppAuthentication: true
-  UseGitHubAppAuthenticationInOtherProjects: false
   GitHubAppServiceConnection: 'dnceng-oneloc-githubapp'
-  GitHubAppClientId: 'Iv23lijBU8x3gc9lDOc9'
   GitHubAppKeyVaultName: 'EngKeyVault'
-  GitHubAppKeyName: 'oneloc-localization-app-key'
+  GitHubAppIdSecretName: 'oneloc-localization-app-app-id'
+  GitHubAppPrivateKeySecretName: 'oneloc-localization-app-app-private-key'
 
   SourcesDirectory: $(System.DefaultWorkingDirectory)
   CreatePr: true
@@ -49,7 +40,6 @@ jobs:
   displayName: OneLocBuild${{ parameters.JobNameSuffix }}
 
   variables:
-    - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat
     - name: _GenerateLocProjectArguments
       value: -SourcesDirectory ${{ parameters.SourcesDirectory }}
         -LanguageSet "${{ parameters.LanguageSet }}"
@@ -80,6 +70,10 @@ jobs:
   steps:
     - ${{ if eq(parameters.is1ESPipeline, '') }}:
       - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error
+    - ${{ if notIn(variables['System.TeamProject'], 'internal', 'DevDiv') }}:
+      - 'OneLocBuild is supported only in dnceng/internal and DevDiv/DevDiv.': error
+    - ${{ if eq(parameters.CeapexServiceConnection, '') }}:
+      - 'CeapexServiceConnection must identify a WIF service connection.': error
 
     - ${{ if ne(parameters.SkipLocProjectJsonGeneration, 'true') }}:
       - task: Powershell@2
@@ -89,17 +83,15 @@ jobs:
         displayName: Generate LocProject.json
         condition: ${{ parameters.condition }}
 
-    # Acquire an Entra token for ceapex feed access in the supported internal and DevDiv projects.
-    - ${{ if and(ne(parameters.CeapexServiceConnection, ''), or(eq(variables['System.TeamProject'], 'internal'), eq(variables['System.TeamProject'], 'DevDiv'))) }}:
-      - template: /eng/common/templates/steps/get-federated-access-token.yml
-        parameters:
-          federatedServiceConnection: ${{ parameters.CeapexServiceConnection }}
-          outputVariableName: 'CeapexEntraToken'
-          condition: ${{ parameters.condition }}
+    # Acquire a short-lived Entra token for Ceapex feed access.
+    - template: /eng/common/templates/steps/get-federated-access-token.yml
+      parameters:
+        federatedServiceConnection: ${{ parameters.CeapexServiceConnection }}
+        outputVariableName: 'CeapexEntraToken'
+        condition: ${{ parameters.condition }}
 
-    # Mint a short-lived GitHub App installation token for the loc check-in PR. Use the connection
-    # provisioned in each supported project; other projects must explicitly opt in and override it.
-    - ${{ if and(eq(parameters.RepoType, 'gitHub'), eq(parameters.UseGitHubAppAuthentication, true), or(eq(variables['System.TeamProject'], 'internal'), eq(variables['System.TeamProject'], 'DevDiv'), eq(parameters.UseGitHubAppAuthenticationInOtherProjects, true))) }}:
+    # Mint a short-lived GitHub App installation token for the loc check-in PR.
+    - ${{ if eq(parameters.RepoType, 'gitHub') }}:
       - template: /eng/common/core-templates/steps/get-github-app-token.yml
         parameters:
           is1ESPipeline: ${{ parameters.is1ESPipeline }}
@@ -108,8 +100,8 @@ jobs:
           ${{ else }}:
             azureSubscription: ${{ parameters.GitHubAppServiceConnection }}
           keyVaultName: ${{ parameters.GitHubAppKeyVaultName }}
-          keyName: ${{ parameters.GitHubAppKeyName }}
-          appClientId: ${{ parameters.GitHubAppClientId }}
+          appIdSecretName: ${{ parameters.GitHubAppIdSecretName }}
+          appPrivateKeySecretName: ${{ parameters.GitHubAppPrivateKeySecretName }}
           installationOwner: ${{ parameters.GitHubOrg }}
           outputVariableName: 'GitHubAppInstallationToken'
           condition: ${{ parameters.condition }}
@@ -129,16 +121,10 @@ jobs:
           isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }}
           isShouldReusePrSelected: ${{ parameters.ReusePr }}
         packageSourceAuth: patAuth
-        ${{ if and(ne(parameters.CeapexServiceConnection, ''), or(eq(variables['System.TeamProject'], 'internal'), eq(variables['System.TeamProject'], 'DevDiv'))) }}:
-          patVariable: $(CeapexEntraToken)
-        ${{ if or(eq(parameters.CeapexServiceConnection, ''), and(ne(variables['System.TeamProject'], 'internal'), ne(variables['System.TeamProject'], 'DevDiv'))) }}:
-          patVariable: ${{ parameters.CeapexPat }}
+        patVariable: $(CeapexEntraToken)
         ${{ if eq(parameters.RepoType, 'gitHub') }}:
           repoType: ${{ parameters.RepoType }}
-          ${{ if and(eq(parameters.UseGitHubAppAuthentication, true), or(eq(variables['System.TeamProject'], 'internal'), eq(variables['System.TeamProject'], 'DevDiv'), eq(parameters.UseGitHubAppAuthenticationInOtherProjects, true))) }}:
-            gitHubPatVariable: "$(GitHubAppInstallationToken)"
-          ${{ else }}:
-            gitHubPatVariable: "${{ parameters.GithubPat }}"
+          gitHubPatVariable: "$(GitHubAppInstallationToken)"
         ${{ if ne(parameters.MirrorRepo, '') }}:
           isMirrorRepoSelected: true
           gitHubOrganization: ${{ parameters.GitHubOrg }}