diff --git a/.github/instructions/gradle.instructions.md b/.github/instructions/gradle.instructions.md index 805e8a9649d..b33da144d31 100644 --- a/.github/instructions/gradle.instructions.md +++ b/.github/instructions/gradle.instructions.md @@ -12,9 +12,6 @@ All `src/*` Gradle projects share two repo config files: **`eng/gradle/plugin-re pluginManagement { apply from: "${rootDir}/../../eng/gradle/plugin-repositories.gradle", to: pluginManagement } -if (System.getenv('ANDROID_MIRROR_MAVEN_DEPENDENCIES') == 'true') { - apply from: "${rootDir}/../../eng/gradle/credential-provider.gradle" -} dependencyResolutionManagement { apply from: "${rootDir}/../../eng/gradle/dependency-repositories.gradle", to: dependencyResolutionManagement } @@ -31,9 +28,9 @@ Both files switch on `System.getenv('RUNNINGONCI')`. Azure DevOps exports the - **`RUNNINGONCI=true`** (Azure DevOps, sourced from `RunningOnCI` in `build-tools/automation/yaml-templates/variables.yaml`) → dnceng `dotnet-public-maven` feed (CFSClean isolation, https://aka.ms/1es/netiso/CFS). Anonymous read of cached packages. - **unset** (local, Dependabot, GitHub Actions) → `google()` + `mavenCentral()` + `gradlePluginPortal()` for plugins, `google()` + `mavenCentral()` for deps. No credentials needed. -CI reads cached packages from the mirror anonymously. The Azure Artifacts -credential provider is loaded only when `ANDROID_MIRROR_MAVEN_DEPENDENCIES=true`; -`mirror-dependencies.ps1` sets this while seeding uncached packages. +CI reads cached packages from the mirror anonymously. `mirror-dependencies.ps1` +runs the same anonymous Gradle resolution, then seeds each missing URL with an +authenticated HTTP request until the build succeeds. Test the CI path locally: `$env:RUNNINGONCI='true'` (PowerShell) or `RUNNINGONCI=true ...` (bash). @@ -41,7 +38,7 @@ Test the CI path locally: `$env:RUNNINGONCI='true'` (PowerShell) or `RUNNINGONCI The new package isn't cached in the dnceng `dotnet-public-maven` feed yet. CI agents only do anonymous reads, so someone has to authenticate once locally to make the feed pull the package (and its transitive deps) from upstream. -Use the helper script — it runs the build, parses any 401 URLs out of the log, re-fetches each one with an Azure DevOps bearer token (so the feed mirrors it), and loops until the build succeeds: +Use the helper script — it runs the build, parses any 401 URLs out of the log, re-fetches each one with an Azure DevOps OAuth token using Basic authentication (so the feed mirrors it), and loops until the build succeeds: ```powershell az login # one-time, corp account with MFA satisfied @@ -57,6 +54,16 @@ The mirror must run in the project that actually needs the new package — a sib After it succeeds, just re-run the failed CI job. No PR edits needed — the packages are now anonymous-readable forever. +Tests that resolve Maven files without Gradle can seed coordinates directly: + +```powershell +pwsh ./eng/gradle/mirror-dependencies.ps1 ` + -MavenArtifact 'androidx.core:core:1.12.0' +``` + +This attempts the coordinate's POM, JAR, AAR, and Gradle module metadata. Append +the exact filename as a fourth segment for a nonstandard payload. + ## Don'ts - Don't hard-code Maven repo URLs in `build.gradle` / `settings.gradle`; use the shared file. diff --git a/build-tools/scripts/Prepare.proj b/build-tools/scripts/Prepare.proj index f2622ff1313..8c51b180699 100644 --- a/build-tools/scripts/Prepare.proj +++ b/build-tools/scripts/Prepare.proj @@ -19,15 +19,14 @@ Text="The specified `%24(AndroidToolchainDirectory)` '$(AndroidToolchainDirectory)' contains a space. Android NDK commands do not support this. Please create a Configuration.Override.props file that sets the AndroidToolchainDirectory property to a different path." Condition=" '$(HostOS)' == 'Windows' And $(AndroidToolchainDirectory.Contains (' '))" /> - + - -[CmdletBinding()] +[CmdletBinding(DefaultParameterSetName='Gradle')] param( - [Parameter(Mandatory=$true)] - [string] $ProjectDir, + [Parameter(Mandatory=$true, ParameterSetName='Gradle')] + [string] $ProjectDir = '.', - [Parameter(Mandatory=$true)] + [Parameter(Mandatory=$true, ParameterSetName='Gradle')] [string] $Task, + [Parameter(Mandatory=$true, ParameterSetName='MavenArtifact')] + [string[]] $MavenArtifact, + + [Parameter(ParameterSetName='Gradle')] [string] $GradleWrapper, + [Parameter(ParameterSetName='Gradle')] [string] $AndroidHome = $env:ANDROID_HOME, + [Parameter(ParameterSetName='Gradle')] [int] $MaxIterations = 15 ) @@ -104,13 +122,14 @@ function Get-AzDevOpsToken { } function Invoke-Mirror($logPath) { - $urls = Select-String -Path $logPath -Pattern "Could not GET 'https://pkgs\.dev\.azure\.com/dnceng/[^']+'" -AllMatches | + $urls = Select-String -Path $logPath -Pattern "Could not (?:GET|HEAD) '(https://pkgs\.dev\.azure\.com/dnceng/[^']+)'" -AllMatches | ForEach-Object { $_.Matches } | - ForEach-Object { $_.Value -replace "^Could not GET '", "" -replace "'$", "" } | + ForEach-Object { $_.Groups[1].Value } | Sort-Object -Unique if ($urls.Count -eq 0) { return 0 } $token = Get-AzDevOpsToken - $headers = @{ Authorization = "Bearer $token" } + $basicCredential = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(":$token")) + $headers = @{ Authorization = "Basic $basicCredential" } $ok = 0; $fail = 0 foreach ($u in $urls) { try { @@ -125,18 +144,59 @@ function Invoke-Mirror($logPath) { return $urls.Count } +function Get-MavenArtifactUrls($artifacts) { + $feedBaseUrl = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-maven/maven/v1' + foreach ($artifact in $artifacts) { + $parts = $artifact.Split(':', 4) + if ($parts.Count -lt 3) { + throw "Invalid Maven artifact '$artifact'. Expected group:artifact:version[:filename]." + } + $group = $parts[0].Replace('.', '/') + $name = $parts[1] + $version = $parts[2] + $filenames = if ($parts.Count -eq 4) { + @($parts[3]) + } else { + @( + "$name-$version.pom" + "$name-$version.jar" + "$name-$version.aar" + "$name-$version.module" + ) + } + foreach ($filename in $filenames) { + "$feedBaseUrl/$group/$name/$version/$filename" + } + } +} + +# Verify az is available and authenticated up front so we fail fast. +Get-AzDevOpsToken | Out-Null + +if ($PSCmdlet.ParameterSetName -eq 'MavenArtifact') { + Write-Host "Mirroring Maven artifacts directly:" + $MavenArtifact | ForEach-Object { Write-Host " $_" } + $log = Join-Path ([IO.Path]::GetTempPath()) 'maven-artifact-mirror.log' + try { + Get-MavenArtifactUrls $MavenArtifact | + ForEach-Object { "Could not GET '$_'" } | + Set-Content $log + Invoke-Mirror $log | Out-Null + } + finally { + Remove-Item $log -ErrorAction SilentlyContinue + } + return +} + Write-Host "Repo root: $repoRoot" Write-Host "Project: $projectDirAbs" Write-Host "Task: $Task" Write-Host "Gradle: $gradlew" if ($AndroidHome) { Write-Host "ANDROID_HOME: $AndroidHome" } -# Verify az is available and authenticated up front so we fail fast. -Get-AzDevOpsToken | Out-Null - if ($AndroidHome) { $env:ANDROID_HOME = $AndroidHome } $env:RUNNINGONCI = 'true' -$env:ANDROID_MIRROR_MAVEN_DEPENDENCIES = 'true' Push-Location $projectDirAbs try { diff --git a/external/Java.Interop/tests/Java.Interop.Tools.Maven-Tests/Extensions/MavenProjectResolver.cs b/external/Java.Interop/tests/Java.Interop.Tools.Maven-Tests/Extensions/MavenProjectResolver.cs index 1d208c32716..37ca8423a3b 100644 --- a/external/Java.Interop/tests/Java.Interop.Tools.Maven-Tests/Extensions/MavenProjectResolver.cs +++ b/external/Java.Interop/tests/Java.Interop.Tools.Maven-Tests/Extensions/MavenProjectResolver.cs @@ -8,6 +8,7 @@ namespace Java.Interop.Tools.Maven_Tests.Extensions; class MavenProjectResolver : IProjectResolver { + const string DotNetPublicMaven = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-maven/maven/v1"; readonly IMavenRepository repository; public MavenProjectResolver (IMavenRepository repository) @@ -19,10 +20,15 @@ static MavenProjectResolver () { var cache_path = Path.Combine (Environment.GetFolderPath (Environment.SpecialFolder.LocalApplicationData), "dotnet-android", "MavenCacheDirectory"); - Central = new MavenProjectResolver (new CachedMavenRepository (cache_path, MavenRepository.Central)); - Google = new MavenProjectResolver (new CachedMavenRepository (cache_path, MavenRepository.Google)); + Central = new MavenProjectResolver (new CachedMavenRepository (cache_path, GetRepository (MavenRepository.Central, "central"))); + Google = new MavenProjectResolver (new CachedMavenRepository (cache_path, GetRepository (MavenRepository.Google, "google"))); } + static MavenRepository GetRepository (MavenRepository localRepository, string name) => + string.Equals (Environment.GetEnvironmentVariable ("RUNNINGONCI"), "true", StringComparison.OrdinalIgnoreCase) + ? new MavenRepository (DotNetPublicMaven, name) + : localRepository; + public Project Resolve (Artifact artifact) { if (repository.TryGetFile (artifact, $"{artifact.Id}-{artifact.Version}.pom", out var stream)) { diff --git a/external/Java.Interop/tests/Xamarin.Android.Tools.Bytecode-Tests/kotlin-gradle.targets b/external/Java.Interop/tests/Xamarin.Android.Tools.Bytecode-Tests/kotlin-gradle.targets index 2a2313bd079..52334d8a425 100644 --- a/external/Java.Interop/tests/Xamarin.Android.Tools.Bytecode-Tests/kotlin-gradle.targets +++ b/external/Java.Interop/tests/Xamarin.Android.Tools.Bytecode-Tests/kotlin-gradle.targets @@ -2,10 +2,9 @@