diff --git a/NuGet.Config b/NuGet.Config new file mode 100644 index 00000000000..c679a5601f0 --- /dev/null +++ b/NuGet.Config @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/eng/common/pipelines/templates/1es-redirect.yml b/eng/common/pipelines/templates/1es-redirect.yml index 7711cb6d7a1..00be16b2438 100644 --- a/eng/common/pipelines/templates/1es-redirect.yml +++ b/eng/common/pipelines/templates/1es-redirect.yml @@ -27,6 +27,7 @@ extends: parameters: settings: skipBuildTagsForGitHubPullRequests: true + networkIsolationPolicy: Permissive, CFSClean sdl: codeql: compiled: diff --git a/eng/common/pipelines/templates/steps/auth-dev-feed.yml b/eng/common/pipelines/templates/steps/auth-dev-feed.yml new file mode 100644 index 00000000000..fc2396c8448 --- /dev/null +++ b/eng/common/pipelines/templates/steps/auth-dev-feed.yml @@ -0,0 +1,49 @@ +parameters: + DevFeedName: "public/azure-sdk-for-python" + EnableTwineAuth: true + EnablePipAuth: true + EnableUvAuth: true + +steps: + - pwsh: | + # For safety default to publishing to the private feed. + # Publish to https://dev.azure.com/azure-sdk/internal/_packaging?_a=feed&feed=azure-sdk-for-python-pr + $devopsFeedName = 'internal/azure-sdk-for-python-pr' + if (-not ('$(Build.Repository.Name)').EndsWith('-pr')) { + # Publish to https://dev.azure.com/azure-sdk/public/_packaging?_a=feed&feed=azure-sdk-for-python + $devopsFeedName = '${{ parameters.DevFeedName }}' + } + echo "##vso[task.setvariable variable=DevFeedName]$devopsFeedName" + echo "Using DevopsFeed = $devopsFeedName" + displayName: Setup DevOpsFeedName + + - ${{ if eq(parameters.EnableTwineAuth, true) }}: + - task: TwineAuthenticate@0 + displayName: "Twine Authenticate to feed" + inputs: + artifactFeeds: $(DevFeedName) + + - ${{ if eq(parameters.EnablePipAuth, true) }}: + - task: PipAuthenticate@1 + displayName: "Pip Authenticate to feed" + inputs: + artifactFeeds: $(DevFeedName) + onlyAddExtraIndex: false + + - ${{ if eq(parameters.EnableUvAuth, true) }}: + - pwsh: | + if ($env:PIP_INDEX_URL) { + Write-Host "Found pip index URL: $($env:PIP_INDEX_URL)" + # UV_DEFAULT_INDEX is the canonical replacement for the deprecated UV_INDEX_URL (uv 0.4.23+). + # PIP_INDEX_URL is set by PipAuthenticate@1 and contains embedded credentials, which uv + # will use for Basic auth against the ADO feed (and its PyPI upstream) per astral-sh/uv#12651. + Write-Host "##vso[task.setvariable variable=UV_DEFAULT_INDEX]$($env:PIP_INDEX_URL)" + # Disable keyring so uv uses the URL-embedded credentials directly. + Write-Host "##vso[task.setvariable variable=UV_KEYRING_PROVIDER]disabled" + } else { + Write-Host "##[warning]PIP_INDEX_URL not set - uv will fall back to public PyPI." + } + # Force any managed Python downloads to go directly to GitHub releases + # rather than the default CDN (releases.astral.sh). + Write-Host "##vso[task.setvariable variable=UV_PYTHON_INSTALL_MIRROR]https://github.com/astral-sh/python-build-standalone/releases/download" + displayName: "Configure UV Authentication" diff --git a/eng/common/pipelines/templates/steps/maven-authenticate.yml b/eng/common/pipelines/templates/steps/maven-authenticate.yml new file mode 100644 index 00000000000..1b84e984add --- /dev/null +++ b/eng/common/pipelines/templates/steps/maven-authenticate.yml @@ -0,0 +1,17 @@ +parameters: + SourceDirectory: $(Build.SourcesDirectory) + +steps: + # Copy mirror settings to default Maven location so all requests go through CFS + - pwsh: | + $m2Dir = if ($env:USERPROFILE) { "$env:USERPROFILE\.m2" } else { "$HOME/.m2" } + New-Item -ItemType Directory -Force -Path $m2Dir | Out-Null + Copy-Item -Path "${{ parameters.SourceDirectory }}/eng/settings.xml" -Destination "$m2Dir/settings.xml" + displayName: "Setup Maven mirror settings" + + # Authenticate with Azure Artifacts feeds + # MavenAuthenticate adds entries to ~/.m2/settings.xml matching mirror id 'azure-sdk-for-java' + - task: MavenAuthenticate@0 + displayName: "Maven Authenticate" + inputs: + artifactsFeeds: "azure-sdk-for-java" diff --git a/eng/emitters/pipelines/templates/steps/build-step.yml b/eng/emitters/pipelines/templates/steps/build-step.yml index d723de94d4b..a0888cd13cb 100644 --- a/eng/emitters/pipelines/templates/steps/build-step.yml +++ b/eng/emitters/pipelines/templates/steps/build-step.yml @@ -61,6 +61,10 @@ steps: - template: /eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml + - template: /eng/common/pipelines/templates/steps/auth-dev-feed.yml + + - template: /eng/common/pipelines/templates/steps/maven-authenticate.yml + - task: NodeTool@0 displayName: Install Node.js retryCountOnTaskFailure: 3 diff --git a/eng/emitters/pipelines/templates/steps/test-step.yml b/eng/emitters/pipelines/templates/steps/test-step.yml index 59a2beb5689..3a5dfb0687e 100644 --- a/eng/emitters/pipelines/templates/steps/test-step.yml +++ b/eng/emitters/pipelines/templates/steps/test-step.yml @@ -53,6 +53,12 @@ steps: - template: /eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml + - ${{ if eq(parameters.LanguageShortName, 'python') }}: + - template: /eng/common/pipelines/templates/steps/auth-dev-feed.yml + + - ${{ if eq(parameters.LanguageShortName, 'java') }}: + - template: /eng/common/pipelines/templates/steps/maven-authenticate.yml + - download: current artifact: ${{ parameters.BuildArtifactName }} displayName: Download build artifacts diff --git a/eng/settings.xml b/eng/settings.xml new file mode 100644 index 00000000000..b1941716f65 --- /dev/null +++ b/eng/settings.xml @@ -0,0 +1,13 @@ + + + + azure-sdk-for-java + Azure Artifacts Maven Mirror + https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-java/maven/v1 + external:* + + + diff --git a/eng/tsp-core/pipelines/templates/install.yml b/eng/tsp-core/pipelines/templates/install.yml index 7820c3a411a..0f5d4d05ec0 100644 --- a/eng/tsp-core/pipelines/templates/install.yml +++ b/eng/tsp-core/pipelines/templates/install.yml @@ -37,6 +37,6 @@ steps: retryCountOnTaskFailure: 3 - ${{ if parameters.useDotNet }}: - - script: dotnet restore + - script: dotnet restore --configfile "$(Build.SourcesDirectory)/NuGet.Config" displayName: Restore .NET Dependencies workingDirectory: packages/typespec-vs diff --git a/packages/http-client-csharp/eng/scripts/Build-Packages.ps1 b/packages/http-client-csharp/eng/scripts/Build-Packages.ps1 index e606da1c6f5..7f8a2150d1f 100644 --- a/packages/http-client-csharp/eng/scripts/Build-Packages.ps1 +++ b/packages/http-client-csharp/eng/scripts/Build-Packages.ps1 @@ -47,7 +47,8 @@ function Pack-And-Write-Info { ) $versionOption = $BuildNumber ? "/p:Version=$version" : "" - Invoke-LoggedCommand "dotnet pack ./$package/src/$package.csproj $versionOption -c Release -o $outputPath/packages" + $ciNugetAuditArg = $env:TF_BUILD ? "-p:NuGetAudit=false" : "" + Invoke-LoggedCommand "dotnet pack ./$package/src/$package.csproj $versionOption $ciNugetAuditArg -c Release -o $outputPath/packages" Write-PackageInfo -packageName $package -directoryPath "packages/http-client-csharp/generator/$package/src" -version $version } diff --git a/packages/http-client-csharp/eng/scripts/Test-Packages.ps1 b/packages/http-client-csharp/eng/scripts/Test-Packages.ps1 index bb0c24c43fe..82959cca970 100644 --- a/packages/http-client-csharp/eng/scripts/Test-Packages.ps1 +++ b/packages/http-client-csharp/eng/scripts/Test-Packages.ps1 @@ -24,8 +24,14 @@ try { # enforce cop static-analysis rules on the generator sources Invoke-LoggedCommand "./eng/scripts/Invoke-Cop.ps1" -GroupOutput + # Use the repo NuGet.Config explicitly so restore always targets the Azure DevOps feed. + $repoRoot = (Resolve-Path "$packageRoot/../..").Path + $nugetConfigPath = (Join-Path $repoRoot "NuGet.Config") + $ciNugetAuditArg = $env:TF_BUILD ? "-p:NuGetAudit=false" : "" + Invoke-LoggedCommand "dotnet nuget list source --configfile `"$nugetConfigPath`"" -GroupOutput + # test the generator - Invoke-LoggedCommand "dotnet test ./generator" -GroupOutput + Invoke-LoggedCommand "dotnet test ./generator -p:RestoreConfigFile=`"$nugetConfigPath`" $ciNugetAuditArg" -GroupOutput Invoke-LoggedCommand "./eng/scripts/Get-Spector-Coverage.ps1" -GroupOutput } diff --git a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/test/Utilities/ExternalTypeReferenceResolverTests.cs b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/test/Utilities/ExternalTypeReferenceResolverTests.cs index a507018bace..50056b82802 100644 --- a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/test/Utilities/ExternalTypeReferenceResolverTests.cs +++ b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/test/Utilities/ExternalTypeReferenceResolverTests.cs @@ -20,6 +20,7 @@ public class ExternalTypeReferenceResolverTests { private string? _tempDirectory; private string? _projectDir; + private string? _nugetCacheDir; private string? _originalNugetPackageDir; [SetUp] @@ -27,14 +28,18 @@ public void Setup() { _tempDirectory = Path.Combine(Path.GetTempPath(), "TestArtifacts", Guid.NewGuid().ToString()); _projectDir = Path.Combine(_tempDirectory, "ProjectDir"); - var nugetCacheDir = Path.Combine(_tempDirectory, "NuGetCache"); + _nugetCacheDir = Path.Combine(_tempDirectory, "NuGetCache"); Directory.CreateDirectory(Path.Combine(_projectDir, "src")); - Directory.CreateDirectory(nugetCacheDir); + Directory.CreateDirectory(_nugetCacheDir); + + // Ensure these tests are fully isolated from machine/user NuGet settings and never probe + // external feeds when package lookup falls through to source probing. + WriteLocalNuGetConfig(_projectDir, _nugetCacheDir); _originalNugetPackageDir = Environment.GetEnvironmentVariable("NUGET_PACKAGES", EnvironmentVariableTarget.Process); - Environment.SetEnvironmentVariable("NUGET_PACKAGES", nugetCacheDir, EnvironmentVariableTarget.Process); + Environment.SetEnvironmentVariable("NUGET_PACKAGES", _nugetCacheDir, EnvironmentVariableTarget.Process); - ExternalTypeReferenceResolver.Reset(); + ResetResolverIfInitialized(); MockHelpers.LoadMockGenerator( outputPath: _projectDir, configuration: "{}"); @@ -43,7 +48,7 @@ public void Setup() [TearDown] public void Cleanup() { - ExternalTypeReferenceResolver.Reset(); + ResetResolverIfInitialized(); Directory.Delete(_tempDirectory!, true); Environment.SetEnvironmentVariable("NUGET_PACKAGES", _originalNugetPackageDir, EnvironmentVariableTarget.Process); } @@ -64,10 +69,9 @@ public void TryResolve_ReturnsNullWhenPackageMissing() [Test] public void TryResolve_LoadsTypeFromNuGetCache() { - var nugetCacheDir = Path.Combine(_tempDirectory!, "NuGetCache"); const string pkgName = "Test.External.Loadable"; const string typeName = "Test.External.Loadable.LoadableType"; - CreateFakeNuGetPackage(nugetCacheDir, pkgName, "1.2.3"); + CreateFakeNuGetPackage(_nugetCacheDir!, pkgName, "1.2.3"); var external = new InputExternalTypeMetadata(typeName, pkgName, null); @@ -80,14 +84,13 @@ public void TryResolve_LoadsTypeFromNuGetCache() [Test] public void TryResolve_PrefersHighestCachedVersionAtOrAboveMinVersion() { - var nugetCacheDir = Path.Combine(_tempDirectory!, "NuGetCache"); const string pkgName = "Test.MultiVersion.Package"; const string typeName = "Test.MultiVersion.Package.SomeType"; // Create three cached versions; MinVersion=2.0.0 must skip 1.0.0 and pick 3.0.0 (highest >= MinVersion). - CreateFakeNuGetPackage(nugetCacheDir, pkgName, "1.0.0"); - CreateFakeNuGetPackage(nugetCacheDir, pkgName, "2.5.0"); - CreateFakeNuGetPackage(nugetCacheDir, pkgName, "3.0.0"); + CreateFakeNuGetPackage(_nugetCacheDir!, pkgName, "1.0.0"); + CreateFakeNuGetPackage(_nugetCacheDir!, pkgName, "2.5.0"); + CreateFakeNuGetPackage(_nugetCacheDir!, pkgName, "3.0.0"); var external = new InputExternalTypeMetadata(typeName, pkgName, "2.0.0"); var resolved = ExternalTypeReferenceResolver.TryResolve(external); @@ -104,10 +107,9 @@ public void TryResolve_PrefersHighestCachedVersionAtOrAboveMinVersion() [Test] public void TryResolve_AddsMetadataReferenceOnce() { - var nugetCacheDir = Path.Combine(_tempDirectory!, "NuGetCache"); const string pkgName = "Test.MetadataRef.Package"; const string typeName = "Test.MetadataRef.Package.RefType"; - CreateFakeNuGetPackage(nugetCacheDir, pkgName, "1.0.0"); + CreateFakeNuGetPackage(_nugetCacheDir!, pkgName, "1.0.0"); var external = new InputExternalTypeMetadata(typeName, pkgName, null); @@ -141,10 +143,9 @@ public void TryResolve_ReturnsNullForUnknownPackage() [Test] public async Task ResolveAllAsync_ResolvesExternalTypesFromInputLibrary() { - var nugetCacheDir = Path.Combine(_tempDirectory!, "NuGetCache"); const string pkgName = "Test.PreWalk.Package"; const string typeName = "Test.PreWalk.Package.PreWalkType"; - CreateFakeNuGetPackage(nugetCacheDir, pkgName, "1.0.0"); + CreateFakeNuGetPackage(_nugetCacheDir!, pkgName, "1.0.0"); var external = new InputExternalTypeMetadata(typeName, pkgName, null); var unionWithExternal = InputFactory.Union( @@ -188,5 +189,35 @@ private static string CreateFakeNuGetPackage(string nugetCacheDir, string packag .Replace("$VERSION$", version); return FakeNuGetPackage.Create(nugetCacheDir, packageName, version, source); } + + private static void WriteLocalNuGetConfig(string projectDir, string nugetCacheDir) + { + var nugetConfigPath = Path.Combine(projectDir, "NuGet.Config"); + var normalizedCachePath = nugetCacheDir.Replace("\\", "/"); + var config = $@" + + + + + + + + +"; + + File.WriteAllText(nugetConfigPath, config); + } + + private static void ResetResolverIfInitialized() + { + try + { + ExternalTypeReferenceResolver.Reset(); + } + catch (InvalidOperationException) + { + // CodeModelGenerator may not be initialized yet when this fixture runs in isolation. + } + } } } diff --git a/packages/typespec-vs/scripts/build.js b/packages/typespec-vs/scripts/build.js index 590200a368a..efb7161f2f4 100644 --- a/packages/typespec-vs/scripts/build.js +++ b/packages/typespec-vs/scripts/build.js @@ -34,7 +34,15 @@ async function main() { if (result.type === "dotnet") { await ensureDotnetVersion({ exitWithSuccessInDevBuilds: true }); await runDotnet( - ["build", "--configuration", "Release", `-p:Version=${version}`, "-clp:NoSummary"], + [ + "build", + "--configuration", + "Release", + `-p:Version=${version}`, + "-clp:NoSummary", + // Restore may fail with NU1900 when vulnerability feed is unavailable. + "-p:NuGetAudit=false", + ], { cwd: pkgRoot, }, @@ -58,7 +66,9 @@ async function buildWithMsbuild(msbuildPath, pkgRoot, version) { msbuildArgs.push("/restore"); } msbuildArgs.push(join(pkgRoot, "Microsoft.TypeSpec.VS.sln")); - const result = await run(msbuildPath, msbuildArgs, { throwOnNonZeroExit: false }); + const result = await run(msbuildPath, msbuildArgs, { + throwOnNonZeroExit: false, + }); process.exit(result.exitCode); } diff --git a/packages/typespec-vs/src/Microsoft.TypeSpec.VS.csproj b/packages/typespec-vs/src/Microsoft.TypeSpec.VS.csproj index 71611c8aa97..50ef30a7dbc 100644 --- a/packages/typespec-vs/src/Microsoft.TypeSpec.VS.csproj +++ b/packages/typespec-vs/src/Microsoft.TypeSpec.VS.csproj @@ -10,6 +10,8 @@ all + + false true 42.42.42