diff --git a/eng/common/pipelines/templates/steps/check-spelling.yml b/eng/common/pipelines/templates/steps/check-spelling.yml index d5faccdbd7..6c8f6a8b62 100644 --- a/eng/common/pipelines/templates/steps/check-spelling.yml +++ b/eng/common/pipelines/templates/steps/check-spelling.yml @@ -15,7 +15,8 @@ # npm commands in the context of this step. If specified, this # will be set as the npm_config_userconfig environment variable # so that npm uses the provided config file instead of the default -# user config. +# user config. If not specified, this template creates and +# authenticates a temporary .npmrc and uses that file. # This check recognizes the setting of variable "Skip.SpellCheck" # if set to 'true', spellchecking will not be invoked. @@ -35,6 +36,11 @@ parameters: steps: - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + - ${{ if eq(parameters.NpmConfigUserConfig, '') }}: + - template: /eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml + parameters: + npmrcPath: $(Agent.TempDirectory)/check-spelling/.npmrc + - task: PowerShell@2 displayName: Check spelling (cspell) condition: and(succeeded(), ne(variables['Skip.SpellCheck'],'true')) @@ -49,6 +55,8 @@ steps: env: ${{ if ne(parameters.NpmConfigUserConfig, '') }}: npm_config_userconfig: ${{ parameters.NpmConfigUserConfig }} + ${{ if eq(parameters.NpmConfigUserConfig, '') }}: + npm_config_userconfig: "$(Agent.TempDirectory)/check-spelling/.npmrc" - ${{ if ne('', parameters.ScriptToValidateUpgrade) }}: - pwsh: | $changedFiles = ./eng/common/scripts/get-changedfiles.ps1 diff --git a/eng/common/scripts/Verify-Links.ps1 b/eng/common/scripts/Verify-Links.ps1 index da8eca8c89..b090587955 100644 --- a/eng/common/scripts/Verify-Links.ps1 +++ b/eng/common/scripts/Verify-Links.ps1 @@ -167,28 +167,60 @@ function ProcessCratesIoLink([System.Uri]$linkUri, $path) { } function ProcessNpmLink([System.Uri]$linkUri) { - # npmjs.com started using Cloudflare which returns 403 and we need to instead check the registry api for existence checks - # https://github.com/orgs/community/discussions/174098#discussioncomment-14461226 - - # Handle versioned URLs: https://www.npmjs.com/package/@azure/ai-agents/v/1.1.0 -> https://registry.npmjs.org/@azure/ai-agents/1.1.0 - # Handle non-versioned URLs: https://www.npmjs.com/package/@azure/ai-agents -> https://registry.npmjs.org/@azure/ai-agents - # The regex captures the package name (which may contain a slash for scoped packages) and optionally the version. - # Query parameters and URL fragments are excluded from the transformation. + # npmjs.com links are verified via the Azure DevOps public feed upstream API to avoid + # direct calls to registry.npmjs.org or npmjs.com under CFS network isolation. + # Upstream versions reflect what npmjs.org publishes, not just what the feed has cached. + # + # Handle versioned URLs: https://www.npmjs.com/package/@azure/ai-agents/v/1.1.0 + # -> checks that version 1.1.0 is present in upstream via ADO feed API + # Handle non-versioned URLs: https://www.npmjs.com/package/@azure/ai-agents + # -> checks that at least one upstream version exists via ADO feed API + # + # ADO feed upstream API: https://pkgs.dev.azure.com/azure-sdk/public/_apis/packaging/feeds/azure-sdk-for-js/npm/packages/{package}/upstreamVersions $urlString = $linkUri.ToString() + $packageName = $null + $version = $null + if ($urlString -match '^https?://(?:www\.)?npmjs\.com/package/([^?#]+)/v/([^?#]+)') { - # Versioned URL: remove the /v/ segment but keep the version - $apiUrl = "https://registry.npmjs.org/$($matches[1])/$($matches[2])" + # Versioned URL: e.g. https://www.npmjs.com/package/@azure/ai-agents/v/1.1.0 + $packageName = $matches[1] + $version = $matches[2] } elseif ($urlString -match '^https?://(?:www\.)?npmjs\.com/package/([^?#]+)') { - # Non-versioned URL: just replace the domain - $apiUrl = "https://registry.npmjs.org/$($matches[1])" + # Non-versioned URL: e.g. https://www.npmjs.com/package/@azure/ai-agents + $packageName = $matches[1] } else { - # Fallback: use the original URL if it doesn't match expected patterns - $apiUrl = $urlString + Write-Verbose "Could not parse npm package name from $linkUri, skipping network verification" + return $true } - return ProcessStandardLink ([System.Uri]$apiUrl) + # Scoped package names (e.g. @azure/ai-agents) must be percent-encoded for the path segment + $encodedPackageName = [System.Uri]::EscapeDataString($packageName) + $upstreamApiUrl = "https://pkgs.dev.azure.com/azure-sdk/public/_apis/packaging/feeds/azure-sdk-for-js/npm/packages/$encodedPackageName/upstreamVersions?api-version=7.1-preview.1" + + Write-Verbose "Checking npm package '$packageName' via ADO upstream feed: $upstreamApiUrl" + + # Invoke-RestMethod throws on non-2xx (e.g. 404 for unknown package); CheckLink's catch block handles it + $response = Invoke-RestMethod -Uri $upstreamApiUrl -Method GET -UserAgent $userAgent -TimeoutSec $requestTimeoutSec + + if ($version) { + # Versioned: the specific version must exist in upstream + $versionExists = $response.value | Where-Object { $_.version -eq $version } + if (!$versionExists) { + Write-Host "Version '$version' of npm package '$packageName' not found in ADO upstream" + return $false + } + } + else { + # Non-versioned: at least one upstream version must exist + if (!$response.value -or $response.value.Count -eq 0) { + Write-Host "npm package '$packageName' has no upstream versions in ADO feed" + return $false + } + } + + return $true } function ProcessStandardLink([System.Uri]$linkUri) { diff --git a/eng/common/scripts/tests/Verify-Links.Tests.ps1 b/eng/common/scripts/tests/Verify-Links.Tests.ps1 new file mode 100644 index 0000000000..4e0e4bed1c --- /dev/null +++ b/eng/common/scripts/tests/Verify-Links.Tests.ps1 @@ -0,0 +1,186 @@ +# Run tests: +# Install-Module -Name Pester -Force -SkipPublisherCheck +# Invoke-Pester -Passthru $PSScriptRoot/Verify-Links.Tests.ps1 + +BeforeAll { + # Load only the functions we need by dot-sourcing the script with a dummy param block. + # We source the script file and then override Invoke-RestMethod / Invoke-WebRequest + # inside each test via Mock so no real network calls are made. + + # Load shared logging functions used by Verify-Links.ps1 + . (Resolve-Path "$PSScriptRoot/../logging.ps1").Path + + # Source only the function definitions (stop before the script-body runs) + # by dot-sourcing within a script block that replaces the parameter-driven + # body with a no-op after loading. + $scriptPath = (Resolve-Path "$PSScriptRoot/../Verify-Links.ps1").Path + + # Extract and invoke only the function definitions by parsing the AST + $ast = [System.Management.Automation.Language.Parser]::ParseFile($scriptPath, [ref]$null, [ref]$null) + $functionDefs = $ast.FindAll({ $args[0] -is [System.Management.Automation.Language.FunctionDefinitionAst] }, $false) + foreach ($fn in $functionDefs) { + Invoke-Expression $fn.Extent.Text + } + + # Script-scope variables referenced inside the functions + $script:userAgent = "TestAgent/1.0" + $script:requestTimeoutSec = 15 +} + +Describe "ProcessNpmLink" { + Context "Unparseable URL" { + It "Returns true and skips verification for a URL that does not match npmjs.com package pattern" { + $result = ProcessNpmLink ([System.Uri]"https://npmjs.com/browse/keyword/azure") + $result | Should -Be $true + } + } + + Context "Non-versioned URL - package found in ADO upstream" { + BeforeEach { + Mock Invoke-RestMethod { + return @{ + value = @( + @{ version = "1.0.0" }, + @{ version = "2.0.0" } + ) + } + } -ParameterFilter { $Uri -like "*upstreamVersions*" } + } + + It "Returns true when package has upstream versions" { + $result = ProcessNpmLink ([System.Uri]"https://www.npmjs.com/package/@azure/ai-agents") + $result | Should -Be $true + } + + It "Encodes scoped package name correctly in the API URL" { + ProcessNpmLink ([System.Uri]"https://www.npmjs.com/package/@azure/ai-agents") | Out-Null + Should -Invoke Invoke-RestMethod -ParameterFilter { + $Uri -like "*%40azure%2Fai-agents*" + } -Times 1 -Exactly + } + + It "Calls the ADO feed upstream API, not registry.npmjs.org or npmjs.com" { + ProcessNpmLink ([System.Uri]"https://www.npmjs.com/package/@azure/core-http") | Out-Null + Should -Invoke Invoke-RestMethod -ParameterFilter { + $Uri -like "https://pkgs.dev.azure.com/azure-sdk/public/_apis/packaging/feeds/azure-sdk-for-js/npm/packages/*" + } -Times 1 -Exactly + } + } + + Context "Non-versioned URL - package not found (no upstream versions)" { + BeforeEach { + Mock Invoke-RestMethod { + return @{ value = @() } + } -ParameterFilter { $Uri -like "*upstreamVersions*" } + } + + It "Returns false when the package has no upstream versions" { + $result = ProcessNpmLink ([System.Uri]"https://www.npmjs.com/package/@azure/nonexistent-package") + $result | Should -Be $false + } + } + + Context "Non-versioned URL - package not found (ADO returns 404)" { + BeforeEach { + Mock Invoke-RestMethod { + $response = [System.Net.HttpWebResponse]::new.Invoke(@()) + throw [System.Net.WebException]::new( + "The remote server returned an error: (404) Not Found.", + $null, + [System.Net.WebExceptionStatus]::ProtocolError, + $null + ) + } -ParameterFilter { $Uri -like "*upstreamVersions*" } + } + + It "Propagates exception (to be handled by CheckLink caller)" { + { ProcessNpmLink ([System.Uri]"https://www.npmjs.com/package/@azure/nonexistent-package") } | Should -Throw + } + } + + Context "Versioned URL - correct version present in ADO upstream" { + BeforeEach { + Mock Invoke-RestMethod { + return @{ + value = @( + @{ version = "1.0.0" }, + @{ version = "1.1.0" }, + @{ version = "2.0.0" } + ) + } + } -ParameterFilter { $Uri -like "*upstreamVersions*" } + } + + It "Returns true when the specific version exists in upstream" { + $result = ProcessNpmLink ([System.Uri]"https://www.npmjs.com/package/@azure/ai-agents/v/1.1.0") + $result | Should -Be $true + } + + It "Encodes scoped package name correctly for versioned URLs" { + ProcessNpmLink ([System.Uri]"https://www.npmjs.com/package/@azure/ai-agents/v/1.1.0") | Out-Null + Should -Invoke Invoke-RestMethod -ParameterFilter { + $Uri -like "*%40azure%2Fai-agents*" + } -Times 1 -Exactly + } + } + + Context "Versioned URL - version not present in ADO upstream" { + BeforeEach { + Mock Invoke-RestMethod { + return @{ + value = @( + @{ version = "1.0.0" }, + @{ version = "2.0.0" } + ) + } + } -ParameterFilter { $Uri -like "*upstreamVersions*" } + } + + It "Returns false when the specific version does not exist in upstream" { + $result = ProcessNpmLink ([System.Uri]"https://www.npmjs.com/package/@azure/ai-agents/v/9999.9.9") + $result | Should -Be $false + } + } + + Context "Versioned URL - unscoped package" { + BeforeEach { + Mock Invoke-RestMethod { + return @{ + value = @( + @{ version = "3.2.1" } + ) + } + } -ParameterFilter { $Uri -like "*upstreamVersions*" } + } + + It "Handles unscoped package names in versioned URLs" { + $result = ProcessNpmLink ([System.Uri]"https://www.npmjs.com/package/typescript/v/3.2.1") + $result | Should -Be $true + } + + It "Returns false when unscoped versioned package version is missing" { + $result = ProcessNpmLink ([System.Uri]"https://www.npmjs.com/package/typescript/v/9999.0.0") + $result | Should -Be $false + } + } + + Context "Query parameters and fragments are excluded from package name" { + BeforeEach { + Mock Invoke-RestMethod { + return @{ + value = @( + @{ version = "1.0.0" } + ) + } + } -ParameterFilter { $Uri -like "*upstreamVersions*" } + } + + It "Strips query string from non-versioned URL when extracting package name" { + $result = ProcessNpmLink ([System.Uri]"https://www.npmjs.com/package/@azure/ai-agents?activeTab=readme") + $result | Should -Be $true + Should -Invoke Invoke-RestMethod -ParameterFilter { + $Uri -like "*%40azure%2Fai-agents*" -and $Uri -notlike "*activeTab*" + } -Times 1 -Exactly + } + } +} diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index b6039aad3d..3b41b9a8d2 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -162,8 +162,14 @@ jobs: -DBUILD_TRANSPORT_CURL=OFF -DBUILD_DOCUMENTATION=YES + - template: /eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml + parameters: + npmrcPath: $(Agent.TempDirectory)/generate-release/.npmrc + - pwsh: npm install -g moxygen displayName: Install Moxygen to generate markdown for learn.microsoft.com + env: + npm_config_userconfig: "$(Agent.TempDirectory)/generate-release/.npmrc" - pwsh: | Write-Host "Using apiview parser version ${{ variables.apiviewParserVersion }}" @@ -276,6 +282,8 @@ jobs: ignoreLASTEXITCODE: true pwsh: true displayName: Generate ${{ artifact.Name }} artifacts for docs.ms + env: + npm_config_userconfig: "$(Agent.TempDirectory)/generate-release/.npmrc" - task: Powershell@2 inputs: