diff --git a/README.md b/README.md index 288f73f..0f2e962 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,22 @@ Windows Server hosts use an exact LTSC mapping: Server 2019 selects `ltsc2019`, Linux releases continue to use `proxy-agent:` for backward compatibility and are also published as `proxy-agent:-ubuntu-26.04`. Both tags reference the same Linux multi-architecture image. Windows releases use the explicit `proxy-agent:-windows-ltsc2019`, `proxy-agent:-windows-ltsc2022`, or `proxy-agent:-windows-ltsc2025` tags. +### Installing behind a network proxy + +Both installers and the `upgrade` command honor `HTTPS_PROXY` (or `HTTP_PROXY`) and `NO_PROXY` for the web requests they make: downloading the management script, listing image tags, and fetching the agent configuration from Fivetran. On Linux, `curl` reads these variables natively. On Windows, Windows PowerShell 5.1 does not, so `install.ps1` and `proxy-agent-manager.ps1` apply them themselves. The URL may include credentials: `http://user:pass@proxy.example.com:3128`. + +```powershell +$env:HTTPS_PROXY = 'http://proxy.example.com:3128' +$env:NO_PROXY = 'localhost,127.0.0.1,.example.com' +$env:RUNTIME = 'docker'; $env:TOKEN = 'YOUR_AGENT_TOKEN'; & .\install.ps1 +``` + +On Windows, the variables take precedence over the proxy in Internet Options; without them, the scripts use the Internet Options proxy as any PowerShell web request would, and print it when one is in effect. In both cases a proxy that asks for NTLM or Kerberos authentication is answered with the current Windows identity, unless the URL carries its own credentials. + +On Windows, download `install.ps1` first, as shown above, rather than fetching it inline with `Invoke-RestMethod`: the download itself runs before the script can apply the proxy settings. + +These variables do not configure Docker. The Docker daemon needs its own proxy settings to pull the agent image, and the agent needs `proxy_host` and `proxy_port` in `config.json` to reach Fivetran through a proxy at runtime. See the [Proxy Agent documentation](https://fivetran.com/docs/destinations/connection-options/proxy-agent) for both. + The installer will: - Check prerequisites - Create the installation directory diff --git a/install.ps1 b/install.ps1 index 554bd30..4bb6db8 100644 --- a/install.ps1 +++ b/install.ps1 @@ -47,10 +47,70 @@ Usage: Options: -InstallDir Installation directory (default: %USERPROFILE%\fivetran-proxy-agent) -WindowsVersion Windows LTSC image variant for Windows containers; detected from the host when omitted + +Environment: + HTTPS_PROXY / HTTP_PROXY Proxy URL for the installer's web requests (http://[user:pass@]host:port) + NO_PROXY Comma-separated hosts that bypass the proxy '@ exit 1 } +# Windows PowerShell 5.1 ignores HTTP_PROXY/HTTPS_PROXY/NO_PROXY, so apply them to this +# script's web requests. curl honors the same variables, so install.sh and install.ps1 +# behave alike behind a proxy. Without them, .NET's default proxy is the user's Internet +# Options proxy, which is kept. Either way the proxy may answer NTLM/Kerberos challenges +# with the current Windows identity, which .NET doesn't do on its own. Docker itself reads +# its own proxy settings; see the README. +function Set-WebProxyFromEnvironment { + param([Parameter(Mandatory)][Uri]$ProbeUri) + + $proxyUrl = if ($env:HTTPS_PROXY) { $env:HTTPS_PROXY } elseif ($env:HTTP_PROXY) { $env:HTTP_PROXY } else { $null } + if (-not $proxyUrl) { + $systemProxy = [System.Net.WebRequest]::DefaultWebProxy + if ($systemProxy) { + $systemProxy.Credentials = [System.Net.CredentialCache]::DefaultCredentials + $effectiveProxy = $systemProxy.GetProxy($ProbeUri) + # Direct connections come back as the probe URI on .NET Framework and as null on .NET Core. + if ($effectiveProxy -and $effectiveProxy -ne $ProbeUri) { + Write-Host "Using system proxy $effectiveProxy" + } + } + return + } + + try { + $proxyUri = [Uri]$proxyUrl + } catch { + Write-Host "ERROR: Invalid proxy URL in HTTPS_PROXY/HTTP_PROXY: $proxyUrl" -ForegroundColor Red + exit 1 + } + + # .NET bypass entries are regular expressions matched against the request URL. As with curl, + # each entry matches the host and its subdomains, with or without a leading dot. CIDR ranges aren't supported. + $bypassList = @() + if ($env:NO_PROXY) { + foreach ($entry in ($env:NO_PROXY -split ',')) { + $bypassHost = $entry.Trim().TrimStart('.') + if (-not $bypassHost) { continue } + $bypassList += '^https?://([^/]+\.)?' + [regex]::Escape($bypassHost) + '(:\d+)?(/|$)' + } + } + + # Keep credentials out of the proxy address; pass them separately. + $proxyAddress = [Uri]"$($proxyUri.Scheme)://$($proxyUri.Authority)" + $webProxy = New-Object System.Net.WebProxy($proxyAddress, $false, [string[]]$bypassList) + if ($proxyUri.UserInfo) { + $userInfo = $proxyUri.UserInfo.Split(':', 2) + $proxyUser = [Uri]::UnescapeDataString($userInfo[0]) + $proxyPassword = if ($userInfo.Length -gt 1) { [Uri]::UnescapeDataString($userInfo[1]) } else { '' } + $webProxy.Credentials = New-Object System.Net.NetworkCredential($proxyUser, $proxyPassword) + } else { + $webProxy.UseDefaultCredentials = $true + } + [System.Net.WebRequest]::DefaultWebProxy = $webProxy + Write-Host "Using proxy $proxyAddress from the environment" +} + # ── Validation ─────────────────────────────────────────────────────────────── function Test-DockerVersion { @@ -250,6 +310,8 @@ if ($env:RUNTIME -ne 'docker') { exit 1 } +Set-WebProxyFromEnvironment -ProbeUri ([Uri]$REGISTRY_TAGS_URL) + $currentPrincipal = [Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent() if ($currentPrincipal.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) { Write-Warning "Running as Administrator is not recommended. Consider running as a regular user." diff --git a/proxy-agent-manager.ps1 b/proxy-agent-manager.ps1 index d0c0445..24dcdb2 100644 --- a/proxy-agent-manager.ps1 +++ b/proxy-agent-manager.ps1 @@ -70,6 +70,62 @@ function Write-Log { Add-Content -Path $LOGFILE -Value "$timestamp UTC - $Message" -Encoding ascii } +# Windows PowerShell 5.1 ignores HTTP_PROXY/HTTPS_PROXY/NO_PROXY, so apply them to this +# script's web requests during upgrade. curl honors the same variables, so install.sh and install.ps1 +# behave alike behind a proxy. Without them, .NET's default proxy is the user's Internet +# Options proxy, which is kept. Either way the proxy may answer NTLM/Kerberos challenges +# with the current Windows identity, which .NET doesn't do on its own. Docker itself reads +# its own proxy settings; see the README. +function Set-WebProxyFromEnvironment { + param([Parameter(Mandatory)][Uri]$ProbeUri) + + $proxyUrl = if ($env:HTTPS_PROXY) { $env:HTTPS_PROXY } elseif ($env:HTTP_PROXY) { $env:HTTP_PROXY } else { $null } + if (-not $proxyUrl) { + $systemProxy = [System.Net.WebRequest]::DefaultWebProxy + if ($systemProxy) { + $systemProxy.Credentials = [System.Net.CredentialCache]::DefaultCredentials + $effectiveProxy = $systemProxy.GetProxy($ProbeUri) + # Direct connections come back as the probe URI on .NET Framework and as null on .NET Core. + if ($effectiveProxy -and $effectiveProxy -ne $ProbeUri) { + Write-Log "Using system proxy $effectiveProxy" + } + } + return + } + + try { + $proxyUri = [Uri]$proxyUrl + } catch { + Write-Host "ERROR: Invalid proxy URL in HTTPS_PROXY/HTTP_PROXY: $proxyUrl" -ForegroundColor Red + exit 1 + } + + # .NET bypass entries are regular expressions matched against the request URL. As with curl, + # each entry matches the host and its subdomains, with or without a leading dot. CIDR ranges aren't supported. + $bypassList = @() + if ($env:NO_PROXY) { + foreach ($entry in ($env:NO_PROXY -split ',')) { + $bypassHost = $entry.Trim().TrimStart('.') + if (-not $bypassHost) { continue } + $bypassList += '^https?://([^/]+\.)?' + [regex]::Escape($bypassHost) + '(:\d+)?(/|$)' + } + } + + # Keep credentials out of the proxy address; pass them separately. + $proxyAddress = [Uri]"$($proxyUri.Scheme)://$($proxyUri.Authority)" + $webProxy = New-Object System.Net.WebProxy($proxyAddress, $false, [string[]]$bypassList) + if ($proxyUri.UserInfo) { + $userInfo = $proxyUri.UserInfo.Split(':', 2) + $proxyUser = [Uri]::UnescapeDataString($userInfo[0]) + $proxyPassword = if ($userInfo.Length -gt 1) { [Uri]::UnescapeDataString($userInfo[1]) } else { '' } + $webProxy.Credentials = New-Object System.Net.NetworkCredential($proxyUser, $proxyPassword) + } else { + $webProxy.UseDefaultCredentials = $true + } + [System.Net.WebRequest]::DefaultWebProxy = $webProxy + Write-Log "Using proxy $proxyAddress from the environment" +} + function Get-ContainerOsType { $osTypeOutput = docker info --format '{{.OSType}}' 2>$null if ($LASTEXITCODE -ne 0) { @@ -404,6 +460,7 @@ switch ($Command) { if (-not (Start-CurrentProxyAgent)) { exit 1 } } 'upgrade' { + Set-WebProxyFromEnvironment -ProbeUri ([Uri]"https://$($IMAGE.Split('/')[0])/") Invoke-Upgrade } 'status' { diff --git a/tests/pwsh/Manager.Tests.ps1 b/tests/pwsh/Manager.Tests.ps1 index 07c8b6f..41f2ec8 100644 --- a/tests/pwsh/Manager.Tests.ps1 +++ b/tests/pwsh/Manager.Tests.ps1 @@ -114,8 +114,9 @@ AfterAll { Describe 'proxy-agent-manager.ps1' { BeforeEach { - # A developer's environment must not leak into the default-URL assertion. - Remove-Item Env:\PROXY_AGENT_REGISTRY_TAGS_URL -ErrorAction SilentlyContinue + # A developer's environment must not leak into the default-URL or proxy assertions. + Remove-Item Env:\PROXY_AGENT_REGISTRY_TAGS_URL, Env:\HTTPS_PROXY, Env:\HTTP_PROXY, Env:\NO_PROXY -ErrorAction SilentlyContinue + $script:OriginalWebProxy = [System.Net.WebRequest]::DefaultWebProxy $global:ProxyAgentStub = @{ OsType = 'linux' LastImage = $null @@ -128,7 +129,8 @@ Describe 'proxy-agent-manager.ps1' { } AfterEach { - Remove-Item Env:\PROXY_AGENT_REGISTRY_TAGS_URL -ErrorAction SilentlyContinue + Remove-Item Env:\PROXY_AGENT_REGISTRY_TAGS_URL, Env:\HTTPS_PROXY, Env:\HTTP_PROXY, Env:\NO_PROXY -ErrorAction SilentlyContinue + [System.Net.WebRequest]::DefaultWebProxy = $script:OriginalWebProxy if ($script:AgentDir -and (Test-Path -LiteralPath $script:AgentDir)) { Remove-Item -LiteralPath $script:AgentDir -Recurse -Force } @@ -271,4 +273,83 @@ Describe 'proxy-agent-manager.ps1' { $decoded | Should -Match 'HEARTBEAT_EXPIRE_AT' } } + + # The manager applies HTTPS_PROXY/HTTP_PROXY/NO_PROXY to the process-wide + # default proxy before the registry lookup, since Windows PowerShell 5.1 + # ignores those variables. The script runs in this process, so the result + # is observable here; AfterEach restores the original default proxy. + Context 'behind a network proxy' { + It 'applies HTTPS_PROXY with credentials and NO_PROXY on upgrade' { + $env:HTTPS_PROXY = 'http://us%40er:p%3Ass@proxy.example.com:3128' + $env:NO_PROXY = 'localhost, .corp.example.com' + $manager = New-AgentDir + $output = Invoke-Manager $manager upgrade + $LASTEXITCODE | Should -Be 0 + $output | Should -Match 'Using proxy http://proxy.example.com:3128/' + + $proxy = [System.Net.WebRequest]::DefaultWebProxy + $proxy.Address | Should -Be 'http://proxy.example.com:3128/' + $proxy.Credentials.UserName | Should -Be 'us@er' + $proxy.Credentials.Password | Should -Be 'p:ss' + $proxy.IsBypassed([Uri]'https://us-docker.pkg.dev/v2/tags/list') | Should -BeFalse + $proxy.IsBypassed([Uri]'http://localhost:8080/') | Should -BeTrue + $proxy.IsBypassed([Uri]'https://corp.example.com/') | Should -BeTrue + $proxy.IsBypassed([Uri]'https://db.corp.example.com:5432/') | Should -BeTrue + $proxy.IsBypassed([Uri]'https://notcorp.example.com/') | Should -BeFalse + } + + It 'falls back to HTTP_PROXY and uses Windows credentials when the URL has none' { + $env:HTTP_PROXY = 'http://proxy.example.com:8080' + $manager = New-AgentDir + $null = Invoke-Manager $manager upgrade + $LASTEXITCODE | Should -Be 0 + $proxy = [System.Net.WebRequest]::DefaultWebProxy + $proxy.Address | Should -Be 'http://proxy.example.com:8080/' + $proxy.UseDefaultCredentials | Should -BeTrue + } + + It 'keeps the system proxy, reports it and lets it use Windows credentials when no variables are set' { + # Stands in for an Internet Options proxy, which .NET exposes as the default. + $systemProxy = New-Object System.Net.WebProxy('http://system-proxy.example.com:8080') + [System.Net.WebRequest]::DefaultWebProxy = $systemProxy + $manager = New-AgentDir + $output = Invoke-Manager $manager upgrade + $LASTEXITCODE | Should -Be 0 + [object]::ReferenceEquals($systemProxy, [System.Net.WebRequest]::DefaultWebProxy) | Should -BeTrue + $systemProxy.Credentials | Should -Be ([System.Net.CredentialCache]::DefaultCredentials) + $output | Should -Match 'Using system proxy http://system-proxy.example.com:8080/' + } + + It 'reports nothing when the system proxy is a direct connection' { + [System.Net.WebRequest]::DefaultWebProxy = New-Object System.Net.WebProxy + $manager = New-AgentDir + $output = Invoke-Manager $manager upgrade + $LASTEXITCODE | Should -Be 0 + $output | Should -Not -Match 'Using (system )?proxy' + } + + It 'rejects an invalid proxy URL before contacting the registry' { + $env:HTTPS_PROXY = 'http://bad host:' + $manager = New-AgentDir + $output = Invoke-Manager $manager upgrade + $LASTEXITCODE | Should -Be 1 + $output | Should -Match 'Invalid proxy URL' + $global:ProxyAgentStub.RegistryUrl | Should -BeNullOrEmpty + } + + It 'shares its proxy handling with install.ps1' { + # The two scripts are downloaded separately, so the function is duplicated + # rather than shared. This keeps the copies from drifting apart; the only + # allowed difference is that the manager logs through Write-Log. + function Get-ProxyFunctionBody { + param([string]$ScriptName) + $tokens = $null; $errors = $null + $ast = [System.Management.Automation.Language.Parser]::ParseFile((Join-Path $script:RepoRoot $ScriptName), [ref]$tokens, [ref]$errors) + $function = $ast.Find({ param($node) $node -is [System.Management.Automation.Language.FunctionDefinitionAst] -and $node.Name -eq 'Set-WebProxyFromEnvironment' }, $false) + $function | Should -Not -BeNullOrEmpty -Because "$ScriptName must define Set-WebProxyFromEnvironment" + return $function.Body.Extent.Text -replace 'Write-Log ', 'Write-Host ' + } + Get-ProxyFunctionBody 'proxy-agent-manager.ps1' | Should -Be (Get-ProxyFunctionBody 'install.ps1') + } + } }