Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions Doc/library/urllib.request.rst
Original file line number Diff line number Diff line change
Expand Up @@ -358,10 +358,11 @@

To disable autodetected proxy pass an empty dictionary.

The :envvar:`no_proxy` environment variable can be used to specify hosts

Check warning on line 361 in Doc/library/urllib.request.rst

View workflow job for this annotation

GitHub Actions / Docs / Docs

'envvar' reference target not found: no_proxy [ref.envvar]
which shouldn't be reached via proxy; if set, it should be a comma-separated
list of hostname suffixes, optionally with ``:port`` appended, for example
``cern.ch,ncsa.uiuc.edu,some.host:8080``.
list of hostname suffixes, optionally with ``:port`` appended, and IP
address CIDR ranges, for example
``cern.ch,ncsa.uiuc.edu,some.host:8080,192.168.0.0/16``.

.. note::

Expand Down
33 changes: 33 additions & 0 deletions Lib/test/test_urllib.py
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,39 @@ def test_proxy_bypass_environment_host_match(self):
self.assertFalse(bypass('newdomain.com')) # no port
self.assertFalse(bypass('newdomain.com:1235')) # wrong port

def test_proxy_bypass_environment_cidr_match(self):
bypass = urllib.request.proxy_bypass_environment
self.env.set('NO_PROXY',
'192.168.0.0/16, 2001:db8::/32, 172.16.1.1/24')
self.assertTrue(bypass('192.168.1.1'))
self.assertTrue(bypass('192.168.1.1:1234'))
self.assertTrue(bypass('2001:db8::1'))
self.assertTrue(bypass('[2001:db8::1]:1234'))
self.assertTrue(bypass('172.16.1.255'))
self.assertFalse(bypass('192.169.1.1'))
self.assertFalse(bypass('2001:db9::1'))
self.assertFalse(bypass('172.16.2.1'))
self.assertFalse(bypass('python.org'))

def test_proxy_bypass_environment_invalid_cidr(self):
bypass = urllib.request.proxy_bypass_environment
self.env.set('NO_PROXY',
'192.168.0.0/33, 2001:db8::/129, anotherdomain.com')
self.assertFalse(bypass('192.168.1.1'))
self.assertFalse(bypass('2001:db8::1'))
self.assertTrue(bypass('anotherdomain.com'))

def test_proxy_bypass_ip_address(self):
bypass = urllib.request.proxy_bypass_environment
self.env.set('NO_PROXY', '169.254.169.254')
self.assertTrue(bypass('169.254.169.254'))
self.assertTrue(bypass('169.254.169.254:1234'))
self.assertFalse(bypass('169.254.169:254'))
self.assertFalse(bypass('169.254.169.254.org'))
self.assertFalse(bypass('2001:db9::1'))
self.assertFalse(bypass('172.16.2.1'))
self.assertFalse(bypass('python.org'))

def test_proxy_bypass_environment_always_match(self):
bypass = urllib.request.proxy_bypass_environment
self.env.set('NO_PROXY', '*')
Expand Down
23 changes: 23 additions & 0 deletions Lib/test/test_urllib2.py
Original file line number Diff line number Diff line change
Expand Up @@ -1552,6 +1552,17 @@ def test_winreg_proxy_bypass(self):
"expect <local> to bypass intranet address '%s'"
% host)

# check IP CIDR bypass
proxy_override = "192.168.0.0/16; 2001:db8::/32"
for host in ("192.168.1.1", "192.168.1.1:443",
"2001:db8::1", "[2001:db8::1]:443"):
self.assertTrue(proxy_bypass(host, proxy_override),
"expected bypass of %s to be true" % host)

for host in ("192.169.1.1", "2001:db9::1"):
self.assertFalse(proxy_bypass(host, proxy_override),
"expected bypass of %s to be False" % host)

@unittest.skipUnless(sys.platform == 'darwin', "only relevant for OSX")
def test_osx_proxy_bypass(self):
bypass = {
Expand Down Expand Up @@ -1586,6 +1597,18 @@ def test_osx_proxy_bypass(self):
self.assertFalse(_proxy_bypass_macosx_sysconf(host, bypass),
'expected bypass of %s to be False' % host)

# Check IPv6 CIDR ranges
bypass = {
'exclude_simple': False,
'exceptions': ['2001:db8::/32']
}
for host in ('2001:db8::1', '[2001:db8::1]:443'):
self.assertTrue(_proxy_bypass_macosx_sysconf(host, bypass),
'expected bypass of %s to be True' % host)
host = '2001:db9::1'
self.assertFalse(_proxy_bypass_macosx_sysconf(host, bypass),
'expected bypass of %s to be False' % host)

def check_basic_auth(self, headers, realm):
with self.subTest(realm=realm, headers=headers):
opener = OpenerDirector()
Expand Down
57 changes: 49 additions & 8 deletions Lib/urllib/request.py
Original file line number Diff line number Diff line change
Expand Up @@ -1908,11 +1908,34 @@ def getproxies_environment():
proxies.pop(proxy_name, None)
return proxies

def _ip_address_from_host(host):
from ipaddress import ip_address

hostonly, port = _splitport(host)
for candidate in (hostonly, host):
candidate = candidate.strip('[]')
try:
return ip_address(candidate)
except ValueError:
pass
return None

def _is_ip_address_in_network(ip_address, network):
if ip_address is None:
return False
from ipaddress import ip_network

try:
return ip_address in ip_network(network, strict=False)
except ValueError:
return False

def proxy_bypass_environment(host, proxies=None):
"""Test if proxies should not be used for a particular host.

Checks the proxy dict for the value of no_proxy, which should
be a list of comma separated DNS suffixes, or '*' for all hosts.
be a list of comma separated DNS suffixes, IP address CIDR ranges,
or '*' for all hosts.

"""
if proxies is None:
Expand All @@ -1928,14 +1951,28 @@ def proxy_bypass_environment(host, proxies=None):
host = host.lower()
# strip port off host
hostonly, port = _splitport(host)
# check if the host ends with any of the DNS suffixes
host_ip = None
checked_host_ip = False
# for every entry in no_proxy...
for name in no_proxy.split(','):
name = name.strip()
if name:
name = name.lstrip('.') # ignore leading dots
name = name.lower()

# check for exact match
if hostonly == name or host == name:
return True

# check if the IP is within CIDR range
if '/' in name:
if not checked_host_ip:
host_ip = _ip_address_from_host(host)
checked_host_ip = True
if _is_ip_address_in_network(host_ip, name):
return True

# check if the host ends with any of the DNS suffixes
name = '.' + name
if hostonly.endswith(name) or host.endswith(name):
return True
Expand All @@ -1958,9 +1995,9 @@ def _proxy_bypass_macosx_sysconf(host, proxy_settings):
}
"""
from fnmatch import fnmatch
from ipaddress import AddressValueError, IPv4Address

hostonly, port = _splitport(host)
host_ip = _ip_address_from_host(host)

def ip2num(ipAddr):
parts = ipAddr.split('.')
Expand All @@ -1975,15 +2012,16 @@ def ip2num(ipAddr):
return True

hostIP = None
try:
hostIP = int(IPv4Address(hostonly))
except AddressValueError:
pass
if host_ip is not None and host_ip.version == 4:
hostIP = int(host_ip)

for value in proxy_settings.get('exceptions', ()):
# Items in the list are strings like these: *.local, 169.254/16
if not value: continue

if '/' in value and _is_ip_address_in_network(host_ip, value):
return True

m = re.match(r"(\d+(?:\.\d+)*)(/\d+)?", value)
if m is not None and hostIP is not None:
base = ip2num(m.group(1))
Expand Down Expand Up @@ -2016,10 +2054,11 @@ def _proxy_bypass_winreg_override(host, override):
Internet settings proxy override registry value.

An example of a proxy override value is:
"www.example.com;*.example.net; 192.168.0.1"
"www.example.com;*.example.net; 192.168.0.1; 192.168.0.0/16"
"""
from fnmatch import fnmatch

host_ip = _ip_address_from_host(host)
host, _ = _splitport(host)
proxy_override = override.split(';')
for test in proxy_override:
Expand All @@ -2028,6 +2067,8 @@ def _proxy_bypass_winreg_override(host, override):
if test == '<local>':
if '.' not in host:
return True
elif '/' in test and _is_ip_address_in_network(host_ip, test):
return True
elif fnmatch(host, test):
return True
return False
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Add support for CIDR-notated IP addresses in urllib Proxy Bypass lists. This
includes the ``NO_PROXY`` environment variable, as well as SystemConfig on
MacOS and the Windows registry.
Loading