[3.15] gh-59515: Support multiple challenges in urllib Digest Auth (GH-160828) (#161160)

gh-59515: Support multiple challenges in urllib Digest Auth (GH-160828)

`HTTPDigestAuthHandler` and `ProxyDigestAuthHandler` only looked at the
first `WWW-Authenticate` / `Proxy-Authenticate` header and took its
first word as the scheme. As a result:

* A Digest challenge was ignored if a Basic challenge came first, in the
  same header or in an earlier one.

* An unsupported scheme listed first (for example `Kerberos, Basic
  realm="x"`) raised `ValueError`. Because the Digest handler runs
  before the Basic handler, that also blocked `HTTPBasicAuthHandler`.

This PR changes only `AbstractDigestAuthHandler`.
`AbstractBasicAuthHandler` already handles multiple challenges since
GH-18284 and is not modified.

* `http_error_auth_reqed()` now reads all headers (`get_all()`) and uses
  the first Digest challenge found. It raises `ValueError` only when
  neither Digest nor Basic is offered; otherwise it returns `None` so
  the next handler can try.

* A new private `_parse_challenges()` splits a header into challenges
  per RFC 9110 §11.6.1. A comma-separated element that starts with a
  word not followed by `=` begins a new challenge. Its quote/escape
  handling matches `parse_http_list()`. It makes a single linear pass
  with no regex.

* `retry_http_digest_auth()` is unchanged. It receives the original text
  of the Digest challenge, so escapes are not processed twice. For a
  single-challenge header the string is the same as before.

🤖 Generated with [Claude Code](https://claude.com/claude-code)
(cherry picked from commit 529d66c)

Co-authored-by: Senthil Kumaran <senthil@python.org>

Author: miss-islington

Lib/test/test_urllib2.py

@@ -1916,6 +1916,120 @@ def test_unsupported_auth_digest_handler(self):
         opener.add_handler(http_handler)
         self.assertRaises(ValueError, opener.open, "http://www.example.com")
 
+    def check_digest_auth(self, headers, realm):
+        with self.subTest(headers=headers):
+            opener = OpenerDirector()
+            password_manager = MockPasswordManager()
+            auth_handler = urllib.request.HTTPDigestAuthHandler(
+                password_manager)
+            auth_handler.add_password(realm, "http://acme.example.com/",
+                                      "wile", "coyote")
+            body = '\r\n'.join(headers) + '\r\n\r\n'
+            http_handler = MockHTTPHandlerRedirect(401, body)
+            opener.add_handler(auth_handler)
+            opener.add_handler(http_handler)
+            opener.open("http://acme.example.com/protected")
+            self.assertEqual(password_manager.target_realm, realm)
+            auth = http_handler.requests[1].unredirected_hdrs["Authorization"]
+            self.assertStartsWith(auth, "Digest ")
+            self.assertIn(f'realm="{realm}"', auth)
+            self.assertIn('nonce="dcd98b7102dd2f0e8b11d0f600bfb0c093"', auth)
+            self.assertIn('opaque="5ccc069c403ebaf9f0171e9517f40e41"', auth)
+
+    def test_digest_auth_multiple_challenges(self):
+        realm = "digest@example.com"
+        digest = (f'Digest realm="{realm}", '
+                  f'qop="auth, auth-int", '
+                  f'nonce="dcd98b7102dd2f0e8b11d0f600bfb0c093", '
+                  f'opaque="5ccc069c403ebaf9f0171e9517f40e41"')
+        basic = 'Basic realm="basic@example.com"'
+        other = 'Otherscheme xxx'
+
+        # Multiple challenges per header.
+        for challenges in (
+            digest,
+            f'{basic}, {digest}',
+            f'{digest}, {basic}',
+            f'{other}, {digest}',
+            f'{basic}, {digest}, {other}',
+        ):
+            self.check_digest_auth([f'WWW-Authenticate: {challenges}'], realm)
+
+        # Multiple headers: one challenge per header.
+        for challenges in (
+            [basic, digest],
+            [digest, basic],
+            [other, digest],
+        ):
+            headers = [f'WWW-Authenticate: {challenge}'
+                       for challenge in challenges]
+            self.check_digest_auth(headers, realm)
+
+    def test_digest_auth_handler_ignores_basic_challenge(self):
+        # Leave Basic challenges to HTTPBasicAuthHandler, even when an
+        # unsupported scheme is present as well.
+        for headers in (
+            'WWW-Authenticate: Basic realm="x"\r\n\r\n',
+            'WWW-Authenticate: Kerberos, Basic realm="x"\r\n\r\n',
+            'WWW-Authenticate: Kerberos\r\n'
+            'WWW-Authenticate: Basic realm="x"\r\n\r\n',
+        ):
+            with self.subTest(headers=headers):
+                opener = OpenerDirector()
+                opener.add_handler(urllib.request.HTTPDigestAuthHandler(None))
+                opener.add_handler(MockHTTPHandlerRedirect(401, headers))
+                self.assertIsNone(opener.open("http://www.example.com"))
+
+    def test_digest_auth_parse_challenges(self):
+        parse = AbstractDigestAuthHandler()._parse_challenges
+        self.assertEqual(list(parse('')), [])
+        self.assertEqual(list(parse('Digest realm="a"')),
+                         [('Digest', 'Digest realm="a"')])
+        # Commas and quotes inside a quoted string do not split challenges.
+        self.assertEqual(
+            list(parse(r'Digest realm="a, Basic realm=\"b\"", nonce="n", '
+                       r'Basic realm="c"')),
+            [('Digest', r'Digest realm="a, Basic realm=\"b\"", nonce="n"'),
+             ('Basic', 'Basic realm="c"')])
+        # Whitespace around "=" does not start a new challenge.
+        self.assertEqual(
+            list(parse('Digest realm = "a", nonce = "n", Basic realm="b"')),
+            [('Digest', 'Digest realm = "a", nonce = "n"'),
+             ('Basic', 'Basic realm="b"')])
+        self.assertEqual(
+            list(parse('Negotiate, Bearer abc==, Digest realm="a",, ')),
+            [('Negotiate', 'Negotiate'), ('Bearer', 'Bearer abc=='),
+             ('Digest', 'Digest realm="a"')])
+
+    def test_proxy_digest_auth_multiple_challenges(self):
+        realm = "digest@example.com"
+        digest = (f'Digest realm="{realm}", '
+                  f'qop="auth", '
+                  f'nonce="dcd98b7102dd2f0e8b11d0f600bfb0c093"')
+        basic = 'Basic realm="basic@example.com"'
+        for headers in (
+            [f'Proxy-Authenticate: {basic}, {digest}'],
+            [f'Proxy-Authenticate: {basic}', f'Proxy-Authenticate: {digest}'],
+        ):
+            with self.subTest(headers=headers):
+                opener = OpenerDirector()
+                opener.add_handler(urllib.request.ProxyHandler(
+                    dict(http="proxy.example.com:3128")))
+                password_manager = MockPasswordManager()
+                auth_handler = urllib.request.ProxyDigestAuthHandler(
+                    password_manager)
+                auth_handler.add_password(realm, "proxy.example.com:3128",
+                                          "wile", "coyote")
+                body = '\r\n'.join(headers) + '\r\n\r\n'
+                http_handler = MockHTTPHandlerRedirect(407, body)
+                opener.add_handler(auth_handler)
+                opener.add_handler(http_handler)
+                opener.open("http://acme.example.com/protected")
+                self.assertEqual(password_manager.target_realm, realm)
+                auth = http_handler.requests[1].unredirected_hdrs[
+                    "Proxy-authorization"]
+                self.assertStartsWith(auth, "Digest ")
+                self.assertIn(f'realm="{realm}"', auth)
     def test_digest_auth_failure_code(self):
         # gh-53852: a failed proxy digest auth must report 407, not 401.
         headers = http.client.HTTPMessage()

Lib/urllib/request.py

@@ -1085,8 +1085,58 @@ def __init__(self, passwd=None):
     def reset_retry_count(self):
         self.retried = 0
 
+    def _parse_challenges(self, header):
+        # Yield (scheme, challenge) for each challenge in a WWW-Authenticate
+        # header value.  Commas separate both challenges and the auth-params
+        # of a challenge, so an element which starts with a token that is not
+        # followed by "=" starts a new challenge (RFC 9110, section 11.6.1).
+        #
+        # For example, the header value
+        #
+        #     Basic realm="a", Digest realm="b", nonce="n"
+        #
+        # is split on commas into the elements 'Basic realm="a"',
+        # 'Digest realm="b"' and 'nonce="n"'.  "Basic" and "Digest" are not
+        # followed by "=", so they start challenges; 'nonce="n"' continues
+        # the Digest one.  This yields:
+        #
+        #     ('Basic', 'Basic realm="a"')
+        #     ('Digest', 'Digest realm="b", nonce="n"')
+        elements = []
+        start = 0
+        quote = escape = False
+        for i, c in enumerate(header):
+            if escape:
+                escape = False
+            elif quote:
+                if c == '\\':
+                    escape = True
+                elif c == '"':
+                    quote = False
+            elif c == '"':
+                quote = True
+            elif c == ',':
+                elements.append((start, i))
+                start = i + 1
+        elements.append((start, len(header)))
+
+        scheme = begin = end = None
+        for elt_start, elt_end in elements:
+            words = header[elt_start:elt_end].split(None, 1)
+            if not words:
+                continue
+            rest = words[1] if len(words) > 1 else ''
+            if '=' not in words[0] and not rest.startswith('='):
+                if scheme is not None:
+                    yield scheme, header[begin:end].strip()
+                scheme = words[0]
+                begin = elt_start
+            end = elt_end
+        if scheme is not None:
+            yield scheme, header[begin:end].strip()
+
     def http_error_auth_reqed(self, auth_header, host, req, headers):
-        authreq = headers.get(auth_header, None)
+        authreqs = headers.get_all(auth_header)
         if self.retried > 5:
             # Don't fail endlessly - if we failed once, we'll probably
             # fail a second time. Hm. Unless the Password Manager is
@@ -1101,13 +1151,21 @@ def http_error_auth_reqed(self, auth_header, host, req, headers):
                             headers, None)
         else:
             self.retried += 1
-        if authreq:
-            scheme = authreq.split()[0]
-            if scheme.lower() == 'digest':
-                return self.retry_http_digest_auth(req, authreq)
-            elif scheme.lower() != 'basic':
-                raise ValueError("AbstractDigestAuthHandler does not support"
-                                 " the following scheme: '%s'" % scheme)
+        if not authreqs:
+            return None
+        unsupported = None
+        basic_found = False
+        for authreq in authreqs:
+            for scheme, challenge in self._parse_challenges(authreq):
+                if scheme.lower() == 'digest':
+                    return self.retry_http_digest_auth(req, challenge)
+                if scheme.lower() == 'basic':
+                    basic_found = True
+                elif unsupported is None:
+                    unsupported = scheme
+        if unsupported is not None and not basic_found:
+            raise ValueError("AbstractDigestAuthHandler does not support"
+                             " the following scheme: '%s'" % unsupported)
 
     def retry_http_digest_auth(self, req, auth):
         token, challenge = auth.split(' ', 1)

Misc/NEWS.d/next/Library/2026-10-28-13-23-28.gh-issue-59515.wt31yNN4.rst

@@ -0,0 +1,4 @@
+:class:`urllib.request.HTTPDigestAuthHandler` and
+:class:`urllib.request.ProxyDigestAuthHandler` now parse all
+``WWW-Authenticate`` (or ``Proxy-Authenticate``) headers and accept
+multiple challenges per header: the first Digest challenge is used.