gh-153795: Stop urllib.robotparser.can_fetch crashing on a malformed URL

RobotFileParser.can_fetch() is documented to return a bool, but it passed the
url straight to urllib.parse._urlsplit(), so a malformed url such as an
unterminated IPv6 authority raised a bare ValueError out of can_fetch(). Treat
a url that cannot be parsed as matching no rule and return True, matching RFC
9309's default that access is not restricted when no rule applies.

Author: tonghuaroot

Lib/test/test_robotparser.py

@@ -483,6 +483,19 @@ class DisallowQueryStringTest(BaseRobotTest, unittest.TestCase):
            '/yet/one/path?name=value&more']
 
 
+class CanFetchMalformedURLTest(unittest.TestCase):
+    def test_malformed_url_is_allowed(self):
+        # A URL that cannot be parsed matches no rule, so can_fetch() returns
+        # True (RFC 9309 default) rather than raising.
+        parser = urllib.robotparser.RobotFileParser()
+        parser.parse(['User-agent: *', 'Disallow: /'])
+        # The rule above disallows a well-formed path.
+        self.assertFalse(parser.can_fetch('*', '/x'))
+        # An unparsable URL is allowed, not raised.
+        self.assertTrue(parser.can_fetch('*', 'http://[::1'))
+        self.assertTrue(parser.can_fetch('*', 'http://['))
+
+
 class PercentEncodingTest(BaseRobotTest, unittest.TestCase):
     robots_txt = """\
 User-agent: *

Lib/urllib/robotparser.py

@@ -178,7 +178,11 @@ def can_fetch(self, useragent, url):
             return False
         # TODO: The private API is used in order to preserve an empty query.
         # This is temporary until the public API starts supporting this feature.
-        parsed_url = urllib.parse._urlsplit(url, '')
+        try:
+            parsed_url = urllib.parse._urlsplit(url, '')
+        except ValueError:
+            # A malformed URL matches no rule.
+            return True
         url = urllib.parse._urlunsplit(None, None, *parsed_url[2:])
         url = normalize_uri(url)
         if not url:

Misc/NEWS.d/next/Library/2026-07-16-11-00-00.gh-issue-153795.Rp7Tn2.rst

@@ -0,0 +1,2 @@
+:meth:`urllib.robotparser.RobotFileParser.can_fetch` no longer raises
+:exc:`ValueError` for a malformed URL. Patch by tonghuaroot.