From 86edfef37410c73e71df034ecf65cae76eba0dd6 Mon Sep 17 00:00:00 2001
From: tonghuaroot <tonghuaroot@gmail.com>
Date: Thu, 16 Jul 2026 10:33:22 +0800
Subject: [PATCH] gh-153795: Stop urllib.robotparser.can_fetch crashing on a
 malformed URL

RobotFileParser.can_fetch() is documented to return a bool, but it passed the
url straight to urllib.parse._urlsplit(), so a malformed url such as an
unterminated IPv6 authority raised a bare ValueError out of can_fetch(). Treat
a url that cannot be parsed as matching no rule and return True, matching RFC
9309's default that access is not restricted when no rule applies.
---
 Lib/test/test_robotparser.py                        | 13 +++++++++++++
 Lib/urllib/robotparser.py                           |  6 +++++-
 .../2026-07-16-11-00-00.gh-issue-153795.Rp7Tn2.rst  |  2 ++
 3 files changed, 20 insertions(+), 1 deletion(-)
 create mode 100644 Misc/NEWS.d/next/Library/2026-07-16-11-00-00.gh-issue-153795.Rp7Tn2.rst

diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py
index cd1477037e94b74..36cea82f0c3198c 100644
--- a/Lib/test/test_robotparser.py
+++ b/Lib/test/test_robotparser.py
@@ -483,6 +483,19 @@ class DisallowQueryStringTest(BaseRobotTest, unittest.TestCase):
            '/yet/one/path?name=value&more']
 
 
+class CanFetchMalformedURLTest(unittest.TestCase):
+    def test_malformed_url_is_allowed(self):
+        # A URL that cannot be parsed matches no rule, so can_fetch() returns
+        # True (RFC 9309 default) rather than raising.
+        parser = urllib.robotparser.RobotFileParser()
+        parser.parse(['User-agent: *', 'Disallow: /'])
+        # The rule above disallows a well-formed path.
+        self.assertFalse(parser.can_fetch('*', '/x'))
+        # An unparsable URL is allowed, not raised.
+        self.assertTrue(parser.can_fetch('*', 'http://[::1'))
+        self.assertTrue(parser.can_fetch('*', 'http://['))
+
+
 class PercentEncodingTest(BaseRobotTest, unittest.TestCase):
     robots_txt = """\
 User-agent: *
diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py
index e70eae800367840..43c1d12680d48b9 100644
--- a/Lib/urllib/robotparser.py
+++ b/Lib/urllib/robotparser.py
@@ -178,7 +178,11 @@ def can_fetch(self, useragent, url):
             return False
         # TODO: The private API is used in order to preserve an empty query.
         # This is temporary until the public API starts supporting this feature.
-        parsed_url = urllib.parse._urlsplit(url, '')
+        try:
+            parsed_url = urllib.parse._urlsplit(url, '')
+        except ValueError:
+            # A malformed URL matches no rule.
+            return True
         url = urllib.parse._urlunsplit(None, None, *parsed_url[2:])
         url = normalize_uri(url)
         if not url:
diff --git a/Misc/NEWS.d/next/Library/2026-07-16-11-00-00.gh-issue-153795.Rp7Tn2.rst b/Misc/NEWS.d/next/Library/2026-07-16-11-00-00.gh-issue-153795.Rp7Tn2.rst
new file mode 100644
index 000000000000000..34fc23e03e0c4fc
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2026-07-16-11-00-00.gh-issue-153795.Rp7Tn2.rst
@@ -0,0 +1,2 @@
+:meth:`urllib.robotparser.RobotFileParser.can_fetch` no longer raises
+:exc:`ValueError` for a malformed URL. Patch by tonghuaroot.