diff --git a/src/rfc3986/normalizers.py b/src/rfc3986/normalizers.py index 79b3585..d5ece2c 100644 --- a/src/rfc3986/normalizers.py +++ b/src/rfc3986/normalizers.py @@ -119,16 +119,26 @@ def normalize_fragment(fragment: t.Optional[str]) -> t.Optional[str]: PERCENT_MATCHER = re.compile("%[A-Fa-f0-9]{2}") +# RFC 3986 Section 2.3 +UNRESERVED_PERCENT_ENCODING = frozenset( + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~" +) + def normalize_percent_characters(s: str) -> str: - """All percent characters should be upper-cased. + """Upper-case percent-encodings; decode those of unreserved characters. - For example, ``"%3afoo%DF%ab"`` should be turned into ``"%3Afoo%DF%AB"``. + For example, ``"%3afoo%DF%ab%7e"`` should be turned into + ``"%3Afoo%DF%AB~"``. """ matches = set(PERCENT_MATCHER.findall(s)) for m in matches: if not m.isupper(): s = s.replace(m, m.upper()) + for m in set(PERCENT_MATCHER.findall(s)): + char = chr(int(m[1:], 16)) + if char in UNRESERVED_PERCENT_ENCODING: + s = s.replace(m, char) return s diff --git a/tests/test_normalizers.py b/tests/test_normalizers.py index 87562ac..a01aa7d 100644 --- a/tests/test_normalizers.py +++ b/tests/test_normalizers.py @@ -1,5 +1,6 @@ import pytest +from rfc3986 import normalize_uri from rfc3986.normalizers import encode_component from rfc3986.normalizers import normalize_host from rfc3986.normalizers import normalize_percent_characters @@ -7,6 +8,10 @@ from rfc3986.normalizers import remove_dot_segments from rfc3986.uri import URIReference +UNRESERVED = ( + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~" +) + def test_normalize_scheme(): assert "http" == normalize_scheme("htTp") @@ -15,7 +20,7 @@ def test_normalize_scheme(): def test_normalize_percent_characters(): - expected = "%3Athis_should_be_lowercase%DF%AB%4C" + expected = "%3Athis_should_be_lowercase%DF%ABL" assert expected == normalize_percent_characters( "%3athis_should_be_lowercase%DF%ab%4c" ) @@ -27,6 +32,31 @@ def test_normalize_percent_characters(): ) +def test_normalize_percent_characters_decodes_unreserved(): + for char in UNRESERVED: + assert normalize_percent_characters(f"%{ord(char):02X}") == char + assert normalize_percent_characters(f"%{ord(char):02x}") == char + + +def test_normalize_percent_characters_keeps_other_octets(): + for octet in range(256): + if chr(octet) in UNRESERVED: + continue + assert normalize_percent_characters(f"%{octet:02X}") == f"%{octet:02X}" + + +def test_normalize_percent_characters_keeps_sub_delimiters(): + sub_delimiters = "%21%24%26%27%28%29%2A%2B%2C%3B%3D" + assert sub_delimiters == normalize_percent_characters(sub_delimiters) + + +def test_normalize_percent_characters_is_idempotent(): + samples = ("%41%42%43", "%3A%2F%DF%AB%7e", "%7Efoo%21%2B", "a/b/%2E%2E") + for sample in samples: + once = normalize_percent_characters(sample) + assert once == normalize_percent_characters(once) + + paths = [ # (Input, expected output) ("/foo/bar/.", "/foo/bar/"), @@ -67,6 +97,15 @@ def test_normalized_equality(uris): assert uris[0] == uris[1] +def test_normalized_equality_with_unreserved_percent_encoding(): + assert URIReference(None, None, "/%7Efoo", None, None) == URIReference( + None, None, "/~foo", None, None + ) + assert URIReference(None, None, "/%41%42%43", None, None) == URIReference( + None, None, "/ABC", None, None + ) + + def test_hostname_normalization(): assert URIReference(None, "EXAMPLE.COM", None, None, None) == URIReference( None, "example.com", None, None, None @@ -78,6 +117,8 @@ def test_hostname_normalization(): [ ("user%2aName@EXAMPLE.COM", "user%2AName@example.com"), ("[::1%eth0]", "[::1%25eth0]"), + ("user%41%7e@EXAMPLE.COM", "userA~@example.com"), + ("user%21@EXAMPLE.COM", "user%21@example.com"), ], ) def test_authority_normalization(authority, expected_authority): @@ -86,8 +127,24 @@ def test_authority_normalization(authority, expected_authority): def test_fragment_normalization(): - uri = URIReference(None, "example.com", None, None, "fiz%DF").normalize() - assert uri.fragment == "fiz%DF" + uri = URIReference( + None, "example.com", None, None, "fiz%DF%7e" + ).normalize() + assert uri.fragment == "fiz%DF~" + + +def test_normalize_path_decodes_before_dot_segment_removal(): + assert ( + URIReference(None, None, "/a/%2E%2E/b", None, None).normalize().path + == "/b" + ) + + +def test_normalize_uri_api(): + assert ( + normalize_uri("http://example.com/%7Efoo/%41?q=%7E#%2D") + == "http://example.com/~foo/A?q=~#-" + ) @pytest.mark.parametrize(