Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 10 additions & 5 deletions website/build.py
100755 → 100644
Original file line number Diff line number Diff line change
Expand Up @@ -125,14 +125,19 @@ class SyntheticCategory(TypedDict):

def detect_source_type(url: str) -> str | None:
"""Detect source type from URL domain. Returns None for GitHub URLs."""
if GITHUB_REPO_URL_RE.match(url):
try:
hostname = urlsplit(url).hostname
except ValueError:
hostname = None
if not hostname:
return "External"
hostname = hostname.rstrip(".")
if hostname == "github.com" or hostname.endswith(".github.com"):
return None
for domain, source_type in SOURCE_TYPE_DOMAINS.items():
if domain in url:
if hostname == domain or hostname.endswith(f".{domain}"):
return source_type
if "github.com" not in url:
return "External"
return None
return "External"


def extract_github_repo(url: str) -> str | None:
Expand Down
26 changes: 26 additions & 0 deletions website/tests/test_build.py
Original file line number Diff line number Diff line change
Expand Up @@ -1445,6 +1445,32 @@ def test_non_github_external(self):
def test_github_non_repo_returns_none(self):
assert detect_source_type("https://github.com/org/repo/wiki") is None

@pytest.mark.parametrize(
"url",
[
"https://docs.python.org.lookalike.example/3/library/asyncio.html",
"https://gitlab.com.lookalike.example/org/repo",
"https://bitbucket.org@lookalike.example/org/repo",
"https://lookalike.example/?next=https://docs.python.org/3/library/",
"https://github.com.lookalike.example/owner/repo",
"https://lookalike.example/github.com/owner/repo",
],
)
def test_lookalike_domains_and_url_text_are_external(self, url):
assert detect_source_type(url) == "External"

@pytest.mark.parametrize(
("url", "source_type"),
[
("https://docs.python.org/3/library/asyncio.html", "Stdlib"),
("https://subdomain.gitlab.com/org/repo", "GitLab"),
("https://bitbucket.org./org/repo", "Bitbucket"),
("https://gist.github.com/owner/gist", None),
],
)
def test_real_domain_and_subdomain_detection(self, url, source_type):
assert detect_source_type(url) == source_type


class TestExtractEntries:
def test_basic_extraction(self):
Expand Down