feat: add 3 practical web scraping scripts for #92

Web-Scraping/GitHub-Trending-Scraper/README.md

@@ -0,0 +1,52 @@
+# GitHub Trending Scraper
+
+Scrapes GitHub trending repositories and displays them in a clean, formatted output.
+
+## Features
+
+- Fetches trending repositories from GitHub
+- Supports filtering by programming language
+- Supports filtering by time period (daily, weekly, monthly)
+- Displays repo name, description, stars, forks, language, and link
+
+## Usage
+
+```bash
+# Basic usage
+python github_trending.py
+
+# Filter by language
+python github_trending.py --language python
+
+# Filter by time period
+python github_trending.py --since weekly
+
+# Combined filters
+python github_trending.py --language javascript --since monthly
+```
+
+## Output Example
+
+```
+================================================================================
+  GitHub Trending Repositories (25 found)
+================================================================================
+
+  1. owner/repo-name
+     ⭐ 1234 stars | 🍴 56 forks | 📈 234 stars today
+     📝 A cool project description
+     🔤 Python
+     🔗 https://github.com/owner/repo-name
+```
+
+## Requirements
+
+- Python 3.6+
+- `requests`
+- `beautifulsoup4`
+
+Install dependencies:
+
+```bash
+pip install requests beautifulsoup4
+```

Web-Scraping/GitHub-Trending-Scraper/github_trending.py

@@ -0,0 +1,130 @@
+"""
+GitHub Trending Scraper
+Scrapes GitHub trending repositories and displays them in a clean format.
+
+Usage:
+    python github_trending.py [--language LANGUAGE] [--since SINCE]
+
+Requirements:
+    pip install requests beautifulsoup4
+"""
+
+import argparse
+import sys
+from urllib.parse import urljoin
+
+import requests
+from bs4 import BeautifulSoup
+
+BASE_URL = "https://github.com/trending"
+
+
+def build_url(language=None, since=None):
+    """Build the GitHub trending URL with optional filters."""
+    url = BASE_URL
+    if language:
+        url += f"/{language}"
+    if since:
+        url += f"?since={since}"
+    return url
+
+
+def parse_repos(soup):
+    """Extract trending repositories from the parsed HTML."""
+    repos = []
+    articles = soup.find_all("article", class_="Box-row")
+
+    for article in articles:
+        repo_data = {}
+
+        # Repo name and link
+        repo_link = article.find("h2").find("a")
+        if repo_link:
+            repo_data["name"] = repo_link.get_text(strip=True).replace("\n", "").replace(" ", "")
+            repo_data["url"] = urljoin("https://github.com", repo_link["href"])
+
+        # Description
+        desc = article.find("p", class_="col-9")
+        repo_data["description"] = desc.get_text(strip=True) if desc else "No description"
+
+        # Language
+        lang = article.find("span", itemprop="programmingLanguage")
+        repo_data["language"] = lang.get_text(strip=True) if lang else "N/A"
+
+        # Stars and forks
+        star_count = article.find("a", href=lambda x: x and "stargazers" in x)
+        repo_data["stars"] = star_count.get_text(strip=True) if star_count else "0"
+
+        fork_count = article.find("a", href=lambda x: x and "forks" in x)
+        repo_data["forks"] = fork_count.get_text(strip=True) if fork_count else "0"
+
+        # Today's stars
+        today_stars = article.find("span", class_="d-inline-block float-sm-right")
+        repo_data["stars_today"] = today_stars.get_text(strip=True) if today_stars else "N/A"
+
+        repos.append(repo_data)
+
+    return repos
+
+
+def display_repos(repos):
+    """Display repos in a formatted way."""
+    if not repos:
+        print("No trending repositories found.")
+        return
+
+    print(f"\n{'='*80}")
+    print(f"  GitHub Trending Repositories ({len(repos)} found)")
+    print(f"{'='*80}\n")
+
+    for i, repo in enumerate(repos, 1):
+        print(f"  {i}. {repo['name']}")
+        print(f"     ⭐ {repo['stars']} stars | 🍴 {repo['forks']} forks | 📈 {repo['stars_today']} today")
+        print(f"     📝 {repo['description']}")
+        print(f"     🔤 {repo['language']}")
+        print(f"     🔗 {repo['url']}")
+        print()
+
+
+def scrape_trending(language=None, since=None):
+    """Main function to scrape and display trending repos."""
+    url = build_url(language, since)
+    headers = {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
+    }
+
+    print(f"Fetching: {url}")
+    response = requests.get(url, headers=headers, timeout=15)
+    response.raise_for_status()
+
+    soup = BeautifulSoup(response.text, "html.parser")
+    repos = parse_repos(soup)
+    display_repos(repos)
+
+    return repos
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Scrape GitHub trending repositories"
+    )
+    parser.add_argument(
+        "--language",
+        help="Filter by programming language (e.g., python, javascript)",
+    )
+    parser.add_argument(
+        "--since",
+        choices=["daily", "weekly", "monthly"],
+        help="Time period for trending (default: daily)",
+    )
+    args = parser.parse_args()
+
+    try:
+        scrape_trending(language=args.language, since=args.since)
+    except requests.RequestException as e:
+        print(f"Error fetching data: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

Web-Scraping/HackerNews-Scraper/README.md

@@ -0,0 +1,63 @@
+# Hacker News Scraper
+
+Scrapes the Hacker News front page and displays top stories with scores, authors, and comment links.
+
+## Features
+
+- Fetches top stories from Hacker News
+- Displays story title, URL, score, author, and comments
+- Filter by minimum score threshold
+- Output as formatted text or JSON
+- Includes discussion links
+
+## Usage
+
+```bash
+# Basic usage (top 30 stories)
+python hackernews_scraper.py
+
+# Limit number of stories
+python hackernews_scraper.py --limit 10
+
+# Filter by minimum score
+python hackernews_scraper.py --min-score 100
+
+# Output as JSON
+python hackernews_scraper.py --json
+
+# Combined options
+python hackernews_scraper.py --limit 20 --min-score 50 --json
+```
+
+## Output Example
+
+```
+================================================================================
+  Hacker News Top Stories (30 stories)
+================================================================================
+
+  1. Show HN: A new open-source project
+     ⬆️  234 points | 👤 devuser | 💬 45 comments
+     🔗 https://example.com/project
+     💬 Discuss: https://news.ycombinator.com/item?id=12345678
+```
+
+## JSON Output
+
+Use `--json` flag for machine-readable output:
+
+```bash
+python hackernews_scraper.py --limit 5 --json
+```
+
+## Requirements
+
+- Python 3.6+
+- `requests`
+- `beautifulsoup4`
+
+Install dependencies:
+
+```bash
+pip install requests beautifulsoup4
+```

Web-Scraping/HackerNews-Scraper/hackernews_scraper.py

@@ -0,0 +1,141 @@
+"""
+Hacker News Scraper
+Scrapes the Hacker News front page and displays top stories.
+
+Usage:
+    python hackernews_scraper.py [--limit N] [--min-score SCORE] [--json]
+
+Requirements:
+    pip install requests beautifulsoup4
+"""
+
+import argparse
+import json
+import sys
+from urllib.parse import urljoin
+
+import requests
+from bs4 import BeautifulSoup
+
+HN_URL = "https://news.ycombinator.com/"
+
+
+def scrape_hn(limit=30, min_score=0):
+    """Scrape Hacker News front page stories."""
+    headers = {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
+    }
+
+    response = requests.get(HN_URL, headers=headers, timeout=15)
+    response.raise_for_status()
+
+    soup = BeautifulSoup(response.text, "html.parser")
+    stories = []
+
+    # Each story is a table row with class "athing"
+    story_rows = soup.find_all("tr", class_="athing")
+
+    for row in story_rows:
+        story_data = {}
+
+        # Story ID
+        story_data["id"] = row.get("id", "")
+
+        # Title and URL
+        title_cell = row.find("td", class_="title")
+        if title_cell:
+            title_link = title_cell.find("a")
+            if title_link:
+                story_data["title"] = title_link.get_text(strip=True)
+                story_data["url"] = title_link.get("href", "")
+                # Handle relative URLs (e.g., item?id=xxx)
+                if story_data["url"].startswith("item?"):
+                    story_data["url"] = urljoin(HN_URL, story_data["url"])
+
+        # Get the subtext row (next sibling)
+        subtext_row = row.find_next_sibling("tr")
+        if subtext_row:
+            subtext = subtext_row.find("td", class_="subtext")
+            if subtext:
+                # Score
+                score_elem = subtext.find("span", class_="score")
+                if score_elem:
+                    score_text = score_elem.get_text(strip=True)
+                    story_data["score"] = int(score_text.split()[0]) if score_text else 0
+                else:
+                    story_data["score"] = 0
+
+                # Author
+                author_elem = subtext.find("a", class_="hnuser")
+                story_data["author"] = author_elem.get_text(strip=True) if author_elem else "N/A"
+
+                # Comments link
+                comments_link = subtext.find("a", href=lambda x: x and "item?id=" in x)
+                story_data["comments"] = comments_link.get_text(strip=True) if comments_link else "0 comments"
+                story_data["comments_url"] = urljoin(HN_URL, comments_link["href"]) if comments_link else ""
+
+        # Filter by minimum score
+        if story_data.get("score", 0) >= min_score:
+            stories.append(story_data)
+
+        if len(stories) >= limit:
+            break
+
+    return stories
+
+
+def display_stories(stories):
+    """Display stories in a formatted way."""
+    if not stories:
+        print("No stories found matching criteria.")
+        return
+
+    print(f"\n{'='*80}")
+    print(f"  Hacker News Top Stories ({len(stories)} stories)")
+    print(f"{'='*80}\n")
+
+    for i, story in enumerate(stories, 1):
+        print(f"  {i}. {story.get('title', 'N/A')}")
+        print(f"     ⬆️  {story.get('score', 0)} points | 👤 {story.get('author', 'N/A')} | 💬 {story.get('comments', 'N/A')}")
+        print(f"     🔗 {story.get('url', 'N/A')}")
+        print(f"     💬 Discuss: {story.get('comments_url', 'N/A')}")
+        print()
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Scrape Hacker News front page"
+    )
+    parser.add_argument(
+        "--limit",
+        type=int,
+        default=30,
+        help="Maximum number of stories to display (default: 30)",
+    )
+    parser.add_argument(
+        "--min-score",
+        type=int,
+        default=0,
+        help="Minimum score threshold (default: 0)",
+    )
+    parser.add_argument(
+        "--json",
+        action="store_true",
+        help="Output as JSON instead of formatted text",
+    )
+    args = parser.parse_args()
+
+    try:
+        stories = scrape_hn(limit=args.limit, min_score=args.min_score)
+
+        if args.json:
+            print(json.dumps(stories, indent=2))
+        else:
+            display_stories(stories)
+    except requests.RequestException as e:
+        print(f"Error fetching data: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()