From c4a29b5bfcced2a48bffc639aa9363584e76ad91 Mon Sep 17 00:00:00 2001 From: rahulrajarapu158-arch Date: Sun, 11 Oct 2026 16:10:11 +0000 Subject: [PATCH] feat: add 3 practical web scraping scripts - GitHub Trending Scraper: scrape trending repos with language/time filters - Weather Forecast Scraper: fetch 3-day weather for any city via wttr.in - Hacker News Scraper: scrape HN front page with score filtering and JSON output All scripts include README docs, CLI arguments, and error handling. Closes #92 --- .../GitHub-Trending-Scraper/README.md | 52 +++++++ .../github_trending.py | 130 ++++++++++++++++ Web-Scraping/HackerNews-Scraper/README.md | 63 ++++++++ .../HackerNews-Scraper/hackernews_scraper.py | 141 ++++++++++++++++++ .../Weather-Forecast-Scraper/README.md | 59 ++++++++ .../weather_forecast.py | 124 +++++++++++++++ 6 files changed, 569 insertions(+) create mode 100644 Web-Scraping/GitHub-Trending-Scraper/README.md create mode 100644 Web-Scraping/GitHub-Trending-Scraper/github_trending.py create mode 100644 Web-Scraping/HackerNews-Scraper/README.md create mode 100644 Web-Scraping/HackerNews-Scraper/hackernews_scraper.py create mode 100644 Web-Scraping/Weather-Forecast-Scraper/README.md create mode 100644 Web-Scraping/Weather-Forecast-Scraper/weather_forecast.py diff --git a/Web-Scraping/GitHub-Trending-Scraper/README.md b/Web-Scraping/GitHub-Trending-Scraper/README.md new file mode 100644 index 00000000..bc59bade --- /dev/null +++ b/Web-Scraping/GitHub-Trending-Scraper/README.md @@ -0,0 +1,52 @@ +# GitHub Trending Scraper + +Scrapes GitHub trending repositories and displays them in a clean, formatted output. + +## Features + +- Fetches trending repositories from GitHub +- Supports filtering by programming language +- Supports filtering by time period (daily, weekly, monthly) +- Displays repo name, description, stars, forks, language, and link + +## Usage + +```bash +# Basic usage +python github_trending.py + +# Filter by language +python github_trending.py --language python + +# Filter by time period +python github_trending.py --since weekly + +# Combined filters +python github_trending.py --language javascript --since monthly +``` + +## Output Example + +``` +================================================================================ + GitHub Trending Repositories (25 found) +================================================================================ + + 1. owner/repo-name + ⭐ 1234 stars | 🍴 56 forks | 📈 234 stars today + 📝 A cool project description + 🔤 Python + 🔗 https://github.com/owner/repo-name +``` + +## Requirements + +- Python 3.6+ +- `requests` +- `beautifulsoup4` + +Install dependencies: + +```bash +pip install requests beautifulsoup4 +``` \ No newline at end of file diff --git a/Web-Scraping/GitHub-Trending-Scraper/github_trending.py b/Web-Scraping/GitHub-Trending-Scraper/github_trending.py new file mode 100644 index 00000000..15e9e213 --- /dev/null +++ b/Web-Scraping/GitHub-Trending-Scraper/github_trending.py @@ -0,0 +1,130 @@ +""" +GitHub Trending Scraper +Scrapes GitHub trending repositories and displays them in a clean format. + +Usage: + python github_trending.py [--language LANGUAGE] [--since SINCE] + +Requirements: + pip install requests beautifulsoup4 +""" + +import argparse +import sys +from urllib.parse import urljoin + +import requests +from bs4 import BeautifulSoup + +BASE_URL = "https://github.com/trending" + + +def build_url(language=None, since=None): + """Build the GitHub trending URL with optional filters.""" + url = BASE_URL + if language: + url += f"/{language}" + if since: + url += f"?since={since}" + return url + + +def parse_repos(soup): + """Extract trending repositories from the parsed HTML.""" + repos = [] + articles = soup.find_all("article", class_="Box-row") + + for article in articles: + repo_data = {} + + # Repo name and link + repo_link = article.find("h2").find("a") + if repo_link: + repo_data["name"] = repo_link.get_text(strip=True).replace("\n", "").replace(" ", "") + repo_data["url"] = urljoin("https://github.com", repo_link["href"]) + + # Description + desc = article.find("p", class_="col-9") + repo_data["description"] = desc.get_text(strip=True) if desc else "No description" + + # Language + lang = article.find("span", itemprop="programmingLanguage") + repo_data["language"] = lang.get_text(strip=True) if lang else "N/A" + + # Stars and forks + star_count = article.find("a", href=lambda x: x and "stargazers" in x) + repo_data["stars"] = star_count.get_text(strip=True) if star_count else "0" + + fork_count = article.find("a", href=lambda x: x and "forks" in x) + repo_data["forks"] = fork_count.get_text(strip=True) if fork_count else "0" + + # Today's stars + today_stars = article.find("span", class_="d-inline-block float-sm-right") + repo_data["stars_today"] = today_stars.get_text(strip=True) if today_stars else "N/A" + + repos.append(repo_data) + + return repos + + +def display_repos(repos): + """Display repos in a formatted way.""" + if not repos: + print("No trending repositories found.") + return + + print(f"\n{'='*80}") + print(f" GitHub Trending Repositories ({len(repos)} found)") + print(f"{'='*80}\n") + + for i, repo in enumerate(repos, 1): + print(f" {i}. {repo['name']}") + print(f" ⭐ {repo['stars']} stars | 🍴 {repo['forks']} forks | 📈 {repo['stars_today']} today") + print(f" 📝 {repo['description']}") + print(f" 🔤 {repo['language']}") + print(f" 🔗 {repo['url']}") + print() + + +def scrape_trending(language=None, since=None): + """Main function to scrape and display trending repos.""" + url = build_url(language, since) + headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" + } + + print(f"Fetching: {url}") + response = requests.get(url, headers=headers, timeout=15) + response.raise_for_status() + + soup = BeautifulSoup(response.text, "html.parser") + repos = parse_repos(soup) + display_repos(repos) + + return repos + + +def main(): + parser = argparse.ArgumentParser( + description="Scrape GitHub trending repositories" + ) + parser.add_argument( + "--language", + help="Filter by programming language (e.g., python, javascript)", + ) + parser.add_argument( + "--since", + choices=["daily", "weekly", "monthly"], + help="Time period for trending (default: daily)", + ) + args = parser.parse_args() + + try: + scrape_trending(language=args.language, since=args.since) + except requests.RequestException as e: + print(f"Error fetching data: {e}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/Web-Scraping/HackerNews-Scraper/README.md b/Web-Scraping/HackerNews-Scraper/README.md new file mode 100644 index 00000000..16508cea --- /dev/null +++ b/Web-Scraping/HackerNews-Scraper/README.md @@ -0,0 +1,63 @@ +# Hacker News Scraper + +Scrapes the Hacker News front page and displays top stories with scores, authors, and comment links. + +## Features + +- Fetches top stories from Hacker News +- Displays story title, URL, score, author, and comments +- Filter by minimum score threshold +- Output as formatted text or JSON +- Includes discussion links + +## Usage + +```bash +# Basic usage (top 30 stories) +python hackernews_scraper.py + +# Limit number of stories +python hackernews_scraper.py --limit 10 + +# Filter by minimum score +python hackernews_scraper.py --min-score 100 + +# Output as JSON +python hackernews_scraper.py --json + +# Combined options +python hackernews_scraper.py --limit 20 --min-score 50 --json +``` + +## Output Example + +``` +================================================================================ + Hacker News Top Stories (30 stories) +================================================================================ + + 1. Show HN: A new open-source project + ⬆️ 234 points | 👤 devuser | 💬 45 comments + 🔗 https://example.com/project + 💬 Discuss: https://news.ycombinator.com/item?id=12345678 +``` + +## JSON Output + +Use `--json` flag for machine-readable output: + +```bash +python hackernews_scraper.py --limit 5 --json +``` + +## Requirements + +- Python 3.6+ +- `requests` +- `beautifulsoup4` + +Install dependencies: + +```bash +pip install requests beautifulsoup4 +``` \ No newline at end of file diff --git a/Web-Scraping/HackerNews-Scraper/hackernews_scraper.py b/Web-Scraping/HackerNews-Scraper/hackernews_scraper.py new file mode 100644 index 00000000..a64ae27c --- /dev/null +++ b/Web-Scraping/HackerNews-Scraper/hackernews_scraper.py @@ -0,0 +1,141 @@ +""" +Hacker News Scraper +Scrapes the Hacker News front page and displays top stories. + +Usage: + python hackernews_scraper.py [--limit N] [--min-score SCORE] [--json] + +Requirements: + pip install requests beautifulsoup4 +""" + +import argparse +import json +import sys +from urllib.parse import urljoin + +import requests +from bs4 import BeautifulSoup + +HN_URL = "https://news.ycombinator.com/" + + +def scrape_hn(limit=30, min_score=0): + """Scrape Hacker News front page stories.""" + headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" + } + + response = requests.get(HN_URL, headers=headers, timeout=15) + response.raise_for_status() + + soup = BeautifulSoup(response.text, "html.parser") + stories = [] + + # Each story is a table row with class "athing" + story_rows = soup.find_all("tr", class_="athing") + + for row in story_rows: + story_data = {} + + # Story ID + story_data["id"] = row.get("id", "") + + # Title and URL + title_cell = row.find("td", class_="title") + if title_cell: + title_link = title_cell.find("a") + if title_link: + story_data["title"] = title_link.get_text(strip=True) + story_data["url"] = title_link.get("href", "") + # Handle relative URLs (e.g., item?id=xxx) + if story_data["url"].startswith("item?"): + story_data["url"] = urljoin(HN_URL, story_data["url"]) + + # Get the subtext row (next sibling) + subtext_row = row.find_next_sibling("tr") + if subtext_row: + subtext = subtext_row.find("td", class_="subtext") + if subtext: + # Score + score_elem = subtext.find("span", class_="score") + if score_elem: + score_text = score_elem.get_text(strip=True) + story_data["score"] = int(score_text.split()[0]) if score_text else 0 + else: + story_data["score"] = 0 + + # Author + author_elem = subtext.find("a", class_="hnuser") + story_data["author"] = author_elem.get_text(strip=True) if author_elem else "N/A" + + # Comments link + comments_link = subtext.find("a", href=lambda x: x and "item?id=" in x) + story_data["comments"] = comments_link.get_text(strip=True) if comments_link else "0 comments" + story_data["comments_url"] = urljoin(HN_URL, comments_link["href"]) if comments_link else "" + + # Filter by minimum score + if story_data.get("score", 0) >= min_score: + stories.append(story_data) + + if len(stories) >= limit: + break + + return stories + + +def display_stories(stories): + """Display stories in a formatted way.""" + if not stories: + print("No stories found matching criteria.") + return + + print(f"\n{'='*80}") + print(f" Hacker News Top Stories ({len(stories)} stories)") + print(f"{'='*80}\n") + + for i, story in enumerate(stories, 1): + print(f" {i}. {story.get('title', 'N/A')}") + print(f" ⬆️ {story.get('score', 0)} points | 👤 {story.get('author', 'N/A')} | 💬 {story.get('comments', 'N/A')}") + print(f" 🔗 {story.get('url', 'N/A')}") + print(f" 💬 Discuss: {story.get('comments_url', 'N/A')}") + print() + + +def main(): + parser = argparse.ArgumentParser( + description="Scrape Hacker News front page" + ) + parser.add_argument( + "--limit", + type=int, + default=30, + help="Maximum number of stories to display (default: 30)", + ) + parser.add_argument( + "--min-score", + type=int, + default=0, + help="Minimum score threshold (default: 0)", + ) + parser.add_argument( + "--json", + action="store_true", + help="Output as JSON instead of formatted text", + ) + args = parser.parse_args() + + try: + stories = scrape_hn(limit=args.limit, min_score=args.min_score) + + if args.json: + print(json.dumps(stories, indent=2)) + else: + display_stories(stories) + except requests.RequestException as e: + print(f"Error fetching data: {e}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/Web-Scraping/Weather-Forecast-Scraper/README.md b/Web-Scraping/Weather-Forecast-Scraper/README.md new file mode 100644 index 00000000..1dc835ca --- /dev/null +++ b/Web-Scraping/Weather-Forecast-Scraper/README.md @@ -0,0 +1,59 @@ +# Weather Forecast Scraper + +Fetches and displays 3-day weather forecast for any city using wttr.in. + +## Features + +- Fetches current weather conditions +- Displays 3-day forecast with temperature, humidity, rain chance +- Clean formatted output with emoji indicators +- No API key required (uses wttr.in) + +## Usage + +```bash +# Default city (London) +python weather_forecast.py + +# Specify city +python weather_forecast.py --city "New York" + +# Indian cities +python weather_forecast.py --city Mumbai +python weather_forecast.py --city Delhi +python weather_forecast.py --city Bangalore +``` + +## Output Example + +``` +============================================================ + Weather Forecast for London +============================================================ + + 🌡️ Current Conditions: + Temperature: 15°C (feels like 14°C) + Condition: Partly cloudy + Humidity: 72% + Wind: 12 km/h + Pressure: 1015 mb + Visibility: 10 km + + 📅 3-Day Forecast: + Date Max Min Avg Rain% Humidity Condition + ------------ ------ ------ ------ ------ -------- --------------- + 2026-10-11 17°C 11°C 14°C 10% 70% Partly cloudy + 2026-10-12 16°C 10°C 13°C 40% 75% Light rain + 2026-10-13 15°C 9°C 12°C 60% 80% Moderate rain +``` + +## Requirements + +- Python 3.6+ +- `requests` + +Install dependencies: + +```bash +pip install requests +``` \ No newline at end of file diff --git a/Web-Scraping/Weather-Forecast-Scraper/weather_forecast.py b/Web-Scraping/Weather-Forecast-Scraper/weather_forecast.py new file mode 100644 index 00000000..44dbcd6d --- /dev/null +++ b/Web-Scraping/Weather-Forecast-Scraper/weather_forecast.py @@ -0,0 +1,124 @@ +""" +Weather Forecast Scraper +Fetches and displays 3-day weather forecast for any city using wttr.in. + +Usage: + python weather_forecast.py [--city CITY] [--format FORMAT] + +Requirements: + pip install requests beautifulsoup4 +""" + +import argparse +import json +import sys + +import requests + +WTTR_URL = "https://wttr.in/{city}?format=j1" + + +def fetch_weather(city): + """Fetch weather data from wttr.in API.""" + url = WTTR_URL.format(city=city) + headers = {"User-Agent": "curl/7.68.0"} + + response = requests.get(url, headers=headers, timeout=15) + response.raise_for_status() + return response.json() + + +def parse_current_condition(data): + """Extract current weather conditions.""" + current = data.get("current_condition", [{}])[0] + return { + "temperature": current.get("temp_C", "N/A"), + "feels_like": current.get("FeelsLikeC", "N/A"), + "humidity": current.get("humidity", "N/A"), + "description": current.get("weatherDesc", [{}])[0].get("value", "N/A"), + "wind_speed": current.get("windspeedKmph", "N/A"), + "pressure": current.get("pressure", "N/A"), + "visibility": current.get("visibility", "N/A"), + } + + +def parse_forecast(data): + """Extract 3-day forecast.""" + forecasts = [] + weather = data.get("weather", []) + + for day in weather: + date = day.get("date", "N/A") + hourly = day.get("hourly", []) + + # Use midday data as representative + midday = next((h for h in hourly if h.get("time") == "1200"), hourly[0] if hourly else {}) + + forecasts.append({ + "date": date, + "max_temp": day.get("maxtempC", "N/A"), + "min_temp": day.get("mintempC", "N/A"), + "avg_temp": day.get("avgtempC", "N/A"), + "description": midday.get("weatherDesc", [{}])[0].get("value", "N/A"), + "chance_of_rain": midday.get("chanceofrain", "N/A"), + "humidity": midday.get("humidity", "N/A"), + "wind_speed": midday.get("windspeedKmph", "N/A"), + }) + + return forecasts + + +def display_weather(city, current, forecasts): + """Display weather in a formatted way.""" + print(f"\n{'='*60}") + print(f" Weather Forecast for {city.title()}") + print(f"{'='*60}\n") + + print(" 🌡️ Current Conditions:") + print(f" Temperature: {current['temperature']}°C (feels like {current['feels_like']}°C)") + print(f" Condition: {current['description']}") + print(f" Humidity: {current['humidity']}%") + print(f" Wind: {current['wind_speed']} km/h") + print(f" Pressure: {current['pressure']} mb") + print(f" Visibility: {current['visibility']} km") + print() + + print(" 📅 3-Day Forecast:") + print(f" {'Date':<12} {'Max':>6} {'Min':>6} {'Avg':>6} {'Rain%':>6} {'Humidity':>8} {'Condition'}") + print(f" {'-'*12} {'-'*6} {'-'*6} {'-'*6} {'-'*6} {'-'*8} {'-'*15}") + + for fc in forecasts: + print( + f" {fc['date']:<12} {fc['max_temp']:>5}°C {fc['min_temp']:>5}°C {fc['avg_temp']:>5}°C " + f"{fc['chance_of_rain']:>5}% {fc['humidity']:>7}% {fc['description']}" + ) + + print() + + +def main(): + parser = argparse.ArgumentParser( + description="Fetch and display weather forecast for any city" + ) + parser.add_argument( + "--city", + default="London", + help="City name for weather forecast (default: London)", + ) + args = parser.parse_args() + + try: + data = fetch_weather(args.city) + current = parse_current_condition(data) + forecasts = parse_forecast(data) + display_weather(args.city, current, forecasts) + except requests.RequestException as e: + print(f"Error fetching weather for '{args.city}': {e}", file=sys.stderr) + sys.exit(1) + except (KeyError, IndexError) as e: + print(f"Error parsing weather data: {e}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() \ No newline at end of file