From 461c261d125c41e9441d11adf1eae9d9cafad649 Mon Sep 17 00:00:00 2001 From: Sony Sebastian Date: Mon, 9 Jun 2025 18:19:49 +0200 Subject: [PATCH 1/3] bot blocking inginx configuration --- deploy/deploy-scripts/analytics-prod.yml | 60 +++++++++++++++++++++++- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/deploy/deploy-scripts/analytics-prod.yml b/deploy/deploy-scripts/analytics-prod.yml index 4419aef..4615692 100644 --- a/deploy/deploy-scripts/analytics-prod.yml +++ b/deploy/deploy-scripts/analytics-prod.yml @@ -105,6 +105,63 @@ metadata: kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/force-ssl-redirect: "true" cert-manager.io/cluster-issuer: bbys-platform-letsencrypt-prod + + # Rate limiting to prevent aggressive scraping + nginx.ingress.kubernetes.io/rate-limit: "10" + nginx.ingress.kubernetes.io/rate-limit-window: "1m" + nginx.ingress.kubernetes.io/rate-limit-connections: "5" + + # Block common bot user agents + nginx.ingress.kubernetes.io/server-snippet: | + # Block common bots and scrapers + if ($http_user_agent ~* (bot|crawler|spider|scraper|curl|wget|python|java|go-http|axios|postman|insomnia|httpie)) { + return 403; + } + + # Block requests without user agent + if ($http_user_agent = "") { + return 403; + } + + # Block suspicious user agents + if ($http_user_agent ~* (scan|hack|exploit|inject|attack|test)) { + return 403; + } + + # Block requests with suspicious headers + if ($http_x_forwarded_for ~* (tor-exit|proxy|vpn)) { + return 403; + } + + # Block common automated tools + if ($http_user_agent ~* (nikto|nmap|masscan|zap|burp|sqlmap|dirb|gobuster|ffuf)) { + return 403; + } + + # Block headless browsers commonly used for scraping + if ($http_user_agent ~* (headless|phantom|selenium|puppeteer)) { + return 403; + } + + # Block requests with no referrer and suspicious patterns + if ($http_referer = "" AND $request_method = "GET" AND $args ~ "(select|union|insert|delete|update|drop|create|alter)") { + return 403; + } + + # Additional security headers + nginx.ingress.kubernetes.io/configuration-snippet: | + more_set_headers "X-Frame-Options: DENY"; + more_set_headers "X-Content-Type-Options: nosniff"; + more_set_headers "X-XSS-Protection: 1; mode=block"; + more_set_headers "Referrer-Policy: strict-origin-when-cross-origin"; + + # Log blocked requests for monitoring + access_log /var/log/nginx/analytics-blocked.log combined if=$blocked_ua; + + # Whitelist legitimate monitoring services (optional) + nginx.ingress.kubernetes.io/whitelist-source-range: | + # Add your monitoring service IPs here if needed + # 1.2.3.4/32,5.6.7.8/32 spec: tls: - hosts: @@ -113,7 +170,6 @@ spec: - hosts: - analytics.barnebys.com secretName: tls-secret-bbys-analytics-com-service - rules: - host: analytics.barnebys.net http: @@ -134,4 +190,4 @@ spec: service: name: bbys-analytics-service-svc port: - number: 80 + number: 80 \ No newline at end of file From 6d93916262f177d6b888f0de29ea69772e299d86 Mon Sep 17 00:00:00 2001 From: Sony Sebastian Date: Mon, 9 Jun 2025 18:24:03 +0200 Subject: [PATCH 2/3] Update deploy/deploy-scripts/analytics-prod.yml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- deploy/deploy-scripts/analytics-prod.yml | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/deploy/deploy-scripts/analytics-prod.yml b/deploy/deploy-scripts/analytics-prod.yml index 4615692..17a3d84 100644 --- a/deploy/deploy-scripts/analytics-prod.yml +++ b/deploy/deploy-scripts/analytics-prod.yml @@ -113,18 +113,8 @@ metadata: # Block common bot user agents nginx.ingress.kubernetes.io/server-snippet: | - # Block common bots and scrapers - if ($http_user_agent ~* (bot|crawler|spider|scraper|curl|wget|python|java|go-http|axios|postman|insomnia|httpie)) { - return 403; - } - - # Block requests without user agent - if ($http_user_agent = "") { - return 403; - } - - # Block suspicious user agents - if ($http_user_agent ~* (scan|hack|exploit|inject|attack|test)) { + # Consolidated user agent checks + if ($http_user_agent ~* (bot|crawler|spider|scraper|curl|wget|python|java|go-http|axios|postman|insomnia|httpie|scan|hack|exploit|inject|attack|test|nikto|nmap|masscan|zap|burp|sqlmap|dirb|gobuster|ffuf|headless|phantom|selenium|puppeteer) OR $http_user_agent = "") { return 403; } @@ -133,16 +123,6 @@ metadata: return 403; } - # Block common automated tools - if ($http_user_agent ~* (nikto|nmap|masscan|zap|burp|sqlmap|dirb|gobuster|ffuf)) { - return 403; - } - - # Block headless browsers commonly used for scraping - if ($http_user_agent ~* (headless|phantom|selenium|puppeteer)) { - return 403; - } - # Block requests with no referrer and suspicious patterns if ($http_referer = "" AND $request_method = "GET" AND $args ~ "(select|union|insert|delete|update|drop|create|alter)") { return 403; From c763ac7d10971e1c7b3b33e7f9a83f5d1e985fab Mon Sep 17 00:00:00 2001 From: Sony Sebastian Date: Mon, 9 Jun 2025 18:26:19 +0200 Subject: [PATCH 3/3] NGINX if directives do not support the 'OR' operator. Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- deploy/deploy-scripts/analytics-prod.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/deploy/deploy-scripts/analytics-prod.yml b/deploy/deploy-scripts/analytics-prod.yml index 17a3d84..c8e2ca0 100644 --- a/deploy/deploy-scripts/analytics-prod.yml +++ b/deploy/deploy-scripts/analytics-prod.yml @@ -114,10 +114,12 @@ metadata: # Block common bot user agents nginx.ingress.kubernetes.io/server-snippet: | # Consolidated user agent checks - if ($http_user_agent ~* (bot|crawler|spider|scraper|curl|wget|python|java|go-http|axios|postman|insomnia|httpie|scan|hack|exploit|inject|attack|test|nikto|nmap|masscan|zap|burp|sqlmap|dirb|gobuster|ffuf|headless|phantom|selenium|puppeteer) OR $http_user_agent = "") { + if ($http_user_agent ~* (bot|crawler|spider|scraper|curl|wget|python|java|go-http|axios|postman|insomnia|httpie|scan|hack|exploit|inject|attack|test|nikto|nmap|masscan|zap|burp|sqlmap|dirb|gobuster|ffuf|headless|phantom|selenium|puppeteer)) { + return 403; + } + if ($http_user_agent = "") { return 403; } - # Block requests with suspicious headers if ($http_x_forwarded_for ~* (tor-exit|proxy|vpn)) { return 403;