fix(search): parse current Bing result links

While looking into #6009 I noticed the Bing fallback in -g never
returns anything, even when Bing itself serves a normal results page.

Two things changed on Bing's side. The result anchor now carries
target="_blank" before href, so '<h2><a href="..." h=' no longer
matches at all. And the href is no longer the target URL but a
tracking redirect (https://www.bing.com/ck/a?...&u=a1<base64url>),
where the real link sits base64url-encoded after the "a1" prefix
in the u parameter.

The regex now allows extra attributes on both tags and captures
either the u payload of a ck/a redirect or a plain href. The payload
is decoded with decodeBase64(), which already deals with the
URL-safe alphabet and the missing padding. Plain hrefs still work
the same as before (with &amp; unescaped), in case Bing serves the
old markup to some clients.

Checked against a saved live results page (0 -> 10 links) and the
old markup, and --smoke-test passes. Google still can't be scraped
since it only serves a JS page now, so this doesn't fully close
#6009, but Bing is usable again as a fallback when DuckDuckGo blocks
the IP.

Author: raju4199

lib/core/settings.py

@@ -214,7 +214,7 @@
 DUCKDUCKGO_REGEX = r'<a class="result__url" href="(htt[^"]+)'
 
 # Regular expression used for extracting results from Bing search
-BING_REGEX = r'<h2><a href="([^"]+)" h='
+BING_REGEX = r'<h2[^>]*><a [^>]*href="(?:https?://www\.bing\.com/ck/a\?[^"]*?[?;]u=a1([^&"]+)[^"]*|([^"]+))"'
 
 # Dummy user agent for search (if default one returns different results)
 DUMMY_SEARCH_USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64; rv:141.0) Gecko/20100101 Firefox/141.0"

lib/utils/search.py

@@ -12,6 +12,7 @@
 from lib.core.common import popValue
 from lib.core.common import pushValue
 from lib.core.common import urlencode
+from lib.core.convert import decodeBase64
 from lib.core.convert import getBytes
 from lib.core.convert import getUnicode
 from lib.core.data import conf
@@ -100,7 +101,7 @@ def _search(dork):
     # Note: (name, url, POST data, regex, regex flags, match->link). Ordered by current scraping reliability; tried in turn until one yields results (DuckDuckGo currently being the only consistently scrapeable one)
     engines = (
         ("DuckDuckGo", "https://html.duckduckgo.com/html/", "q=%s&s=%d" % (encoded, (gpage - 1) * 30), DUCKDUCKGO_REGEX, re.I | re.S, lambda match: match.group(1).replace("&", "&")),
-        ("Bing", "https://www.bing.com/search?q=%s&first=%d" % (encoded, (gpage - 1) * 10 + 1), None, BING_REGEX, re.I | re.S, lambda match: match.group(1)),
+        ("Bing", "https://www.bing.com/search?q=%s&first=%d" % (encoded, (gpage - 1) * 10 + 1), None, BING_REGEX, re.I | re.S, lambda match: decodeBase64(match.group(1), binary=False) if match.group(1) else match.group(2).replace("&", "&")),
         ("Google", "https://www.google.com/search?q=%s&num=100&hl=en&complete=0&safe=off&filter=0&btnG=Search&start=%d" % (encoded, (gpage - 1) * 100), None, GOOGLE_REGEX, re.I, lambda match: match.group(1) or match.group(2)),
     )