Doing some cleaning for web search requests

Author: stamparm

lib/core/settings.py

@@ -12,15 +12,14 @@
 import re
 import string
 import sys
-import time
 
 from lib.core.enums import DBMS
 from lib.core.enums import DBMS_DIRECTORY_NAME
 from lib.core.enums import OS
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.10.16"
+VERSION = "1.10.10.17"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -204,12 +203,6 @@
     ("", 200),
 )
 
-# Regular expression used for extracting results from Google search
-GOOGLE_REGEX = r"webcache\.googleusercontent\.com/search\?q=cache:[^:]+:([^+]+)\+&amp;cd=|url\?\w+=((?![^>]+webcache\.googleusercontent\.com)http[^>]+)&(sa=U|rct=j)"
-
-# Google Search consent cookie
-GOOGLE_CONSENT_COOKIE = "CONSENT=YES+shp.gws-%s-0-RC1.%s+FX+740" % (time.strftime("%Y%m%d"), "".join(random.sample(string.ascii_lowercase, 2)))
-
 # Regular expression used for extracting results from DuckDuckGo search
 DUCKDUCKGO_REGEX = r'<a class="result__url" href="(htt[^"]+)'
 

lib/parse/cmdline.py

@@ -138,8 +138,8 @@ def cmdLineParser(argv=None):
         target.add_argument("-r", dest="requestFile",
             help="Load HTTP request from a file")
 
-        target.add_argument("-g", dest="googleDork",
-            help="Process Google dork results as target URLs")
+        target.add_argument("-g", "--dork", dest="googleDork", metavar="DORK",
+            help="Process search engine dork results as target URLs")
 
         target.add_argument("-c", dest="configFile",
             help="Load options from a configuration INI file")
@@ -736,8 +736,8 @@ def cmdLineParser(argv=None):
         general.add_argument("--fresh-queries", dest="freshQueries", action="store_true",
             help="Ignore query results stored in session file")
 
-        general.add_argument("--gpage", dest="googlePage", type=int,
-            help="Use Google dork results from specified page number")
+        general.add_argument("--gpage", dest="googlePage", type=int, metavar="PAGE",
+            help="Use dork results from specified page number")
 
         general.add_argument("--har", dest="harFile",
             help="Log all HTTP traffic into a HAR file")

lib/utils/search.py

@@ -26,8 +26,6 @@
 from lib.core.settings import BING_REGEX
 from lib.core.settings import DUCKDUCKGO_REGEX
 from lib.core.settings import DUMMY_SEARCH_USER_AGENT
-from lib.core.settings import GOOGLE_CONSENT_COOKIE
-from lib.core.settings import GOOGLE_REGEX
 from lib.core.settings import HTTP_ACCEPT_ENCODING_HEADER_VALUE
 from lib.core.settings import SEARCH_ABUSE_MARKERS
 from lib.core.settings import UNICODE_ENCODING
@@ -91,19 +89,20 @@ def _search(dork):
     requestHeaders = {
         HTTP_HEADER.USER_AGENT: dict(conf.httpHeaders).get(HTTP_HEADER.USER_AGENT, DUMMY_SEARCH_USER_AGENT),
         HTTP_HEADER.ACCEPT_ENCODING: HTTP_ACCEPT_ENCODING_HEADER_VALUE,
-        HTTP_HEADER.COOKIE: GOOGLE_CONSENT_COOKIE,
     }
 
     gpage = conf.googlePage if conf.googlePage > 1 else 1
     logger.info("using search result page #%d" % gpage)
 
     encoded = urlencode(dork, convall=True)
 
-    # Note: (name, url, POST data, regex, regex flags, match->link). Ordered by current scraping reliability; tried in turn until one yields results (DuckDuckGo currently being the only consistently scrapeable one)
+    # Note: DuckDuckGo's first result page holds 10 entries, while each subsequent one holds 15 (hence the non-linear offset)
+    ddgOffset = 0 if gpage == 1 else 10 + (gpage - 2) * 15
+
+    # Note: (name, url, POST data, regex, regex flags, match->link). Ordered by current scraping reliability; tried in turn until one yields results (Google is not in here as it serves results only to JavaScript capable clients)
     engines = (
-        ("DuckDuckGo", "https://html.duckduckgo.com/html/", "q=%s&s=%d" % (encoded, (gpage - 1) * 30), DUCKDUCKGO_REGEX, re.I | re.S, lambda match: match.group(1).replace("&", "&")),
+        ("DuckDuckGo", "https://html.duckduckgo.com/html/", "q=%s&s=%d" % (encoded, ddgOffset), DUCKDUCKGO_REGEX, re.I | re.S, lambda match: match.group(1).replace("&", "&")),
         ("Bing", "https://www.bing.com/search?q=%s&first=%d" % (encoded, (gpage - 1) * 10 + 1), None, BING_REGEX, re.I | re.S, lambda match: decodeBase64(match.group(1), binary=False) if match.group(1) else match.group(2).replace("&", "&")),
-        ("Google", "https://www.google.com/search?q=%s&num=100&hl=en&complete=0&safe=off&filter=0&btnG=Search&start=%d" % (encoded, (gpage - 1) * 100), None, GOOGLE_REGEX, re.I, lambda match: match.group(1) or match.group(2)),
     )
 
     for name, url, data, regex, flags, extract in engines:

sqlmap.conf

@@ -25,8 +25,8 @@ bulkFile =
 # Example (file content): POST /login.jsp HTTP/1.1\nHost: example.com\nUser-Agent: Mozilla/4.0\n\nuserid=joe&password=guessme
 requestFile = 
 
-# Rather than providing a target URL, let Google return target
-# hosts as result of your Google dork expression. For a list of Google
+# Rather than providing a target URL, let the search engine return target
+# hosts as result of your dork expression. For a list of
 # dorks see Google Hacking Database at
 # https://www.exploit-db.com/google-hacking-database
 # Example: +ext:php +inurl:"&id=" +intext:"powered by "
@@ -797,7 +797,7 @@ forms = False
 # Valid: True or False
 freshQueries = False
 
-# Use Google dork results from specified page number.
+# Use dork results from specified page number.
 # Valid: integer
 # Default: 1
 googlePage = 1

tests/test_dork_search.py

@@ -42,7 +42,13 @@ def _pages(self, *pages):
         """Patches _fetch to serve `pages` in engine order (None meaning no reply)"""
 
         served = list(pages)
-        smod._fetch = lambda *args, **kwargs: served.pop(0) if served else None
+        self.requested = []
+
+        def fetch(url, headers, data=None):
+            self.requested.append((url, data))
+            return served.pop(0) if served else None
+
+        smod._fetch = fetch
 
     def test_duckduckgo_results(self):
         self._pages(DUCKDUCKGO_PAGE)
@@ -52,6 +58,14 @@ def test_bing_fallback_decodes_redirect(self):
         self._pages(None, BING_PAGE)
         self.assertEqual(smod._search("dork"), ["https://www.example.com/a.php?id=1"])
 
+    def test_duckduckgo_paging_offset(self):
+        # Note: first result page holds 10 entries, each subsequent one 15 (and not the uniform 30 as previously used)
+        for gpage, offset in ((1, 0), (2, 10), (3, 25), (4, 40)):
+            conf.googlePage = gpage
+            self._pages(DUCKDUCKGO_PAGE)
+            smod._search("dork")
+            self.assertEqual(self.requested[0][1], "q=dork&s=%d" % offset)
+
     def test_abuse_page_switches_proxy(self):
         # Note: without '--proxy-list' there is nothing to switch to, hence no exception
         for marker in SEARCH_ABUSE_MARKERS: