Minor patch for sitemap parsing

Author: stamparm

data/txt/sha256sums.txt

@@ -188,7 +188,7 @@ c1cb56f2a43e9f2f6b25d5f3d504e856ea21df6fc14af5e37b1000feef2bdb5a  lib/core/optio
 48797d6c34dd9bb8a53f7f3794c85f4288d82a9a1d6be7fcf317d388cb20d4b3  lib/core/replication.py
 0b8c38a01bb01f843d94a6c5f2075ee47520d0c4aa799cecea9c3e2c5a4a23a6  lib/core/revision.py
 888daba83fd4a34e9503fe21f01fef4cc730e5cde871b1d40e15d4cbc847d56c  lib/core/session.py
-66cbde8c76851d5e8de6b93d6dbd3cd58e4473a1f307326aa672c3628253775e  lib/core/settings.py
+8508162b2a95e54102ee8aec95888d7e2061d73b7d0e9ecd47d4f5e22ca94820  lib/core/settings.py
 cd5a66deee8963ba8e7e9af3dd36eb5e8127d4d68698811c29e789655f507f82  lib/core/shell.py
 bcb5d8090d5e3e0ef2a586ba09ba80eef0c6d51feb0f611ed25299fbb254f725  lib/core/subprocessng.py
 d35650179816193164a5f177102f18379dfbe6bb6d40fbb67b78d907b41c8038  lib/core/target.py
@@ -206,7 +206,7 @@ c5b258be7485089fac9d9cd179960e774fbd85e62836dc67cce76cc028bb6aeb  lib/parse/hand
 1ad9054cd8476a520d4e2c141085ae45d94519df5c66f25fac41fe7d552ab952  lib/parse/html.py
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/parse/__init__.py
 d2e771cdacef25ee3fdc0e0355b92e7cd1b68f5edc2756ffc19f75d183ba2c73  lib/parse/payloads.py
-80d26a30abe948faf817a14f746cc8b3e2341ea8286830cccaae253b8ac0cdff  lib/parse/sitemap.py
+455ab0ec63e55cd56ce4a884b85bdc089223155008cab0f3696da5a33118f95b  lib/parse/sitemap.py
 1be3da334411657461421b8a26a0f2ff28e1af1e28f1e963c6c92768f9b0847c  lib/request/basicauthhandler.py
 a1c638493ecdc5194db7186bbfed815c6eed2344f2607cac8c9fa50534824266  lib/request/basic.py
 bc61bc944b81a7670884f82231033a6ac703324b34b071c9834886a92e249d0e  lib/request/chunkedhandler.py

lib/core/settings.py

@@ -20,7 +20,7 @@
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.2.2"
+VERSION = "1.10.2.3"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)

lib/parse/sitemap.py

@@ -17,7 +17,7 @@
 
 abortedFlag = None
 
-def parseSitemap(url, retVal=None):
+def parseSitemap(url, retVal=None, visited=None):
     global abortedFlag
 
     if retVal is not None:
@@ -27,25 +27,41 @@ def parseSitemap(url, retVal=None):
         if retVal is None:
             abortedFlag = False
             retVal = OrderedSet()
+            visited = set()
+
+        if url in visited:
+            return retVal
+
+        visited.add(url)
 
         try:
             content = Request.getPage(url=url, raise404=True)[0] if not abortedFlag else ""
         except _http_client.InvalidURL:
             errMsg = "invalid URL given for sitemap ('%s')" % url
             raise SqlmapSyntaxException(errMsg)
 
-        for match in re.finditer(r"<loc>\s*([^<]+)", content or ""):
-            if abortedFlag:
-                break
-            url = match.group(1).strip()
-            if url.endswith(".xml") and "sitemap" in url.lower():
-                if kb.followSitemapRecursion is None:
-                    message = "sitemap recursion detected. Do you want to follow? [y/N] "
-                    kb.followSitemapRecursion = readInput(message, default='N', boolean=True)
-                if kb.followSitemapRecursion:
-                    parseSitemap(url, retVal)
-            else:
-                retVal.add(url)
+        if content:
+            content = re.sub(r"", "", content, flags=re.DOTALL)
+
+            for match in re.finditer(r"<\w*?loc[^>]*>\s*([^<]+)", content, re.I):
+                if abortedFlag:
+                    break
+
+                foundUrl = match.group(1).strip()
+
+                # Basic validation to avoid junk
+                if not foundUrl.startswith("http"):
+                    continue
+
+                if foundUrl.endswith(".xml") and "sitemap" in foundUrl.lower():
+                    if kb.followSitemapRecursion is None:
+                        message = "sitemap recursion detected. Do you want to follow? [y/N] "
+                        kb.followSitemapRecursion = readInput(message, default='N', boolean=True)
+
+                    if kb.followSitemapRecursion:
+                        parseSitemap(foundUrl, retVal, visited)
+                else:
+                    retVal.add(foundUrl)
 
     except KeyboardInterrupt:
         abortedFlag = True