[pull] master from sqlmapproject:master

data/txt/sha256sums.txt

@@ -160,19 +160,19 @@ ca86d61d3349ed2d94a6b164d4648cff9701199b5e32378c3f40fca0f517b128  extra/shutils/
 df768bcb9838dc6c46dab9b4a877056cb4742bd6cfaaf438c4a3712c5cc0d264  extra/shutils/recloak.sh
 1972990a67caf2d0231eacf60e211acf545d9d0beeb3c145a49ba33d5d491b3f  extra/shutils/strip.sh
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  extra/vulnserver/__init__.py
-11fd73d2a49ae110dff6ee9c28a6703d7573187d639a11a190f699221612b488  extra/vulnserver/vulnserver.py
+6ce9405808514d27e7600c4d37f0bacca99205df732f1319170efffccb1ee1ad  extra/vulnserver/vulnserver.py
 b8411d1035bb49b073476404e61e1be7f4c61e205057730e2f7880beadcd5f60  lib/controller/action.py
-460d3da652b8f55c9eaf0f90be33eddf3355355e5c5b1c98b7fc4d83b1c54fda  lib/controller/checks.py
+e376093d4f6e42ee38b050af329179df9c1c136b7667b2f1cb559f5d4b69ebd9  lib/controller/checks.py
 430475857a37fd997e73a47d7485c5dd4aa0985ef32c5a46b5e7bff01749ba66  lib/controller/controller.py
 ccec2373f6393f3d644db3de2910e17ef705817063c03e7ca4417f9d7f622527  lib/controller/handler.py
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/controller/__init__.py
 6da126b359e67f73cea7848d3f35dd0890aece16374d04b60490b85e26bf7224  lib/core/agent.py
 1da4ec9cd9b67c8b54e4a3d314f8237d58778d8f3a00bc26a1e0540294dca30f  lib/core/bigarray.py
-ed02b196398b8351ed6989c8fd8ec2a8244f2f9da6ca7b08691219dcc63422d8  lib/core/common.py
+5c05d5e27b987b47c4c66e4233e3f33eae77cffc8d1b2d90cb5439c9fafd9b7c  lib/core/common.py
 a6397b10de7ae7c56ed6b0fa3b3c58eb7a9dbede61bf93d786e73258175c981e  lib/core/compat.py
 a9997e97ebe88e0bf7efcf21e878bc5f62c72348e5aba18f64d6861390a4dcf2  lib/core/convert.py
 c03dc585f89642cfd81b087ac2723e3e1bb3bfa8c60e6f5fe58ef3b0113ebfe6  lib/core/data.py
-421509c42dab738d908f2453cbdd6eb75eb672a7b6de68bee8c95d867fac79f1  lib/core/datatype.py
+e396b7971d38896e0e20b973a3a6a3fbc3171d080a21bc6e66a65bee452fd69c  lib/core/datatype.py
 35e99a327a357e22dcbcd5229a347102dfc58ff8105b420afdeffc8236a9ecf4  lib/core/decorators.py
 147823c37596bd6a56d677697781f34b8d1d1671d5a2518fbc9468d623c6d07d  lib/core/defaults.py
 86fa0ffa7a3e7a7141eab730e3981faf6f0249125ea9a29a57aaa8b65b7503f9  lib/core/dicts.py
@@ -189,11 +189,11 @@ f5272cda54f7cdd07fb6154d5a1ed1f1141a2a4f39b6a85d3f325fd60ac8dc9a  lib/core/enums
 48797d6c34dd9bb8a53f7f3794c85f4288d82a9a1d6be7fcf317d388cb20d4b3  lib/core/replication.py
 3574639db4942d16a2dc0a2f04bb7c0913c40c3862b54d34c44075a760e0c194  lib/core/revision.py
 888daba83fd4a34e9503fe21f01fef4cc730e5cde871b1d40e15d4cbc847d56c  lib/core/session.py
-fbc29e35ddd484f2b8969692337f77bdeb709ff646f08281b69fd5da29062b67  lib/core/settings.py
+3b56bc9e795809f99cf502cc0f0c1c5b046f03720a6fd43192add592c5e90fdd  lib/core/settings.py
 cd5a66deee8963ba8e7e9af3dd36eb5e8127d4d68698811c29e789655f507f82  lib/core/shell.py
 bcb5d8090d5e3e0ef2a586ba09ba80eef0c6d51feb0f611ed25299fbb254f725  lib/core/subprocessng.py
 d35650179816193164a5f177102f18379dfbe6bb6d40fbb67b78d907b41c8038  lib/core/target.py
-85b7d6a724536bfcadd317972d4baec291e3813d6773921ee31755046a950a9a  lib/core/testing.py
+3167680c58037217ae1a0c4f65d278dbe5d72c8837ed209c725451670dde53ae  lib/core/testing.py
 cf4dca323645d623109a82277a8e8a63eb9abb3fff6c8a57095eb171c1ef91b3  lib/core/threads.py
 b9aacb840310173202f79c2ba125b0243003ee6b44c92eca50424f2bdfc83c02  lib/core/unescaper.py
 10719f5ca450610ad28242017b2d8a77354ca357ffa26948c5f62d20cac29a8b  lib/core/update.py
@@ -209,7 +209,7 @@ c5b258be7485089fac9d9cd179960e774fbd85e62836dc67cce76cc028bb6aeb  lib/parse/hand
 4ca378496510a02c0184b45107889625dc7faf459073e83b3520c66674049af4  lib/parse/payloads.py
 80d26a30abe948faf817a14f746cc8b3e2341ea8286830cccaae253b8ac0cdff  lib/parse/sitemap.py
 1be3da334411657461421b8a26a0f2ff28e1af1e28f1e963c6c92768f9b0847c  lib/request/basicauthhandler.py
-7302c38b8b418530a988b0294d09f0a9a08b7b35bc488956fee491a6dd88b2d1  lib/request/basic.py
+a63147dff1fd4675c800122768d9d6564ef4b2f8f8d57abc2799c6bfe7a52c48  lib/request/basic.py
 bc61bc944b81a7670884f82231033a6ac703324b34b071c9834886a92e249d0e  lib/request/chunkedhandler.py
 2daf0ce19eacda64687f441c90ef8da51714c3e8947c993ba08fb4ecdc4f5287  lib/request/comparison.py
 626bb6f3316a906a4629c0feb8ecbbcf473fb59e5bc532603c35b6b8f63f1deb  lib/request/connect.py
@@ -478,7 +478,7 @@ eb45fd711efa71ab9d91d815cc8abebc9abc4770311fbb827159008b000f4fc2  plugins/generi
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  plugins/__init__.py
 423d9bfaddb3cf527d02ddda97e53c4853d664c51ef7be519e4f45b9e399bc30  README.md
 c6ad39bfd1810413402dedfc275fc805fa13f85fc490e236c1e725bde4e5100b  sqlmapapi.py
-168309215af7dd5b0b71070e1770e72f1cbb29a3d8025143fb8aa0b88cd56b62  sqlmapapi.yaml
+4e993cfe2889bf0f86ad0abafd9a6a25849580284ea279b2115e99707e14bb97  sqlmapapi.yaml
 a40607ce164eb2d21865288d24b863edb1c734b56db857e130ac1aef961c80b9  sqlmap.conf
 e9d3d52d4c0698b956cc0dc92c177d432b1f97c5918f750baa3e737de4ae574b  sqlmap.py
 eb37a88357522fd7ad00d90cdc5da6b57442b4fec49366aadb2944c4fbf8b804  tamper/0eunion.py

extra/vulnserver/vulnserver.py

@@ -11,8 +11,10 @@
 
 import base64
 import json
+import random
 import re
 import sqlite3
+import string
 import sys
 import threading
 import traceback
@@ -49,9 +51,20 @@
     );
     INSERT INTO users (id, name, surname) VALUES (1, 'luther', 'blisset');
     INSERT INTO users (id, name, surname) VALUES (2, 'fluffy', 'bunny');
-    INSERT INTO users (id, name, surname) VALUES (3, 'wu', '179ad45c6ce2cb97cf1029e212046e81');
-    INSERT INTO users (id, name, surname) VALUES (4, 'sqlmap/1.0-dev (https://sqlmap.org)', 'user agent header');
-    INSERT INTO users (id, name, surname) VALUES (5, NULL, 'nameisnull');
+    INSERT INTO users (id, name, surname) VALUES (3, 'wu', 'ming');
+    INSERT INTO users (id, name, surname) VALUES (4, NULL, 'nameisnull');
+    INSERT INTO users (id, name, surname) VALUES (5, 'sqlmap/1.0-dev (https://sqlmap.org)', 'user agent header');
+
+    CREATE TABLE creds (
+        user_id INTEGER,
+        password_hash TEXT,
+        FOREIGN KEY (user_id) REFERENCES users(id)
+    );
+    INSERT INTO creds (user_id, password_hash) VALUES (1, 'db3a16990a0008a3b04707fdef6584a0');
+    INSERT INTO creds (user_id, password_hash) VALUES (2, '4db967ce67b15e7fb84c266a76684729');
+    INSERT INTO creds (user_id, password_hash) VALUES (3, 'f5a2950eaa10f9e99896800eacbe8275');
+    INSERT INTO creds (user_id, password_hash) VALUES (4, NULL);
+    INSERT INTO creds (user_id, password_hash) VALUES (5, '179ad45c6ce2cb97cf1029e212046e81');
 """
 
 LISTEN_ADDRESS = "localhost"
@@ -62,11 +75,15 @@
 _lock = None
 _server = None
 _alive = False
+_csrf_token = None
 
 def init(quiet=False):
     global _conn
     global _cursor
     global _lock
+    global _csrf_token
+
+    _csrf_token = "".join(random.sample(string.ascii_letters + string.digits, 20))
 
     _conn = sqlite3.connect(":memory:", isolation_level=None, check_same_thread=False)
     _cursor = _conn.cursor()
@@ -131,6 +148,28 @@ def do_REQUEST(self):
 
         self.url, self.params = path, params
 
+        if self.url == "/csrf":
+            if self.params.get("csrf_token") == _csrf_token:
+                self.url = "/"
+            else:
+                self.send_response(OK)
+                self.send_header("Content-type", "text/html; charset=%s" % UNICODE_ENCODING)
+                self.end_headers()
+
+                form = (
+                    "<html><body>"
+                    "CSRF protection check<br>"
+                    "<form action='/csrf' method='POST'>"
+                    "<input type='hidden' name='csrf_token' value='%s'>"
+                    "id: <input type='text' name='id'>"
+                    "<input type='submit' value='Submit'>"
+                    "</form>"
+                    "</body></html>"
+                ) % _csrf_token
+
+                self.wfile.write(form.encode(UNICODE_ENCODING))
+                return
+
         if self.url == '/':
             if not any(_ in self.params for _ in ("id", "query")):
                 self.send_response(OK)
@@ -139,7 +178,7 @@ def do_REQUEST(self):
                 self.end_headers()
                 self.wfile.write(b"<!DOCTYPE html><html><head><title>vulnserver</title></head><body><h3>GET:</h3><a href='/?id=1'>link</a><hr><h3>POST:</h3><form method='post'>ID: <input type='text' name='id'><input type='submit' value='Submit'></form></body></html>")
             else:
-                code, output = OK, ""
+                code, output = OK, "<body><html>"
 
                 try:
                     if self.params.get("echo", ""):
@@ -177,6 +216,11 @@ def do_REQUEST(self):
                         else:
                             output += "no results found"
 
+                        if not results:
+                            output = "<title>No results</title>" + output
+                        else:
+                            output = "<title>Results</title>" + output
+
                     output += "</body></html>"
                 except Exception as ex:
                     code = INTERNAL_SERVER_ERROR

lib/controller/checks.py

@@ -554,7 +554,7 @@ def genCmpPayload():
 
                                     injectable = True
 
-                                elif (threadData.lastComparisonRatio or 0) > UPPER_RATIO_BOUND and not any((conf.string, conf.notString, conf.regexp, conf.code, kb.nullConnection)):
+                                elif (threadData.lastComparisonRatio or 0) > UPPER_RATIO_BOUND and not any((conf.string, conf.notString, conf.regexp, conf.code, conf.titles, kb.nullConnection)):
                                     originalSet = set(getFilteredPageContent(kb.pageTemplate, True, "\n").split("\n"))
                                     trueSet = set(getFilteredPageContent(truePage, True, "\n").split("\n"))
                                     falseSet = set(getFilteredPageContent(falsePage, True, "\n").split("\n"))
@@ -580,7 +580,7 @@ def genCmpPayload():
                                                     break
 
                             if injectable:
-                                if kb.pageStable and not any((conf.string, conf.notString, conf.regexp, conf.code, kb.nullConnection)):
+                                if kb.pageStable and not any((conf.string, conf.notString, conf.regexp, conf.code, conf.titles, kb.nullConnection)):
                                     if all((falseCode, trueCode)) and falseCode != trueCode and trueCode != kb.heuristicCode:
                                         suggestion = conf.code = trueCode
 

lib/core/common.py

@@ -3461,7 +3461,10 @@ def parseSqliteTableSchema(value):
             columns[column] = match.group(3) or "TEXT"
 
         table[safeSQLIdentificatorNaming(conf.tbl, True)] = columns
-        kb.data.cachedColumns[conf.db] = table
+        if conf.db in kb.data.cachedColumns:
+            kb.data.cachedColumns[conf.db].update(table)
+        else:
+            kb.data.cachedColumns[conf.db] = table
 
     return retVal
 

lib/core/datatype.py

@@ -170,7 +170,7 @@ def __setitem__(self, key, value):
             except KeyError:
                 if len(self.cache) >= self.capacity:
                     self.cache.popitem(last=False)
-        self.cache[key] = value
+            self.cache[key] = value
 
     def set(self, key, value):
         self.__setitem__(key, value)

lib/core/settings.py

@@ -19,7 +19,7 @@
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.1.13"
+VERSION = "1.10.1.19"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)

lib/core/testing.py

@@ -43,7 +43,7 @@ def vulnTest():
         ("-u <url> --data=\"reflect=1\" --flush-session --wizard --disable-coloring", ("Please choose:", "back-end DBMS: SQLite", "current user is DBA: True", "banner: '3.")),
         ("-u <url> --data=\"code=1\" --code=200 --technique=B --banner --no-cast --flush-session", ("back-end DBMS: SQLite", "banner: '3.", "~COALESCE(CAST(")),
         (u"-c <config> --flush-session --output-dir=\"<tmpdir>\" --smart --roles --statements --hostname --privileges --sql-query=\"SELECT '\u0161u\u0107uraj'\" --technique=U", (u": '\u0161u\u0107uraj'", "on SQLite it is not possible", "as the output directory")),
-        (u"-u <url> --flush-session --sql-query=\"SELECT '\u0161u\u0107uraj'\" --technique=B --no-escape --string=luther --unstable", (u": '\u0161u\u0107uraj'",)),
+        (u"-u <url> --flush-session --sql-query=\"SELECT '\u0161u\u0107uraj'\" --titles --technique=B --no-escape --string=luther --unstable", (u": '\u0161u\u0107uraj'", "~with --string",)),
         ("-m <multiple> --flush-session --technique=B --banner", ("/3] URL:", "back-end DBMS: SQLite", "banner: '3.")),
         ("--dummy", ("all tested parameters do not appear to be injectable", "does not seem to be injectable", "there is not at least one", "~might be injectable")),
         ("-u \"<url>&id2=1\" -p id2 -v 5 --flush-session --level=5 --text-only --test-filter=\"AND boolean-based blind - WHERE or HAVING clause (MySQL comment)\"", ("~1AND",)),
@@ -62,7 +62,7 @@ def vulnTest():
         ("-u <base> --flush-session -H \"Foo: Bar\" -H \"Sna: Fu\" --data=\"<root><param name=\\\"id\\\" value=\\\"1*\\\"/></root>\" --union-char=1 --mobile --answers=\"smartphone=3\" --banner --smart -v 5", ("might be injectable", "Payload: <root><param name=\"id\" value=\"1", "Type: boolean-based blind", "Type: time-based blind", "Type: UNION query", "banner: '3.", "Nexus", "Sna: Fu", "Foo: Bar")),
         ("-u <base> --flush-session --technique=BU --method=PUT --data=\"a=1;id=1;b=2\" --param-del=\";\" --skip-static --har=<tmpfile> --dump -T users --start=1 --stop=2", ("might be injectable", "Parameter: id (PUT)", "Type: boolean-based blind", "Type: UNION query", "2 entries")),
         ("-u <url> --flush-session -H \"id: 1*\" --tables -t <tmpfile>", ("might be injectable", "Parameter: id #1* ((custom) HEADER)", "Type: boolean-based blind", "Type: time-based blind", "Type: UNION query", " users ")),
-        ("-u <url> --flush-session --banner --invalid-logical --technique=B --predict-output --test-filter=\"OR boolean\" --tamper=space2dash", ("banner: '3.", " LIKE ")),
+        ("-u <url> --flush-session --banner --invalid-logical --technique=B --predict-output --titles --test-filter=\"OR boolean\" --tamper=space2dash", ("banner: '3.", " LIKE ")),
         ("-u <url> --flush-session --cookie=\"PHPSESSID=d41d8cd98f00b204e9800998ecf8427e; id=1*; id2=2\" --tables --union-cols=3", ("might be injectable", "Cookie #1* ((custom) HEADER)", "Type: boolean-based blind", "Type: time-based blind", "Type: UNION query", " users ")),
         ("-u <url> --flush-session --null-connection --technique=B --tamper=between,randomcase --banner --count -T users", ("NULL connection is supported with HEAD method", "banner: '3.", "users | 5")),
         ("-u <base> --data=\"aWQ9MQ==\" --flush-session --base64=POST -v 6", ("aWQ9MTtXQUlURk9SIERFTEFZICcwOjA",)),
@@ -73,8 +73,9 @@ def vulnTest():
         ("-u <url> -z \"tec=B\" --hex --fresh-queries --threads=4 --sql-query=\"SELECT * FROM users\"", ("SELECT * FROM users [5]", "nameisnull")),
         ("-u \"<url>&echo=foobar*\" --flush-session", ("might be vulnerable to cross-site scripting",)),
         ("-u \"<url>&query=*\" --flush-session --technique=Q --banner", ("Title: SQLite inline queries", "banner: '3.")),
-        ("-d \"<direct>\" --flush-session --dump -T users --dump-format=SQLITE --binary-fields=name --where \"id=3\"", ("7775", "179ad45c6ce2cb97cf1029e212046e81 (testpass)", "dumped to SQLITE database")),
-        ("-d \"<direct>\" --flush-session --banner --schema --sql-query=\"UPDATE users SET name='foobar' WHERE id=5; SELECT * FROM users; SELECT 987654321\"", ("banner: '3.", "INTEGER", "TEXT", "id", "name", "surname", "5,foobar,nameisnull", "'987654321'",)),
+        ("-d \"<direct>\" --flush-session --dump -T creds --dump-format=SQLITE --binary-fields=password_hash --where \"user_id=5\"", ("3137396164343563366365326362393763663130323965323132303436653831", "dumped to SQLITE database")),
+        ("-d \"<direct>\" --flush-session --banner --schema --sql-query=\"UPDATE users SET name='foobar' WHERE id=4; SELECT * FROM users; SELECT 987654321\"", ("banner: '3.", "INTEGER", "TEXT", "id", "name", "surname", "4,foobar,nameisnull", "'987654321'",)),
+        ("-u <base>csrf --data=\"id=1&csrf_token=1\" --banner --answers=\"update=y\" --flush-session", ("back-end DBMS: SQLite", "banner: '3.")),
         ("--purge -v 3", ("~ERROR", "~CRITICAL", "deleting the whole directory tree")),
     )
 

lib/request/basic.py

@@ -259,8 +259,11 @@ def getHeuristicCharEncoding(page):
     """
 
     key = (len(page), hash(page))
-    retVal = kb.cache.encoding[key] if key in kb.cache.encoding else detect(page[:HEURISTIC_PAGE_SIZE_THRESHOLD])["encoding"]
-    kb.cache.encoding[key] = retVal
+
+    retVal = kb.cache.encoding.get(key)
+    if retVal is None:
+        retVal = detect(page[:HEURISTIC_PAGE_SIZE_THRESHOLD])["encoding"]
+        kb.cache.encoding[key] = retVal
 
     if retVal and retVal.lower().replace('-', "") == UNICODE_ENCODING.lower().replace('-', ""):
         infoMsg = "heuristics detected web page charset '%s'" % retVal