CERBERUS/sqlmap
1
0
Fork 0
mirror of https://github.com/sqlmapproject/sqlmap.git synced 2026-07-10 18:43:07 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Switching from pickle to json for serialization

Browse source
This commit is contained in:
Miroslav Štampar 2026-07-08 12:13:49 +02:00
parent e2c55c1ddc
commit 4504edaeb1
12 changed files with 933 additions and 273 deletions
Download patch file Download diff file Expand all files Collapse all files

20
lib/core/bigarray.py
View file

@ -5,11 +5,6 @@ Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org)
See the file 'LICENSE' for copying permission
"""
try:
import cPickle as pickle
except:
import pickle
import itertools
import os
import shutil
@ -86,7 +81,7 @@ class BigArray(list):
>>> _[0] = [None]
>>> _.index(0)
20
>>> import pickle; __ = pickle.loads(pickle.dumps(_))
>>> import copy; __ = copy.deepcopy(_)
>>> __.append(1)
>>> len(_)
100001
@ -158,9 +153,10 @@ class BigArray(list):
self.chunks[-1] = self.cache.data
self.cache.dirty = False
else:
from lib.core.convert import deserializeValue
try:
with open(filename, "rb") as f:
self.chunks[-1] = pickle.loads(zlib.decompress(f.read()))
self.chunks[-1] = deserializeValue(zlib.decompress(f.read()))
except IOError as ex:
errMsg = "exception occurred while retrieving data "
errMsg += "from a temporary file ('%s')" % ex
@ -207,11 +203,13 @@ class BigArray(list):
self.close()
def _dump(self, chunk):
from lib.core.convert import getBytes, serializeValue
try:
handle, filename = tempfile.mkstemp(prefix=MKSTEMP_PREFIX.BIG_ARRAY)
self.filenames.add(filename)
with os.fdopen(handle, "w+b") as f:
f.write(zlib.compress(pickle.dumps(chunk, pickle.HIGHEST_PROTOCOL), BIGARRAY_COMPRESS_LEVEL))
# serializeValue() returns text; encode to bytes for the compressed on-disk chunk
f.write(zlib.compress(getBytes(serializeValue(chunk)), BIGARRAY_COMPRESS_LEVEL))
return filename
except (OSError, IOError) as ex:
errMsg = "exception occurred while storing data "
@ -240,9 +238,10 @@ class BigArray(list):
pass
if not (self.cache and self.cache.index == index):
from lib.core.convert import deserializeValue
try:
with open(self.chunks[index], "rb") as f:
self.cache = Cache(index, pickle.loads(zlib.decompress(f.read())), False)
self.cache = Cache(index, deserializeValue(zlib.decompress(f.read())), False)
except Exception as ex:
errMsg = "exception occurred while retrieving data "
errMsg += "from a temporary file ('%s')" % ex
@ -361,8 +360,9 @@ class BigArray(list):
if cache_index == idx and cache_data is not None:
data = cache_data
else:
from lib.core.convert import deserializeValue
with open(chunk, "rb") as f:
data = pickle.loads(zlib.decompress(f.read()))
data = deserializeValue(zlib.decompress(f.read()))
except Exception as ex:
errMsg = "exception occurred while retrieving data "
errMsg += "from a temporary file ('%s')" % ex

12
lib/core/common.py
View file

@ -53,14 +53,14 @@ from lib.core.compat import LooseVersion
from lib.core.compat import RecursionError
from lib.core.compat import round
from lib.core.compat import xrange
from lib.core.convert import base64pickle
from lib.core.convert import base64unpickle
from lib.core.convert import decodeBase64
from lib.core.convert import deserializeValue
from lib.core.convert import decodeHex
from lib.core.convert import getBytes
from lib.core.convert import getText
from lib.core.convert import getUnicode
from lib.core.convert import htmlUnescape
from lib.core.convert import serializeValue
from lib.core.convert import stdoutEncode
from lib.core.data import cmdLineOptions
from lib.core.data import conf
@ -5071,7 +5071,7 @@ def serializeObject(object_):
True
"""
return base64pickle(object_)
return serializeValue(object_)
def unserializeObject(value):
"""
@ -5079,11 +5079,11 @@ def unserializeObject(value):
>>> unserializeObject(serializeObject([1, 2, 3])) == [1, 2, 3]
True
>>> unserializeObject('gAJVBmZvb2JhcnEBLg==')
'foobar'
>>> unserializeObject(serializeObject('foobar')) == 'foobar'
True
"""
return base64unpickle(value) if value else None
return deserializeValue(value) if value else None
def resetCounter(technique):
"""

240
lib/core/convert.py
View file

@ -5,14 +5,11 @@ Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org)
See the file 'LICENSE' for copying permission
"""
try:
import cPickle as pickle
except:
import pickle
import base64
import binascii
import codecs
import datetime
import decimal
import json
import re
import sys
@ -26,7 +23,6 @@ from lib.core.settings import INVALID_UNICODE_PRIVATE_AREA
from lib.core.settings import IS_TTY
from lib.core.settings import IS_WIN
from lib.core.settings import NULL
from lib.core.settings import PICKLE_PROTOCOL
from lib.core.settings import SAFE_HEX_MARKER
from lib.core.settings import UNICODE_ENCODING
from thirdparty import six
@ -41,48 +37,228 @@ except ImportError:
htmlEscape = _escape
def base64pickle(value):
"""
Serializes (with pickle) and encodes to Base64 format supplied (binary) value
# Safe (no arbitrary code execution) serialization used for the session store (HashDB)
# and BigArray disk chunks. The former serializer could execute code while loading, so
# deserializing sqlmap's own (locally writable) session/cache files was a recurring
# report magnet. This codec serializes to plain JSON with explicit type tags, so nothing
# is ever executed on load.
#
# JSON natively covers only str/int/float/bool/None/list, and silently loses the rest
# (int/tuple dict keys become strings, set/tuple/bytes are rejected). The tagged wrappers
# below preserve every type sqlmap actually stores: bytes, tuple, set/frozenset, dict with
# arbitrary (non-string) keys, DB-driver scalars (Decimal/datetime/...), and the handful of
# sqlmap's own classes below. Reconstruction of classes is limited to that explicit
# allowlist (no module/namespace wildcard), so no dangerous callable is ever reachable.
>>> base64unpickle(base64pickle([1, 2, 3])) == [1, 2, 3]
True
>>> isinstance(base64unpickle(base64pickle(BigArray([1, 2, 3]))), BigArray)
True
# reserved wrapper key; data mappings are encoded as tagged pair-lists (never as bare JSON
# objects), so any decoded JSON object is one of our wrappers and this key can never collide
_SERIALIZE_TAG = "$T"
# fully-qualified names of the ONLY classes that may be reconstructed on deserialization
_SERIALIZE_CLASSES = frozenset((
"lib.core.datatype.AttribDict",
"lib.core.datatype.InjectionDict",
"lib.utils.har.RawPair",
))
def _serializeEncode(value):
"""
Turns a Python value into a JSON-serializable (tagged) structure
"""
retVal = None
if value is None or isinstance(value, bool) or isinstance(value, float) or isinstance(value, six.integer_types):
return value
if isinstance(value, six.text_type):
return value
# Note: on Python 2 'str' is binary; base64-tagging it (rather than emitting a native JSON
# string that would round-trip as 'unicode') keeps the exact byte type across versions
if isinstance(value, (six.binary_type, bytearray)):
raw = bytes(value) if isinstance(value, bytearray) else value
return {_SERIALIZE_TAG: "b", "v": encodeBase64(raw, binary=False), "a": 1 if isinstance(value, bytearray) else 0}
if isinstance(value, memoryview):
return {_SERIALIZE_TAG: "b", "v": encodeBase64(value.tobytes(), binary=False), "a": 0}
try:
retVal = encodeBase64(pickle.dumps(value, PICKLE_PROTOCOL), binary=False)
except:
warnMsg = "problem occurred while serializing "
warnMsg += "instance of a type '%s'" % type(value)
singleTimeWarnMessage(warnMsg)
if isinstance(value, buffer): # noqa: F821 # Python 2 only
return {_SERIALIZE_TAG: "b", "v": encodeBase64(bytes(value), binary=False), "a": 0}
except NameError:
pass
try:
retVal = encodeBase64(pickle.dumps(value), binary=False)
except:
raise
# Note: BigArray is a 'list' subclass, so it must be matched before the plain-list branch
# (otherwise it would round-trip as a plain list, losing its type)
if isinstance(value, BigArray):
return {_SERIALIZE_TAG: "ba", "v": [_serializeEncode(_) for _ in value]}
if isinstance(value, list):
return [_serializeEncode(_) for _ in value]
if isinstance(value, tuple):
return {_SERIALIZE_TAG: "t", "v": [_serializeEncode(_) for _ in value]}
if isinstance(value, frozenset):
return {_SERIALIZE_TAG: "f", "v": [_serializeEncode(_) for _ in value]}
if isinstance(value, (set, _collections.Set)):
return {_SERIALIZE_TAG: "s", "v": [_serializeEncode(_) for _ in value]}
if isinstance(value, dict):
name = "%s.%s" % (value.__class__.__module__, value.__class__.__name__)
if name in _SERIALIZE_CLASSES:
return {_SERIALIZE_TAG: "o", "c": name, "d": [[_serializeEncode(k), _serializeEncode(v)] for (k, v) in value.items()], "s": _serializeEncode(dict(value.__dict__))}
elif value.__class__ is dict:
return {_SERIALIZE_TAG: "m", "v": [[_serializeEncode(k), _serializeEncode(v)] for (k, v) in value.items()]}
else:
return _serializeUnknown(value, name)
if isinstance(value, decimal.Decimal):
return {_SERIALIZE_TAG: "dec", "v": getUnicode(value)}
if isinstance(value, datetime.datetime):
return {_SERIALIZE_TAG: "dt", "v": [value.year, value.month, value.day, value.hour, value.minute, value.second, value.microsecond]}
if isinstance(value, datetime.date):
return {_SERIALIZE_TAG: "date", "v": [value.year, value.month, value.day]}
if isinstance(value, datetime.time):
return {_SERIALIZE_TAG: "time", "v": [value.hour, value.minute, value.second, value.microsecond]}
if isinstance(value, datetime.timedelta):
return {_SERIALIZE_TAG: "td", "v": [value.days, value.seconds, value.microseconds]}
name = "%s.%s" % (value.__class__.__module__, value.__class__.__name__)
if name in _SERIALIZE_CLASSES:
return {_SERIALIZE_TAG: "o", "c": name, "s": _serializeEncode(dict(value.__dict__))}
return _serializeUnknown(value, name)
def _serializeUnknown(value, name):
"""
Fallback for a type not explicitly handled by the serializer
"""
# sqlmap's own (or bundled) classes MUST be added to the allowlist explicitly - fail loudly
# (caught by the regression tests) rather than silently store something that cannot be restored
if (name or "").split(".")[0] in ("lib", "plugins", "thirdparty"):
raise TypeError("serialization of type '%s' is not supported" % name)
# a foreign/exotic scalar (e.g. an unusual DB-driver value): degrade to its textual form rather
# than crash a user's session - session values are only ever rendered (getUnicode) downstream
singleTimeWarnMessage("serializing value of unsupported type '%s' as text" % name)
return getUnicode(value)
def _serializeDecode(struct):
"""
Restores a Python value from a JSON-deserialized (tagged) structure
"""
if struct is None or isinstance(struct, bool) or isinstance(struct, float) or isinstance(struct, six.integer_types):
return struct
if isinstance(struct, six.text_type):
return struct
if isinstance(struct, six.binary_type): # defensive - json.loads() yields text, not bytes
return getUnicode(struct)
if isinstance(struct, list):
return [_serializeDecode(_) for _ in struct]
if isinstance(struct, dict):
tag = struct.get(_SERIALIZE_TAG)
if tag == "b":
raw = decodeBase64(struct["v"], binary=True)
return bytearray(raw) if struct.get("a") else raw
elif tag == "t":
return tuple(_serializeDecode(_) for _ in struct["v"])
elif tag == "f":
return frozenset(_serializeDecode(_) for _ in struct["v"])
elif tag == "ba":
return BigArray([_serializeDecode(_) for _ in struct["v"]])
elif tag == "s":
return set(_serializeDecode(_) for _ in struct["v"])
elif tag == "m":
return dict((_serializeDecode(k), _serializeDecode(v)) for (k, v) in struct["v"])
elif tag == "dec":
return decimal.Decimal(struct["v"])
elif tag == "dt":
return datetime.datetime(*struct["v"])
elif tag == "date":
return datetime.date(*struct["v"])
elif tag == "time":
return datetime.time(*struct["v"])
elif tag == "td":
return datetime.timedelta(struct["v"][0], struct["v"][1], struct["v"][2])
elif tag == "o":
return _serializeDecodeObject(struct)
elif tag is None: # defensive - a bare mapping should never occur
return dict((_serializeDecode(k), _serializeDecode(v)) for (k, v) in struct.items())
else:
raise ValueError("unsupported serialized tag '%s'" % tag)
raise ValueError("unsupported serialized structure of type '%s'" % type(struct))
def _serializeResolveClass(name):
"""
Resolves an allowlisted class name to its class (nothing else may be reconstructed)
"""
if name not in _SERIALIZE_CLASSES:
raise ValueError("deserialization of class '%s' is forbidden" % name)
if name == "lib.utils.har.RawPair":
from lib.utils.har import RawPair
return RawPair
else:
from lib.core.datatype import AttribDict, InjectionDict
return InjectionDict if name.endswith("InjectionDict") else AttribDict
def _serializeDecodeObject(struct):
"""
Reconstructs an allowlisted class instance from its serialized form
"""
_class = _serializeResolveClass(struct.get("c"))
retVal = _class.__new__(_class)
if isinstance(retVal, dict):
for pair in (struct.get("d") or []):
dict.__setitem__(retVal, _serializeDecode(pair[0]), _serializeDecode(pair[1]))
state = _serializeDecode(struct.get("s") or {})
if isinstance(state, dict):
retVal.__dict__.update(state)
return retVal
def base64unpickle(value):
def serializeValue(value):
"""
Decodes value from Base64 to plain format and deserializes (with pickle) its content
Safely serializes a Python value to its canonical serialized form (JSON text), without any
code-execution risk
>>> type(base64unpickle('gAJjX19idWlsdGluX18Kb2JqZWN0CnEBKYFxAi4=')) == object
Note: the output is pure ASCII text, so it is stored verbatim in the (TEXT) session store - no
Base64 (or any base-N) wrapping is needed (that was only required by the former binary
serialization), which also keeps the stored form as small as possible. Callers that need raw
bytes (e.g. a compressed BigArray disk chunk) simply encode the returned text.
>>> deserializeValue(serializeValue({1: 'a', 'b': (2, 3), 'c': {4, 5}})) == {1: 'a', 'b': (2, 3), 'c': {4, 5}}
True
>>> deserializeValue(serializeValue([1, 2, (3, {4: b'5'})])) == [1, 2, (3, {4: b'5'})]
True
"""
retVal = None
return json.dumps(_serializeEncode(value), ensure_ascii=True, separators=(',', ':'))
try:
retVal = pickle.loads(decodeBase64(value))
except TypeError:
retVal = pickle.loads(decodeBase64(bytes(value)))
def deserializeValue(value):
"""
Restores a Python value from its serialized form (accepts the serialized data as either text or
bytes)
"""
return retVal
return _serializeDecode(json.loads(getText(value)))
def htmlUnescape(value):
"""

46
lib/core/patch.py
View file

@ -178,52 +178,6 @@ def dirtyPatches():
et.parse = _safe_parse
et._patched = True
import io
import pickle
if not getattr(pickle, "_patched", False):
class RestrictedUnpickler(pickle.Unpickler):
# Note: allowlist (not blacklist) - a module blacklist is bypassable (e.g. importlib/ctypes/operator), so only
# explicitly-safe builtin data types and sqlmap's own (and bundled) classes are permitted to be unpickled
def find_class(self, module, name):
# Note: protocol-2 pickling of a 'bytes' value on Python 3 emits a _codecs.encode global; allow that one
# (it only runs a codec, e.g. latin1 - it cannot execute arbitrary code) so serialized values containing
# bytes round-trip. Everything else from _codecs (e.g. lookup) stays blocked by the rule below.
if module == "_codecs" and name == "encode":
pass
# safe builtin data types only (blocks eval/exec/__import__/getattr/etc.)
elif module in ("builtins", "__builtin__"):
if name not in ("set", "frozenset", "dict", "list", "tuple", "int", "float", "bool", "str", "bytes", "bytearray", "object", "NoneType", "complex"):
raise ValueError("unpickling of '%s.%s' is forbidden" % (module, name))
# everything else must be one of sqlmap's own (or bundled) classes (e.g. lib.core.datatype.AttribDict)
elif (module or "").split(".")[0] not in ("lib", "plugins", "thirdparty"):
raise ValueError("unpickling of module '%s' is forbidden" % module)
# Python 2/3 method resolution
if hasattr(pickle.Unpickler, "find_class"):
return pickle.Unpickler.find_class(self, module, name)
__import__(module)
return getattr(sys.modules[module], name)
def _safe_loads(data):
try:
stream = io.BytesIO(data)
except TypeError:
stream = io.StringIO(data)
return RestrictedUnpickler(stream).load()
pickle.loads = _safe_loads
pickle._patched = True
try:
import cPickle
if not getattr(cPickle, "_patched", False):
cPickle.loads = pickle.loads
cPickle._patched = True
except ImportError:
pass
try:
import builtins
except ImportError:

9
lib/core/settings.py
View file

@ -20,7 +20,7 @@ from lib.core.enums import OS
from thirdparty import six
# sqlmap version (<major>.<minor>.<month>.<monthly commit>)
VERSION = "1.10.7.39"
VERSION = "1.10.7.40"
TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@ -868,11 +868,8 @@ HASHDB_RETRIEVE_RETRIES = 3
# Number of retries for unsuccessful HashDB end transaction attempts
HASHDB_END_TRANSACTION_RETRIES = 3
# Unique milestone value used for forced deprecation of old HashDB values (e.g. when changing hash/pickle mechanism)
HASHDB_MILESTONE_VALUE = "GpqxbkWTfz" # python -c 'import random, string; print "".join(random.sample(string.ascii_letters, 10))'
# Pickle protocl used for storage of serialized data inside HashDB (https://docs.python.org/3/library/pickle.html#data-stream-format)
PICKLE_PROTOCOL = 2
# Unique milestone value used for forced deprecation of old HashDB values (e.g. when changing the hash/serialization mechanism)
HASHDB_MILESTONE_VALUE = "MvKpZrBqTn" # python -c 'import random, string; print "".join(random.sample(string.ascii_letters, 10))'
# Warn user of possible delay due to large page dump in full UNION query injections
LARGE_OUTPUT_THRESHOLD = 1024 ** 2

5
lib/core/unescaper.py
View file

@ -6,10 +6,11 @@ See the file 'LICENSE' for copying permission
"""
from lib.core.common import Backend
from lib.core.datatype import AttribDict
from lib.core.settings import EXCLUDE_UNESCAPE
class Unescaper(AttribDict):
# Note: a plain dict (DBMS -> escape function) with a helper method; it is a runtime registry, never
# serialized, so it deliberately does NOT use AttribDict (no attribute-style access is needed here)
class Unescaper(dict):
def escape(self, expression, quote=True, dbms=None):
if expression is None:
return expression

36
lib/utils/hashdb.py
View file

@ -165,8 +165,9 @@ class HashDB(object):
self._write_cache = {}
self._last_flush_time = time.time()
began = False
try:
self.beginTransaction()
began = self.beginTransaction()
for hash_, value in flush_cache.items():
retries = 0
while True:
@ -197,23 +198,30 @@ class HashDB(object):
else:
break
finally:
self.endTransaction()
# Only close a transaction we actually opened; when flush() runs nested inside an
# outer batch (e.g. lib/utils/hash.py wrapping cracked-password writes) beginTransaction()
# returns False and the outer owner keeps ownership - ending it here would commit it early
if began:
self.endTransaction()
def beginTransaction(self):
threadData = getCurrentThreadData()
if not threadData.inTransaction:
if threadData.inTransaction:
return False # already inside an (outer) transaction; do not nest
try:
self.cursor.execute("BEGIN TRANSACTION")
except Exception: # Note: deliberately not bare - a KeyboardInterrupt here must propagate
try:
self.cursor.execute("BEGIN TRANSACTION")
except:
try:
# Reference: http://stackoverflow.com/a/25245731
self.cursor.close()
except sqlite3.ProgrammingError:
pass
threadData.hashDBCursor = None
self.cursor.execute("BEGIN TRANSACTION")
finally:
threadData.inTransaction = True
# Reference: http://stackoverflow.com/a/25245731
self.cursor.close()
except sqlite3.ProgrammingError:
pass
threadData.hashDBCursor = None
self.cursor.execute("BEGIN TRANSACTION")
threadData.inTransaction = True # set only on a genuine BEGIN (not if the retry above raised)
return True
def endTransaction(self):
threadData = getCurrentThreadData()