diff --git a/extra/dbwire/clickhouse.py b/extra/dbwire/clickhouse.py index a172c25d0..b6a9cae58 100644 --- a/extra/dbwire/clickhouse.py +++ b/extra/dbwire/clickhouse.py @@ -25,24 +25,34 @@ except ImportError: from extra.dbwire import OperationalError from extra.dbwire import ProgrammingError +# TabSeparated backslash escapes -> the literal byte they denote +_ESCAPE = {ord("t"): 9, ord("n"): 10, ord("r"): 13, ord("0"): 0, ord("b"): 8, + ord("f"): 12, ord("a"): 7, ord("v"): 11, ord("\\"): 92, ord("'"): 39} + def _unescape(value): - if value == "\\N": + # value: the raw bytes of one TSV field -> None (\N) or the unescaped bytes. Operates on bytes because a + # String/FixedString column can hold arbitrary non-UTF-8 data, which a whole-body utf-8 decode would destroy. + if value == b"\\N": return None - if "\\" not in value: + if b"\\" not in value: return value - out = [] - i = 0 - n = len(value) + src, out, i, n = bytearray(value), bytearray(), 0, len(value) while i < n: - ch = value[i] - if ch == "\\" and i + 1 < n: - nxt = value[i + 1] - out.append({"t": "\t", "n": "\n", "r": "\r", "0": "\0", "b": "\b", "f": "\f", "a": "\a", "v": "\v", "\\": "\\", "'": "'"}.get(nxt, nxt)) - i += 2 + c = src[i] + if c == 0x5c and i + 1 < n: # backslash + out.append(_ESCAPE.get(src[i + 1], src[i + 1])); i += 2 else: - out.append(ch) - i += 1 - return "".join(out) + out.append(c); i += 1 + return bytes(out) + +def _decode_cell(value): + # keep text as str; hand back raw bytes only when a value is not valid UTF-8 (sqlmap then hex-encodes it) + if value is None: + return None + try: + return value.decode("utf-8") + except UnicodeDecodeError: + return value class Cursor(object): def __init__(self, connection): @@ -99,7 +109,7 @@ class Connection(object): def _query(self, query): req = Request(self._url, data=query.encode("utf-8"), headers=self._headers) try: - body = urlopen(req, timeout=self._timeout).read().decode("utf-8", "replace") + body = urlopen(req, timeout=self._timeout).read() # bytes: column data may be non-UTF-8 except HTTPError as ex: raise ProgrammingError("(remote) %s" % ex.read().decode("utf-8", "replace").strip()) except URLError as ex: @@ -109,13 +119,13 @@ class Connection(object): if not body: return None, [] - lines = body.split("\n") - if lines and lines[-1] == "": + lines = body.split(b"\n") + if lines and lines[-1] == b"": lines.pop() if not lines: return None, [] - description = [(name, None, None, None, None, None, None) for name in (_unescape(_) for _ in lines[0].split("\t"))] - rows = [tuple(_unescape(_) for _ in line.split("\t")) for line in lines[1:]] + description = [(name, None, None, None, None, None, None) for name in (_decode_cell(_unescape(_)) for _ in lines[0].split(b"\t"))] + rows = [tuple(_decode_cell(_unescape(_)) for _ in line.split(b"\t")) for line in lines[1:]] return description, rows def connect(host=None, port=8123, user=None, password=None, database=None, connect_timeout=None, **kwargs): diff --git a/extra/dbwire/mysql.py b/extra/dbwire/mysql.py index ba497b7d6..4f9b67262 100644 --- a/extra/dbwire/mysql.py +++ b/extra/dbwire/mysql.py @@ -35,7 +35,11 @@ _CLIENT_PLUGIN_AUTH = 0x00080000 _MAX_PACKET = 0x1000000 _MAX_MESSAGE_LENGTH = 0x40000000 # cap on a (re-assembled) payload, to bound a hostile/corrupt stream -_BINARY_CHARSET = 63 # collation id 63 == 'binary' (BLOB/BINARY/VARBINARY columns) +_BINARY_CHARSET = 63 # collation id 63 == 'binary' +# field types for which charset==63 genuinely denotes raw bytes (BLOB/BINARY/VARBINARY/BIT/GEOMETRY family). +# Numeric & temporal columns ALSO report charset 63 in the text protocol, but carry their ASCII text form - +# they must be decoded, not returned as bytes (else -d hexifies e.g. the int 12345 to '3132333435'). +_BINARY_TYPES = frozenset((15, 16, 249, 250, 251, 252, 253, 254, 255)) # VARCHAR,BIT,*BLOB,VAR_STRING,STRING,GEOMETRY def _xor(a, b): if str is bytes: # Python 2 @@ -210,8 +214,9 @@ class Connection(object): _, off = _lenc_str(cpay, off) # org_name _, off = _lenc_int(cpay, off) # length of the fixed-length block (0x0c) charset = struct.unpack("= 0 else ("-", -offset) + s += " %s%02d:%02d" % (sign, mins // 60, mins % 60) + return s + +# SQL Server COLLATION -> Python codec. The 5-byte collation is a little-endian uint32 (low 20 bits = LCID) +# plus a 1-byte sort id: a non-zero sort id fixes the code page, else the LCID does. Only single-byte / DBCS +# code pages need a codec (NVARCHAR is UTF-16, handled separately). Derived from pytds; default cp1252 (the +# stock SQL_Latin1_General code page - NOT latin-1, whose 0x80-0x9F differ, corrupting e.g. the euro sign). +_LCID_CP = { + 0x405: "cp1250", 0x40e: "cp1250", 0x415: "cp1250", 0x418: "cp1250", 0x41a: "cp1250", 0x41b: "cp1250", + 0x41c: "cp1250", 0x424: "cp1250", 0x402: "cp1251", 0x419: "cp1251", 0x422: "cp1251", 0x423: "cp1251", + 0x42f: "cp1251", 0x408: "cp1253", 0x41f: "cp1254", 0x42c: "cp1254", 0x443: "cp1254", 0x40d: "cp1255", + 0x401: "cp1256", 0x420: "cp1256", 0x429: "cp1256", 0x425: "cp1257", 0x426: "cp1257", 0x427: "cp1257", + 0x42a: "cp1258", 0x41e: "cp874", 0x411: "cp932", 0x804: "cp936", 0x1004: "cp936", 0x412: "cp949", + 0x404: "cp950", 0xc04: "cp950", 0x1404: "cp950", +} + +def _sortid_cp(sid): + if 30 <= sid <= 34: + return "cp437" + if 40 <= sid <= 44 or sid == 49 or 55 <= sid <= 61: + return "cp850" + if sid in (51, 52, 53, 54) or 183 <= sid <= 186: + return "cp1252" + if 80 <= sid <= 96: + return "cp1250" + if 104 <= sid <= 108: + return "cp1251" + if 112 <= sid <= 124: + return "cp1253" + if 128 <= sid <= 130: + return "cp1254" + if 136 <= sid <= 138: + return "cp1255" + if 144 <= sid <= 146: + return "cp1256" + if 152 <= sid <= 160: + return "cp1257" + return None + +def _collation_codec(collation): + if not collation or len(collation) < 5: + return "cp1252" + lump = struct.unpack(" raw bytes + if base in (0xa7, 0xaf): # (var)char: metadata = 5-byte collation + 2-byte max length + return val.decode(_collation_codec(meta[:5]), "replace") + if base == 0x28: + return _decode_temporal(base, 0, val) + if base in (0x29, 0x2a, 0x2b): # metadata = scale + return _decode_temporal(base, bytearray(meta)[0], val) + return "".join("%02x" % x for x in bytearray(val)) # unknown base type -> hex (never desyncs) + class _Column(object): - __slots__ = ("name", "type", "size", "scale", "binary") + __slots__ = ("name", "type", "size", "scale", "binary", "collation") def _parse_type_info(data, off): col = _Column() col.type = _u8(data, off); off += 1 - col.size, col.scale, col.binary = 0, 0, False + col.size, col.scale, col.binary, col.collation = 0, 0, False, None t = col.type if t in (0x30, 0x32, 0x34, 0x38, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x7a, 0x7f, 0x1f): pass # fixed-length types, size implied by type - elif t in (0x26, 0x68, 0x6d, 0x6e, 0x6f, 0x24, 0x2e, 0x37): # INTN/BITN/FLTN/MONEYN/DATETIMN/GUID + elif t in (0x26, 0x68, 0x6d, 0x6e, 0x6f, 0x24): # INTN/BITN/FLTN/MONEYN/DATETIMN/GUID col.size = _u8(data, off); off += 1 - elif t in (0x6a, 0x6c): # DECIMALN / NUMERICN + elif t in (0x6a, 0x6c, 0x37, 0x3f): # DECIMALN/NUMERICN + legacy DECIMAL/NUMERIC (size, precision, scale) col.size = _u8(data, off); off += 1 off += 1 # precision col.scale = _u8(data, off); off += 1 elif t in (0xa7, 0xaf, 0xe7, 0xef): # (BIG)VARCHAR/CHAR, N(VAR)CHAR col.size = struct.unpack(" raw bytes if t in (0xe7, 0xef): return raw.decode("utf-16-le", "replace"), off - return raw.decode("latin-1"), off # (var)char is the collation's single-byte codepage; latin-1 round-trips every byte losslessly + return raw.decode(_collation_codec(col.collation), "replace"), off # (var)char: the collation's code page if t in (0x23, 0x63, 0x22): # TEXT/NTEXT/IMAGE: 1-byte textptr len (0 = NULL) then textptr+timestamp then 4-byte len ptr_len = _u8(data, off); off += 1 @@ -258,7 +414,21 @@ def _decode_value(col, data, off): return raw, off if t == 0x63: return raw.decode("utf-16-le", "replace"), off - return raw.decode("latin-1"), off # TEXT is single-byte codepage; latin-1 is lossless + return raw.decode(_collation_codec(col.collation), "replace"), off # TEXT: the collation's code page + + if t == 0xf1: # XML: PLP-encoded UTF-16-LE + raw, off = _read_plp(data, off) + return (raw.decode("utf-16-le", "replace") if raw is not None else None), off + + if t == 0xf0: # UDT (geometry/geography/hierarchyid): PLP raw bytes -> sqlmap hex-encodes them + return _read_plp(data, off) + + if t == 0x62: # SQL_VARIANT: 4-byte total length (0 = NULL) then a self-describing value body + (total,) = struct.unpack("= 0 else -value) - s = ("-" if value < 0 else "") + s[:-col.scale] + "." + s[-col.scale:] - return s, off - return str(value), off + return _decode_money(raw), off + if t in (0x6a, 0x6c, 0x37, 0x3f): # DECIMALN / NUMERICN (+ legacy DECIMAL / NUMERIC) + return _decode_numeric(raw, col.scale), off if t == 0x24: # GUID - a, b, c = struct.unpack("...) -VERSION = "1.10.7.86" +VERSION = "1.10.7.87" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)