Hardening Brotli/Zstandard decoding against hostile input
Some checks are pending
/ build (macos-latest, 3.8) (push) Waiting to run
/ build (ubuntu-latest, pypy-2.7) (push) Waiting to run
/ build (windows-latest, 3.14) (push) Waiting to run

This commit is contained in:
Miroslav Štampar 2026-07-25 09:59:53 +02:00
parent a4c983ae06
commit 5a5aebfc6e
4 changed files with 219 additions and 91 deletions

View file

@ -20,7 +20,7 @@ from lib.core.enums import OS
from thirdparty import six
# sqlmap version (<major>.<minor>.<month>.<monthly commit>)
VERSION = "1.10.7.186"
VERSION = "1.10.7.187"
TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)

View file

@ -325,9 +325,14 @@ def decodePage(page, contentEncoding, contentType, percentDecode=True):
elif contentEncoding == "zstd":
if _zstd is None:
raise Exception("no Zstandard decoder available")
page = _zstd.decompress(page)
# bounded streaming decode: cap output at MAX_CONNECTION_TOTAL_SIZE without allocating the
# full result first, and enforce the 8 MB window the HTTP 'zstd' coding mandates
decompressor = _zstd.ZstdDecompressor(options={_zstd.DecompressionParameter.window_log_max: 23})
page = decompressor.decompress(page, max_length=MAX_CONNECTION_TOTAL_SIZE + 1)
if len(page) > MAX_CONNECTION_TOTAL_SIZE:
raise Exception("size too large")
if not decompressor.eof:
raise Exception("incomplete Zstandard stream")
else:
data = gzip.GzipFile("", "rb", 9, io.BytesIO(page))
page = data.read(MAX_CONNECTION_TOTAL_SIZE + 1)

View file

@ -12,11 +12,24 @@ See the file 'LICENSE' for copying permission
# reference encoder across every quality/window/size. The 122 KB static dictionary + context-lookup
# table live ZIP-packed in data/txt/brotli-dictionary.tx_ (same convention as wordlist.tx_). Py 2.7 / 3.x.
import hashlib
import os
import threading
import zipfile
_DICTIONARY = None # 122784-byte static dictionary (lazy-loaded)
_CONTEXT = None # 2048-byte context-lookup table (4 modes x 2 halves x 256)
_TABLES = None # (dictionary, context) published atomically on first use
_TABLES_LOCK = threading.Lock()
# provenance: the RFC 7932 Appendix A static dictionary (122784 bytes) + the 2048-byte context-lookup
# table, extracted byte-for-byte from libbrotlicommon; verified on load so a swapped/corrupt resource
# fails loudly instead of silently mis-decoding
_TABLES_SHA256 = "20e42eb1b511c21806d4d227d07e5dd06877d8ce7b3a817f378f313653f35c70" # sha256 of the 122784-byte dictionary
_DICTIONARY_SIZE = 122784
_CONTEXT_SIZE = 2048
# per-stream ceiling on total Huffman lookup-table entries: bounds decoder memory independently of the
# output cap (a hostile stream can declare many maximal 2^15-entry trees). ~10x the worst legitimate need.
_MAX_HUFFMAN_TABLE_ENTRIES = 1 << 20
# RFC 7932 Appendix A: words are bucketed by length (4..24); size_bits gives the index width per bucket,
# offsets the cumulative byte offset of each bucket (derived from size_bits; last bucket end == 122784).
@ -180,28 +193,45 @@ class BrotliError(Exception):
def _loadTables():
global _DICTIONARY, _CONTEXT
if _DICTIONARY is not None:
return
global _TABLES
tables = _TABLES
if tables is not None: # fast path: already published (dict, context) tuple
return tables
path = None
try:
from lib.core.data import paths
path = getattr(paths, "BROTLI_DICTIONARY", None)
except ImportError:
pass
if not path or not os.path.isfile(path):
path = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, "data", "txt", "brotli-dictionary.tx_")
with _TABLES_LOCK:
if _TABLES is not None: # another thread won the race
return _TABLES
try:
path = None
try:
from lib.core.data import paths
path = getattr(paths, "BROTLI_DICTIONARY", None)
except ImportError:
pass
if not path or not os.path.isfile(path):
path = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, "data", "txt", "brotli-dictionary.tx_")
archive = zipfile.ZipFile(path) # ZIP-packed like wordlist.tx_ / catalog-identifiers.tx_
try:
raw = archive.read(archive.namelist()[0])
finally:
archive.close()
if len(raw) != 122784 + 2048:
raise BrotliError("invalid Brotli dictionary table")
_DICTIONARY = raw[:122784]
_CONTEXT = bytearray(raw[122784:])
archive = zipfile.ZipFile(path) # ZIP-packed like wordlist.tx_ / catalog-identifiers.tx_
try:
names = archive.namelist()
if len(names) != 1:
raise BrotliError("unexpected Brotli dictionary archive layout")
raw = archive.read(names[0])
finally:
archive.close()
except BrotliError:
raise
except Exception as ex:
raise BrotliError("could not load the Brotli dictionary (%s)" % ex)
if len(raw) != _DICTIONARY_SIZE + _CONTEXT_SIZE:
raise BrotliError("invalid Brotli dictionary length")
if hashlib.sha256(raw[:_DICTIONARY_SIZE]).hexdigest() != _TABLES_SHA256:
raise BrotliError("Brotli dictionary integrity check failed")
# build both, then publish the pair atomically so a concurrent reader never sees a half-set state
_TABLES = (raw[:_DICTIONARY_SIZE], bytearray(raw[_DICTIONARY_SIZE:]))
return _TABLES
class _BitReader(object):
@ -225,17 +255,23 @@ class _BitReader(object):
return 0
if self.bits < count:
self._fill()
if self.bits < count: # ran off the end of the stream -> truncated, not zero-padded
raise BrotliError("truncated Brotli stream")
value = self.acc & ((1 << count) - 1)
self.acc >>= count
self.bits -= count
return value
def peek(self, count):
# lenient lookahead (a prefix-code peek may legitimately reach past the final byte); only the
# matching drop() actually consumes, and drop() rejects consuming more than really remains
if self.bits < count:
self._fill()
return self.acc & ((1 << count) - 1)
def drop(self, count):
if self.bits < count: # the matched code needs bits the stream does not have
raise BrotliError("truncated Brotli stream")
self.acc >>= count
self.bits -= count
@ -253,10 +289,17 @@ class _BitReader(object):
self.bits -= 8
count -= 1
if count > 0:
if self.pos + count > self.size:
raise BrotliError("truncated Brotli stream")
out += self.data[self.pos:self.pos + count]
self.pos += count
return bytes(out)
def exhausted(self):
# true once no whole real bytes remain beyond the current (partial) byte - used to reject
# trailing garbage after the final meta-block
return self.pos >= self.size and self.bits < 8
def _reverseBits(value, count):
result = 0
@ -269,54 +312,66 @@ def _reverseBits(value, count):
class _Huffman(object):
__slots__ = ("maxLength", "table", "single")
def __init__(self, lengths):
def __init__(self, lengths, budget=None):
self.single = None
self.table = None
self.maxLength = max(lengths) if lengths else 0
if self.maxLength == 0:
self.single = 0
for symbol, length in enumerate(lengths):
if length > 0:
self.single = symbol
used = [(symbol, length) for symbol, length in enumerate(lengths) if length]
if not used:
raise BrotliError("empty Brotli prefix code")
if self.maxLength == 0 or len(used) == 1: # a one-symbol code is always that symbol (0 bits)
self.single = used[0][0]
self.maxLength = 0
return
if budget is not None:
budget[0] -= (1 << self.maxLength)
if budget[0] < 0:
raise BrotliError("Brotli decoder table budget exceeded")
counts = [0] * (self.maxLength + 1)
for length in lengths:
if length:
counts[length] += 1
for _, length in used:
counts[length] += 1
nextCode = [0] * (self.maxLength + 2)
code = 0
space = 0
for bits in range(1, self.maxLength + 1):
code = (code + counts[bits - 1]) << 1
nextCode[bits] = code
space += counts[bits] << (self.maxLength - bits)
if space != (1 << self.maxLength): # over- or under-subscribed prefix code (must be complete)
raise BrotliError("invalid Brotli prefix code")
self.table = [(0, 0)] * (1 << self.maxLength)
for symbol, length in enumerate(lengths):
if length:
reversed_ = _reverseBits(nextCode[length], length)
nextCode[length] += 1
step = 1 << length
for index in range(reversed_, 1 << self.maxLength, step):
self.table[index] = (symbol, length)
self.table = [None] * (1 << self.maxLength) # None = unreachable slot (rejected on decode)
for symbol, length in used:
reversed_ = _reverseBits(nextCode[length], length)
nextCode[length] += 1
step = 1 << length
for index in range(reversed_, 1 << self.maxLength, step):
self.table[index] = (symbol, length)
def decode(self, reader):
if self.table is None:
return self.single
symbol, length = self.table[reader.peek(self.maxLength)]
reader.drop(length)
return symbol
entry = self.table[reader.peek(self.maxLength)]
if entry is None: # bits matched no code -> malformed stream
raise BrotliError("invalid Brotli prefix code")
reader.drop(entry[1])
return entry[0]
def _readSimplePrefix(reader, alphabetSize):
def _readSimplePrefix(reader, alphabetSize, budget):
count = reader.readBits(2) + 1
symbolBits = (alphabetSize - 1).bit_length() or 1
symbols = [reader.readBits(symbolBits) for _ in range(count)]
lengths = [0] * alphabetSize
for symbol in symbols:
if symbol >= alphabetSize:
raise BrotliError("out-of-range symbol in Brotli simple prefix code")
if len(set(symbols)) != count:
raise BrotliError("duplicate symbol in Brotli simple prefix code")
if count == 1:
huffman = _Huffman([])
huffman.single = symbols[0]
return huffman
if count == 2:
pairs = [(symbols[0], 1)] # one symbol -> _Huffman makes it a 0-bit code
elif count == 2:
pairs = [(symbols[0], 1), (symbols[1], 1)]
elif count == 3:
pairs = [(symbols[0], 1), (symbols[1], 2), (symbols[2], 2)]
@ -324,12 +379,13 @@ def _readSimplePrefix(reader, alphabetSize):
pairs = [(symbols[0], 1), (symbols[1], 2), (symbols[2], 3), (symbols[3], 3)]
else:
pairs = [(symbols[0], 2), (symbols[1], 2), (symbols[2], 2), (symbols[3], 2)]
lengths = [0] * alphabetSize
for symbol, length in pairs:
lengths[symbol] = length
return _Huffman(lengths)
return _Huffman(lengths, budget)
def _readComplexPrefix(reader, alphabetSize, skip):
def _readComplexPrefix(reader, alphabetSize, skip, budget):
codeLengths = [0] * 18
space = 32
for symbol in _CL_ORDER[skip:]:
@ -340,7 +396,7 @@ def _readComplexPrefix(reader, alphabetSize, skip):
space -= 32 >> codeLengths[symbol]
if space <= 0:
break
codeLengthHuffman = _Huffman(codeLengths)
codeLengthHuffman = _Huffman(codeLengths, budget)
lengths = [0] * alphabetSize
symbol = 0
@ -370,20 +426,20 @@ def _readComplexPrefix(reader, alphabetSize, skip):
repeat += delta + 3
emit = repeat - old
for _ in range(emit):
if symbol >= alphabetSize:
break
if symbol >= alphabetSize: # a run past the alphabet is a malformed stream
raise BrotliError("Brotli code-length run exceeds alphabet")
lengths[symbol] = repeatLength
symbol += 1
if repeatLength:
space -= emit << (15 - repeatLength)
return _Huffman(lengths)
return _Huffman(lengths, budget)
def _readPrefix(reader, alphabetSize):
def _readPrefix(reader, alphabetSize, budget):
header = reader.readBits(2)
if header == 1:
return _readSimplePrefix(reader, alphabetSize)
return _readComplexPrefix(reader, alphabetSize, header)
return _readSimplePrefix(reader, alphabetSize, budget)
return _readComplexPrefix(reader, alphabetSize, header, budget)
def _readBlockTypeCount(reader):
@ -393,19 +449,24 @@ def _readBlockTypeCount(reader):
return (1 << bits) + 1 + reader.readBits(bits)
def _readContextMap(reader, treeCount, size):
def _readContextMap(reader, treeCount, size, budget):
maxRun = reader.readBits(4) + 1 if reader.readBits(1) else 0
huffman = _readPrefix(reader, treeCount + maxRun)
huffman = _readPrefix(reader, treeCount + maxRun, budget)
contextMap = []
while len(contextMap) < size:
code = huffman.decode(reader)
if code == 0:
contextMap.append(0)
elif code <= maxRun:
contextMap.extend([0] * ((1 << code) + reader.readBits(code)))
run = (1 << code) + reader.readBits(code)
if len(contextMap) + run > size: # a run past the declared map size is malformed
raise BrotliError("Brotli context map run overruns the map")
contextMap.extend([0] * run)
else:
contextMap.append(code - maxRun)
del contextMap[size:]
value = code - maxRun
if value >= treeCount: # references a tree that was not declared
raise BrotliError("Brotli context map references an undefined tree")
contextMap.append(value)
if reader.readBits(1): # inverse move-to-front
moveToFront = list(range(256))
for i in range(len(contextMap)):
@ -456,11 +517,8 @@ def decompress(data, maxOutput=100 * 1024 * 1024):
"""Decompress a Brotli (RFC 7932) stream, returning the original bytes. Raises BrotliError on a
malformed stream or if the output would exceed 'maxOutput' (an anti-decompression-bomb cap)."""
_loadTables()
dictionary = _DICTIONARY
context = _CONTEXT
try:
dictionary, context = _loadTables()
reader = _BitReader(data)
header = reader.readBits(1)
if header == 0:
@ -506,12 +564,14 @@ def decompress(data, maxOutput=100 * 1024 * 1024):
raise BrotliError("output too large")
continue
budget = [_MAX_HUFFMAN_TABLE_ENTRIES] # per-meta-block Huffman memory ceiling
typesL = _readBlockTypeCount(reader)
blockL, typeHuffmanL, lengthHuffmanL, prevTypeL = 1 << 28, None, None, 1
typeL = 0
if typesL >= 2:
typeHuffmanL = _readPrefix(reader, typesL + 2)
lengthHuffmanL = _readPrefix(reader, 26)
typeHuffmanL = _readPrefix(reader, typesL + 2, budget)
lengthHuffmanL = _readPrefix(reader, 26, budget)
code = lengthHuffmanL.decode(reader)
blockL = _BLEN_BASE[code] + reader.readBits(_BLEN_EXTRA[code])
@ -519,8 +579,8 @@ def decompress(data, maxOutput=100 * 1024 * 1024):
blockI, typeHuffmanI, lengthHuffmanI, prevTypeI = 1 << 28, None, None, 1
typeI = 0
if typesI >= 2:
typeHuffmanI = _readPrefix(reader, typesI + 2)
lengthHuffmanI = _readPrefix(reader, 26)
typeHuffmanI = _readPrefix(reader, typesI + 2, budget)
lengthHuffmanI = _readPrefix(reader, 26, budget)
code = lengthHuffmanI.decode(reader)
blockI = _BLEN_BASE[code] + reader.readBits(_BLEN_EXTRA[code])
@ -528,8 +588,8 @@ def decompress(data, maxOutput=100 * 1024 * 1024):
blockD, typeHuffmanD, lengthHuffmanD, prevTypeD = 1 << 28, None, None, 1
typeD = 0
if typesD >= 2:
typeHuffmanD = _readPrefix(reader, typesD + 2)
lengthHuffmanD = _readPrefix(reader, 26)
typeHuffmanD = _readPrefix(reader, typesD + 2, budget)
lengthHuffmanD = _readPrefix(reader, 26, budget)
code = lengthHuffmanD.decode(reader)
blockD = _BLEN_BASE[code] + reader.readBits(_BLEN_EXTRA[code])
@ -538,14 +598,14 @@ def decompress(data, maxOutput=100 * 1024 * 1024):
contextModes = [reader.readBits(2) for _ in range(typesL)]
treesL = _readBlockTypeCount(reader)
contextMapL = _readContextMap(reader, treesL, typesL * 64) if treesL >= 2 else [0] * (typesL * 64)
contextMapL = _readContextMap(reader, treesL, typesL * 64, budget) if treesL >= 2 else [0] * (typesL * 64)
treesD = _readBlockTypeCount(reader)
contextMapD = _readContextMap(reader, treesD, typesD * 4) if treesD >= 2 else [0] * (typesD * 4)
contextMapD = _readContextMap(reader, treesD, typesD * 4, budget) if treesD >= 2 else [0] * (typesD * 4)
huffmanL = [_readPrefix(reader, 256) for _ in range(treesL)]
huffmanI = [_readPrefix(reader, 704) for _ in range(typesI)]
huffmanL = [_readPrefix(reader, 256, budget) for _ in range(treesL)]
huffmanI = [_readPrefix(reader, 704, budget) for _ in range(typesI)]
distanceAlphabet = 16 + direct + (48 << postfix)
huffmanD = [_readPrefix(reader, distanceAlphabet) for _ in range(treesD)]
huffmanD = [_readPrefix(reader, distanceAlphabet, budget) for _ in range(treesD)]
produced = 0
while produced < metaLength:
@ -614,6 +674,9 @@ def decompress(data, maxOutput=100 * 1024 * 1024):
low = value & ((1 << postfix) - 1)
distance = ((((2 + (high & 1)) << extraBits) - 4 + extra) << postfix) + low + direct + 1
if distance <= 0: # a ring/short-code computation must yield >= 1
raise BrotliError("invalid Brotli distance")
maxDistance = min(len(out), maxBackward)
if distanceCode != 0 and distance <= maxDistance:
distRing[distIndex & 3] = distance
@ -637,6 +700,8 @@ def decompress(data, maxOutput=100 * 1024 * 1024):
raise BrotliError("invalid dictionary transform")
start = _OFFSETS[copyLength] + index * copyLength
word = _applyTransform(transformId, dictionary[start:start + copyLength])
if produced + len(word) > metaLength: # a transformed word must still fit the block
raise BrotliError("dictionary word exceeds meta-block length")
out += word
produced += len(word)
@ -645,6 +710,13 @@ def decompress(data, maxOutput=100 * 1024 * 1024):
if isLast:
break
# after the final meta-block only zero byte-alignment padding may remain: no whole leftover bytes
# (trailing garbage) and the padding bits themselves must be zero (RFC 7932)
if reader.bits + (reader.size - reader.pos) * 8 >= 8:
raise BrotliError("trailing data after Brotli stream")
if reader.acc != 0:
raise BrotliError("non-zero Brotli padding bits")
return bytes(out)
except BrotliError:
raise

View file

@ -4,12 +4,14 @@
Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org)
See the file 'LICENSE' for copying permission
Tests for the dependency-free Brotli (RFC 7932) decompressor under lib/utils/brotli.py. The compressed
fixtures were produced by the reference encoder at various quality levels; the expected plaintext is
reconstructed here by construction, so the suite validates the decoder fully offline (no third-party
'brotli' module at test time) on Python 2.7 / 3.x. Cases deliberately exercise the static dictionary +
word transforms, long overlapping copies, UTF-8, the low-quality (near-uniform tree) path and the
repetitive content that relies on the higher insert-and-copy command ranges.
Tests for the dependency-free Brotli (RFC 7932) decompressor under lib/utils/brotli.py, and its wiring
into lib/request/basic.py::decodePage. The compressed fixtures were produced by the reference encoder at
various quality levels; the expected plaintext is reconstructed here by construction, so the suite
validates the decoder fully offline (no third-party 'brotli' module at test time) on Python 2.7 / 3.x.
Positive cases exercise the static dictionary + word transforms, long overlapping copies, UTF-8, the
low-quality (near-uniform tree) path and the repetitive content that relies on the higher insert-and-copy
command ranges. Negative cases assert that hostile input (truncation, garbage, output bombs) is rejected
with a BrotliError rather than silently producing corrupted output.
"""
import binascii
@ -35,12 +37,15 @@ _CASES = [
(b"the quick brown fox " * 30,
"1b570200047463a92ee78362f22082d628041695d90acef9a3f135e9c701"),
(b"AB" * 400, "1b1f0300a48284a2b230b009"),
((u"caf\xe9 na\xefve \u4f60\u597d ").encode("utf-8") * 12,
(u"caf\xe9 na\xefve \u4f60\u597d ".encode("utf-8") * 12,
"1bef00004427477ad6d60ac38c93200a288ab462c2a06461d22d186dbbe0263e0707"),
(b"hello hello hello world world foo bar baz " * 6,
"8b7d000080aaaaaaeaff74e5f355048415f8c0000c201701d0ffbbeadf736f75cfa82e6f63b82b5e2c2c2c6c6cacea654675f0e1c38fc160308e33595583c16030180ce65067442a4aa370586827d97b828968074727f5b21e97eebd045d8baeefef94c3fca4fb1e"),
]
# a valid stream (~1.2 KB of text) whose truncations feed the negative corpus
_TRUNCATION_SAMPLE = binascii.unhexlify("1b570200047463a92ee78362f22082d628041695d90acef9a3f135e9c701")
class TestBrotli(unittest.TestCase):
def test_known_fixtures(self):
@ -50,20 +55,66 @@ class TestBrotli(unittest.TestCase):
def test_empty_stream(self):
self.assertEqual(decompress(binascii.unhexlify("3b")), b"")
def test_malformed_raises(self):
# a hostile/truncated stream must surface as BrotliError, never a raw exception
for blob in (b"\xff", b"\x00\x00\x00", b"\x1b\xff\xff\xff\xff", os.urandom(32)):
def test_truncation_is_rejected(self):
# every proper prefix of a valid stream is truncated -> must raise, never silently return
# corrupted or zero-padded output
for cut in range(1, len(_TRUNCATION_SAMPLE)):
self.assertRaises(BrotliError, decompress, _TRUNCATION_SAMPLE[:cut])
def test_malformed_is_rejected(self):
for blob in (b"\xff", b"\x00\x00\x00", b"\x1b\xff\xff\xff\xff",
_TRUNCATION_SAMPLE + b"\x00\x00\x00\x00", # trailing garbage
binascii.unhexlify("3b") + b"\xde\xad"): # data after a complete empty stream
self.assertRaises(BrotliError, decompress, blob)
def test_non_brotlierror_never_escapes(self):
# arbitrary bytes must terminate quickly and only ever raise BrotliError (never a raw exception)
for i in range(400):
blob = bytes(bytearray((i * 37 + j * 13) & 0xff for j in range(i % 60)))
try:
decompress(blob)
except BrotliError:
pass
except Exception as ex:
self.fail("non-BrotliError on malformed input: %s" % ex)
def test_bomb_cap(self):
# a small stream must not be allowed to expand past the output cap
self.assertRaises(BrotliError, decompress, binascii.unhexlify("1b1f0300a48284a2b230b009"), 16)
class TestBrotliDecodePage(unittest.TestCase):
_KB = ("pageCompress", "pageEncoding", "disableHtmlDecoding", "singleLogFlags")
_CONF = ("encoding", "nullConnection")
def setUp(self):
from lib.core.data import conf, kb
self._kb = dict((name, kb.get(name)) for name in self._KB)
self._conf = dict((name, conf.get(name)) for name in self._CONF)
def tearDown(self):
from lib.core.data import conf, kb
for name, value in self._kb.items():
kb[name] = value
for name, value in self._conf.items():
conf[name] = value
def test_decodepage_br(self):
from lib.core.data import conf, kb
from lib.request.basic import decodePage
from lib.core.convert import getBytes
conf.encoding = None
conf.nullConnection = False
kb.pageCompress = True
kb.pageEncoding = None
kb.singleLogFlags = set()
kb.disableHtmlDecoding = True
body = b"<html><body>secret uid=admin</body></html>" * 25
# brotli-compressed 'body' (reference encoder, quality 11)
compressed = binascii.unhexlify(
"1b190488c56d6c1ff52d8742bd820d3870892cd08016f661030e310d82f520b7a3513810bf66edf05d3500")
self.assertEqual(getBytes(decodePage(compressed, "br", "text/html; charset=utf-8")), body)
if __name__ == "__main__":
unittest.main()