mirror of
https://github.com/sqlmapproject/sqlmap.git
synced 2026-08-04 06:50:14 +00:00
Hardening Brotli/Zstandard decoding against hostile input
This commit is contained in:
parent
a4c983ae06
commit
5a5aebfc6e
4 changed files with 219 additions and 91 deletions
|
|
@ -20,7 +20,7 @@ from lib.core.enums import OS
|
|||
from thirdparty import six
|
||||
|
||||
# sqlmap version (<major>.<minor>.<month>.<monthly commit>)
|
||||
VERSION = "1.10.7.186"
|
||||
VERSION = "1.10.7.187"
|
||||
TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
|
||||
TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
|
||||
VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
|
||||
|
|
|
|||
|
|
@ -325,9 +325,14 @@ def decodePage(page, contentEncoding, contentType, percentDecode=True):
|
|||
elif contentEncoding == "zstd":
|
||||
if _zstd is None:
|
||||
raise Exception("no Zstandard decoder available")
|
||||
page = _zstd.decompress(page)
|
||||
# bounded streaming decode: cap output at MAX_CONNECTION_TOTAL_SIZE without allocating the
|
||||
# full result first, and enforce the 8 MB window the HTTP 'zstd' coding mandates
|
||||
decompressor = _zstd.ZstdDecompressor(options={_zstd.DecompressionParameter.window_log_max: 23})
|
||||
page = decompressor.decompress(page, max_length=MAX_CONNECTION_TOTAL_SIZE + 1)
|
||||
if len(page) > MAX_CONNECTION_TOTAL_SIZE:
|
||||
raise Exception("size too large")
|
||||
if not decompressor.eof:
|
||||
raise Exception("incomplete Zstandard stream")
|
||||
else:
|
||||
data = gzip.GzipFile("", "rb", 9, io.BytesIO(page))
|
||||
page = data.read(MAX_CONNECTION_TOTAL_SIZE + 1)
|
||||
|
|
|
|||
|
|
@ -12,11 +12,24 @@ See the file 'LICENSE' for copying permission
|
|||
# reference encoder across every quality/window/size. The 122 KB static dictionary + context-lookup
|
||||
# table live ZIP-packed in data/txt/brotli-dictionary.tx_ (same convention as wordlist.tx_). Py 2.7 / 3.x.
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import threading
|
||||
import zipfile
|
||||
|
||||
_DICTIONARY = None # 122784-byte static dictionary (lazy-loaded)
|
||||
_CONTEXT = None # 2048-byte context-lookup table (4 modes x 2 halves x 256)
|
||||
_TABLES = None # (dictionary, context) published atomically on first use
|
||||
_TABLES_LOCK = threading.Lock()
|
||||
|
||||
# provenance: the RFC 7932 Appendix A static dictionary (122784 bytes) + the 2048-byte context-lookup
|
||||
# table, extracted byte-for-byte from libbrotlicommon; verified on load so a swapped/corrupt resource
|
||||
# fails loudly instead of silently mis-decoding
|
||||
_TABLES_SHA256 = "20e42eb1b511c21806d4d227d07e5dd06877d8ce7b3a817f378f313653f35c70" # sha256 of the 122784-byte dictionary
|
||||
_DICTIONARY_SIZE = 122784
|
||||
_CONTEXT_SIZE = 2048
|
||||
|
||||
# per-stream ceiling on total Huffman lookup-table entries: bounds decoder memory independently of the
|
||||
# output cap (a hostile stream can declare many maximal 2^15-entry trees). ~10x the worst legitimate need.
|
||||
_MAX_HUFFMAN_TABLE_ENTRIES = 1 << 20
|
||||
|
||||
# RFC 7932 Appendix A: words are bucketed by length (4..24); size_bits gives the index width per bucket,
|
||||
# offsets the cumulative byte offset of each bucket (derived from size_bits; last bucket end == 122784).
|
||||
|
|
@ -180,28 +193,45 @@ class BrotliError(Exception):
|
|||
|
||||
|
||||
def _loadTables():
|
||||
global _DICTIONARY, _CONTEXT
|
||||
if _DICTIONARY is not None:
|
||||
return
|
||||
global _TABLES
|
||||
tables = _TABLES
|
||||
if tables is not None: # fast path: already published (dict, context) tuple
|
||||
return tables
|
||||
|
||||
path = None
|
||||
try:
|
||||
from lib.core.data import paths
|
||||
path = getattr(paths, "BROTLI_DICTIONARY", None)
|
||||
except ImportError:
|
||||
pass
|
||||
if not path or not os.path.isfile(path):
|
||||
path = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, "data", "txt", "brotli-dictionary.tx_")
|
||||
with _TABLES_LOCK:
|
||||
if _TABLES is not None: # another thread won the race
|
||||
return _TABLES
|
||||
try:
|
||||
path = None
|
||||
try:
|
||||
from lib.core.data import paths
|
||||
path = getattr(paths, "BROTLI_DICTIONARY", None)
|
||||
except ImportError:
|
||||
pass
|
||||
if not path or not os.path.isfile(path):
|
||||
path = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, "data", "txt", "brotli-dictionary.tx_")
|
||||
|
||||
archive = zipfile.ZipFile(path) # ZIP-packed like wordlist.tx_ / catalog-identifiers.tx_
|
||||
try:
|
||||
raw = archive.read(archive.namelist()[0])
|
||||
finally:
|
||||
archive.close()
|
||||
if len(raw) != 122784 + 2048:
|
||||
raise BrotliError("invalid Brotli dictionary table")
|
||||
_DICTIONARY = raw[:122784]
|
||||
_CONTEXT = bytearray(raw[122784:])
|
||||
archive = zipfile.ZipFile(path) # ZIP-packed like wordlist.tx_ / catalog-identifiers.tx_
|
||||
try:
|
||||
names = archive.namelist()
|
||||
if len(names) != 1:
|
||||
raise BrotliError("unexpected Brotli dictionary archive layout")
|
||||
raw = archive.read(names[0])
|
||||
finally:
|
||||
archive.close()
|
||||
except BrotliError:
|
||||
raise
|
||||
except Exception as ex:
|
||||
raise BrotliError("could not load the Brotli dictionary (%s)" % ex)
|
||||
|
||||
if len(raw) != _DICTIONARY_SIZE + _CONTEXT_SIZE:
|
||||
raise BrotliError("invalid Brotli dictionary length")
|
||||
if hashlib.sha256(raw[:_DICTIONARY_SIZE]).hexdigest() != _TABLES_SHA256:
|
||||
raise BrotliError("Brotli dictionary integrity check failed")
|
||||
|
||||
# build both, then publish the pair atomically so a concurrent reader never sees a half-set state
|
||||
_TABLES = (raw[:_DICTIONARY_SIZE], bytearray(raw[_DICTIONARY_SIZE:]))
|
||||
return _TABLES
|
||||
|
||||
|
||||
class _BitReader(object):
|
||||
|
|
@ -225,17 +255,23 @@ class _BitReader(object):
|
|||
return 0
|
||||
if self.bits < count:
|
||||
self._fill()
|
||||
if self.bits < count: # ran off the end of the stream -> truncated, not zero-padded
|
||||
raise BrotliError("truncated Brotli stream")
|
||||
value = self.acc & ((1 << count) - 1)
|
||||
self.acc >>= count
|
||||
self.bits -= count
|
||||
return value
|
||||
|
||||
def peek(self, count):
|
||||
# lenient lookahead (a prefix-code peek may legitimately reach past the final byte); only the
|
||||
# matching drop() actually consumes, and drop() rejects consuming more than really remains
|
||||
if self.bits < count:
|
||||
self._fill()
|
||||
return self.acc & ((1 << count) - 1)
|
||||
|
||||
def drop(self, count):
|
||||
if self.bits < count: # the matched code needs bits the stream does not have
|
||||
raise BrotliError("truncated Brotli stream")
|
||||
self.acc >>= count
|
||||
self.bits -= count
|
||||
|
||||
|
|
@ -253,10 +289,17 @@ class _BitReader(object):
|
|||
self.bits -= 8
|
||||
count -= 1
|
||||
if count > 0:
|
||||
if self.pos + count > self.size:
|
||||
raise BrotliError("truncated Brotli stream")
|
||||
out += self.data[self.pos:self.pos + count]
|
||||
self.pos += count
|
||||
return bytes(out)
|
||||
|
||||
def exhausted(self):
|
||||
# true once no whole real bytes remain beyond the current (partial) byte - used to reject
|
||||
# trailing garbage after the final meta-block
|
||||
return self.pos >= self.size and self.bits < 8
|
||||
|
||||
|
||||
def _reverseBits(value, count):
|
||||
result = 0
|
||||
|
|
@ -269,54 +312,66 @@ def _reverseBits(value, count):
|
|||
class _Huffman(object):
|
||||
__slots__ = ("maxLength", "table", "single")
|
||||
|
||||
def __init__(self, lengths):
|
||||
def __init__(self, lengths, budget=None):
|
||||
self.single = None
|
||||
self.table = None
|
||||
self.maxLength = max(lengths) if lengths else 0
|
||||
if self.maxLength == 0:
|
||||
self.single = 0
|
||||
for symbol, length in enumerate(lengths):
|
||||
if length > 0:
|
||||
self.single = symbol
|
||||
used = [(symbol, length) for symbol, length in enumerate(lengths) if length]
|
||||
if not used:
|
||||
raise BrotliError("empty Brotli prefix code")
|
||||
if self.maxLength == 0 or len(used) == 1: # a one-symbol code is always that symbol (0 bits)
|
||||
self.single = used[0][0]
|
||||
self.maxLength = 0
|
||||
return
|
||||
|
||||
if budget is not None:
|
||||
budget[0] -= (1 << self.maxLength)
|
||||
if budget[0] < 0:
|
||||
raise BrotliError("Brotli decoder table budget exceeded")
|
||||
|
||||
counts = [0] * (self.maxLength + 1)
|
||||
for length in lengths:
|
||||
if length:
|
||||
counts[length] += 1
|
||||
for _, length in used:
|
||||
counts[length] += 1
|
||||
nextCode = [0] * (self.maxLength + 2)
|
||||
code = 0
|
||||
space = 0
|
||||
for bits in range(1, self.maxLength + 1):
|
||||
code = (code + counts[bits - 1]) << 1
|
||||
nextCode[bits] = code
|
||||
space += counts[bits] << (self.maxLength - bits)
|
||||
if space != (1 << self.maxLength): # over- or under-subscribed prefix code (must be complete)
|
||||
raise BrotliError("invalid Brotli prefix code")
|
||||
|
||||
self.table = [(0, 0)] * (1 << self.maxLength)
|
||||
for symbol, length in enumerate(lengths):
|
||||
if length:
|
||||
reversed_ = _reverseBits(nextCode[length], length)
|
||||
nextCode[length] += 1
|
||||
step = 1 << length
|
||||
for index in range(reversed_, 1 << self.maxLength, step):
|
||||
self.table[index] = (symbol, length)
|
||||
self.table = [None] * (1 << self.maxLength) # None = unreachable slot (rejected on decode)
|
||||
for symbol, length in used:
|
||||
reversed_ = _reverseBits(nextCode[length], length)
|
||||
nextCode[length] += 1
|
||||
step = 1 << length
|
||||
for index in range(reversed_, 1 << self.maxLength, step):
|
||||
self.table[index] = (symbol, length)
|
||||
|
||||
def decode(self, reader):
|
||||
if self.table is None:
|
||||
return self.single
|
||||
symbol, length = self.table[reader.peek(self.maxLength)]
|
||||
reader.drop(length)
|
||||
return symbol
|
||||
entry = self.table[reader.peek(self.maxLength)]
|
||||
if entry is None: # bits matched no code -> malformed stream
|
||||
raise BrotliError("invalid Brotli prefix code")
|
||||
reader.drop(entry[1])
|
||||
return entry[0]
|
||||
|
||||
|
||||
def _readSimplePrefix(reader, alphabetSize):
|
||||
def _readSimplePrefix(reader, alphabetSize, budget):
|
||||
count = reader.readBits(2) + 1
|
||||
symbolBits = (alphabetSize - 1).bit_length() or 1
|
||||
symbols = [reader.readBits(symbolBits) for _ in range(count)]
|
||||
lengths = [0] * alphabetSize
|
||||
for symbol in symbols:
|
||||
if symbol >= alphabetSize:
|
||||
raise BrotliError("out-of-range symbol in Brotli simple prefix code")
|
||||
if len(set(symbols)) != count:
|
||||
raise BrotliError("duplicate symbol in Brotli simple prefix code")
|
||||
if count == 1:
|
||||
huffman = _Huffman([])
|
||||
huffman.single = symbols[0]
|
||||
return huffman
|
||||
if count == 2:
|
||||
pairs = [(symbols[0], 1)] # one symbol -> _Huffman makes it a 0-bit code
|
||||
elif count == 2:
|
||||
pairs = [(symbols[0], 1), (symbols[1], 1)]
|
||||
elif count == 3:
|
||||
pairs = [(symbols[0], 1), (symbols[1], 2), (symbols[2], 2)]
|
||||
|
|
@ -324,12 +379,13 @@ def _readSimplePrefix(reader, alphabetSize):
|
|||
pairs = [(symbols[0], 1), (symbols[1], 2), (symbols[2], 3), (symbols[3], 3)]
|
||||
else:
|
||||
pairs = [(symbols[0], 2), (symbols[1], 2), (symbols[2], 2), (symbols[3], 2)]
|
||||
lengths = [0] * alphabetSize
|
||||
for symbol, length in pairs:
|
||||
lengths[symbol] = length
|
||||
return _Huffman(lengths)
|
||||
return _Huffman(lengths, budget)
|
||||
|
||||
|
||||
def _readComplexPrefix(reader, alphabetSize, skip):
|
||||
def _readComplexPrefix(reader, alphabetSize, skip, budget):
|
||||
codeLengths = [0] * 18
|
||||
space = 32
|
||||
for symbol in _CL_ORDER[skip:]:
|
||||
|
|
@ -340,7 +396,7 @@ def _readComplexPrefix(reader, alphabetSize, skip):
|
|||
space -= 32 >> codeLengths[symbol]
|
||||
if space <= 0:
|
||||
break
|
||||
codeLengthHuffman = _Huffman(codeLengths)
|
||||
codeLengthHuffman = _Huffman(codeLengths, budget)
|
||||
|
||||
lengths = [0] * alphabetSize
|
||||
symbol = 0
|
||||
|
|
@ -370,20 +426,20 @@ def _readComplexPrefix(reader, alphabetSize, skip):
|
|||
repeat += delta + 3
|
||||
emit = repeat - old
|
||||
for _ in range(emit):
|
||||
if symbol >= alphabetSize:
|
||||
break
|
||||
if symbol >= alphabetSize: # a run past the alphabet is a malformed stream
|
||||
raise BrotliError("Brotli code-length run exceeds alphabet")
|
||||
lengths[symbol] = repeatLength
|
||||
symbol += 1
|
||||
if repeatLength:
|
||||
space -= emit << (15 - repeatLength)
|
||||
return _Huffman(lengths)
|
||||
return _Huffman(lengths, budget)
|
||||
|
||||
|
||||
def _readPrefix(reader, alphabetSize):
|
||||
def _readPrefix(reader, alphabetSize, budget):
|
||||
header = reader.readBits(2)
|
||||
if header == 1:
|
||||
return _readSimplePrefix(reader, alphabetSize)
|
||||
return _readComplexPrefix(reader, alphabetSize, header)
|
||||
return _readSimplePrefix(reader, alphabetSize, budget)
|
||||
return _readComplexPrefix(reader, alphabetSize, header, budget)
|
||||
|
||||
|
||||
def _readBlockTypeCount(reader):
|
||||
|
|
@ -393,19 +449,24 @@ def _readBlockTypeCount(reader):
|
|||
return (1 << bits) + 1 + reader.readBits(bits)
|
||||
|
||||
|
||||
def _readContextMap(reader, treeCount, size):
|
||||
def _readContextMap(reader, treeCount, size, budget):
|
||||
maxRun = reader.readBits(4) + 1 if reader.readBits(1) else 0
|
||||
huffman = _readPrefix(reader, treeCount + maxRun)
|
||||
huffman = _readPrefix(reader, treeCount + maxRun, budget)
|
||||
contextMap = []
|
||||
while len(contextMap) < size:
|
||||
code = huffman.decode(reader)
|
||||
if code == 0:
|
||||
contextMap.append(0)
|
||||
elif code <= maxRun:
|
||||
contextMap.extend([0] * ((1 << code) + reader.readBits(code)))
|
||||
run = (1 << code) + reader.readBits(code)
|
||||
if len(contextMap) + run > size: # a run past the declared map size is malformed
|
||||
raise BrotliError("Brotli context map run overruns the map")
|
||||
contextMap.extend([0] * run)
|
||||
else:
|
||||
contextMap.append(code - maxRun)
|
||||
del contextMap[size:]
|
||||
value = code - maxRun
|
||||
if value >= treeCount: # references a tree that was not declared
|
||||
raise BrotliError("Brotli context map references an undefined tree")
|
||||
contextMap.append(value)
|
||||
if reader.readBits(1): # inverse move-to-front
|
||||
moveToFront = list(range(256))
|
||||
for i in range(len(contextMap)):
|
||||
|
|
@ -456,11 +517,8 @@ def decompress(data, maxOutput=100 * 1024 * 1024):
|
|||
"""Decompress a Brotli (RFC 7932) stream, returning the original bytes. Raises BrotliError on a
|
||||
malformed stream or if the output would exceed 'maxOutput' (an anti-decompression-bomb cap)."""
|
||||
|
||||
_loadTables()
|
||||
dictionary = _DICTIONARY
|
||||
context = _CONTEXT
|
||||
|
||||
try:
|
||||
dictionary, context = _loadTables()
|
||||
reader = _BitReader(data)
|
||||
header = reader.readBits(1)
|
||||
if header == 0:
|
||||
|
|
@ -506,12 +564,14 @@ def decompress(data, maxOutput=100 * 1024 * 1024):
|
|||
raise BrotliError("output too large")
|
||||
continue
|
||||
|
||||
budget = [_MAX_HUFFMAN_TABLE_ENTRIES] # per-meta-block Huffman memory ceiling
|
||||
|
||||
typesL = _readBlockTypeCount(reader)
|
||||
blockL, typeHuffmanL, lengthHuffmanL, prevTypeL = 1 << 28, None, None, 1
|
||||
typeL = 0
|
||||
if typesL >= 2:
|
||||
typeHuffmanL = _readPrefix(reader, typesL + 2)
|
||||
lengthHuffmanL = _readPrefix(reader, 26)
|
||||
typeHuffmanL = _readPrefix(reader, typesL + 2, budget)
|
||||
lengthHuffmanL = _readPrefix(reader, 26, budget)
|
||||
code = lengthHuffmanL.decode(reader)
|
||||
blockL = _BLEN_BASE[code] + reader.readBits(_BLEN_EXTRA[code])
|
||||
|
||||
|
|
@ -519,8 +579,8 @@ def decompress(data, maxOutput=100 * 1024 * 1024):
|
|||
blockI, typeHuffmanI, lengthHuffmanI, prevTypeI = 1 << 28, None, None, 1
|
||||
typeI = 0
|
||||
if typesI >= 2:
|
||||
typeHuffmanI = _readPrefix(reader, typesI + 2)
|
||||
lengthHuffmanI = _readPrefix(reader, 26)
|
||||
typeHuffmanI = _readPrefix(reader, typesI + 2, budget)
|
||||
lengthHuffmanI = _readPrefix(reader, 26, budget)
|
||||
code = lengthHuffmanI.decode(reader)
|
||||
blockI = _BLEN_BASE[code] + reader.readBits(_BLEN_EXTRA[code])
|
||||
|
||||
|
|
@ -528,8 +588,8 @@ def decompress(data, maxOutput=100 * 1024 * 1024):
|
|||
blockD, typeHuffmanD, lengthHuffmanD, prevTypeD = 1 << 28, None, None, 1
|
||||
typeD = 0
|
||||
if typesD >= 2:
|
||||
typeHuffmanD = _readPrefix(reader, typesD + 2)
|
||||
lengthHuffmanD = _readPrefix(reader, 26)
|
||||
typeHuffmanD = _readPrefix(reader, typesD + 2, budget)
|
||||
lengthHuffmanD = _readPrefix(reader, 26, budget)
|
||||
code = lengthHuffmanD.decode(reader)
|
||||
blockD = _BLEN_BASE[code] + reader.readBits(_BLEN_EXTRA[code])
|
||||
|
||||
|
|
@ -538,14 +598,14 @@ def decompress(data, maxOutput=100 * 1024 * 1024):
|
|||
contextModes = [reader.readBits(2) for _ in range(typesL)]
|
||||
|
||||
treesL = _readBlockTypeCount(reader)
|
||||
contextMapL = _readContextMap(reader, treesL, typesL * 64) if treesL >= 2 else [0] * (typesL * 64)
|
||||
contextMapL = _readContextMap(reader, treesL, typesL * 64, budget) if treesL >= 2 else [0] * (typesL * 64)
|
||||
treesD = _readBlockTypeCount(reader)
|
||||
contextMapD = _readContextMap(reader, treesD, typesD * 4) if treesD >= 2 else [0] * (typesD * 4)
|
||||
contextMapD = _readContextMap(reader, treesD, typesD * 4, budget) if treesD >= 2 else [0] * (typesD * 4)
|
||||
|
||||
huffmanL = [_readPrefix(reader, 256) for _ in range(treesL)]
|
||||
huffmanI = [_readPrefix(reader, 704) for _ in range(typesI)]
|
||||
huffmanL = [_readPrefix(reader, 256, budget) for _ in range(treesL)]
|
||||
huffmanI = [_readPrefix(reader, 704, budget) for _ in range(typesI)]
|
||||
distanceAlphabet = 16 + direct + (48 << postfix)
|
||||
huffmanD = [_readPrefix(reader, distanceAlphabet) for _ in range(treesD)]
|
||||
huffmanD = [_readPrefix(reader, distanceAlphabet, budget) for _ in range(treesD)]
|
||||
|
||||
produced = 0
|
||||
while produced < metaLength:
|
||||
|
|
@ -614,6 +674,9 @@ def decompress(data, maxOutput=100 * 1024 * 1024):
|
|||
low = value & ((1 << postfix) - 1)
|
||||
distance = ((((2 + (high & 1)) << extraBits) - 4 + extra) << postfix) + low + direct + 1
|
||||
|
||||
if distance <= 0: # a ring/short-code computation must yield >= 1
|
||||
raise BrotliError("invalid Brotli distance")
|
||||
|
||||
maxDistance = min(len(out), maxBackward)
|
||||
if distanceCode != 0 and distance <= maxDistance:
|
||||
distRing[distIndex & 3] = distance
|
||||
|
|
@ -637,6 +700,8 @@ def decompress(data, maxOutput=100 * 1024 * 1024):
|
|||
raise BrotliError("invalid dictionary transform")
|
||||
start = _OFFSETS[copyLength] + index * copyLength
|
||||
word = _applyTransform(transformId, dictionary[start:start + copyLength])
|
||||
if produced + len(word) > metaLength: # a transformed word must still fit the block
|
||||
raise BrotliError("dictionary word exceeds meta-block length")
|
||||
out += word
|
||||
produced += len(word)
|
||||
|
||||
|
|
@ -645,6 +710,13 @@ def decompress(data, maxOutput=100 * 1024 * 1024):
|
|||
|
||||
if isLast:
|
||||
break
|
||||
|
||||
# after the final meta-block only zero byte-alignment padding may remain: no whole leftover bytes
|
||||
# (trailing garbage) and the padding bits themselves must be zero (RFC 7932)
|
||||
if reader.bits + (reader.size - reader.pos) * 8 >= 8:
|
||||
raise BrotliError("trailing data after Brotli stream")
|
||||
if reader.acc != 0:
|
||||
raise BrotliError("non-zero Brotli padding bits")
|
||||
return bytes(out)
|
||||
except BrotliError:
|
||||
raise
|
||||
|
|
|
|||
|
|
@ -4,12 +4,14 @@
|
|||
Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org)
|
||||
See the file 'LICENSE' for copying permission
|
||||
|
||||
Tests for the dependency-free Brotli (RFC 7932) decompressor under lib/utils/brotli.py. The compressed
|
||||
fixtures were produced by the reference encoder at various quality levels; the expected plaintext is
|
||||
reconstructed here by construction, so the suite validates the decoder fully offline (no third-party
|
||||
'brotli' module at test time) on Python 2.7 / 3.x. Cases deliberately exercise the static dictionary +
|
||||
word transforms, long overlapping copies, UTF-8, the low-quality (near-uniform tree) path and the
|
||||
repetitive content that relies on the higher insert-and-copy command ranges.
|
||||
Tests for the dependency-free Brotli (RFC 7932) decompressor under lib/utils/brotli.py, and its wiring
|
||||
into lib/request/basic.py::decodePage. The compressed fixtures were produced by the reference encoder at
|
||||
various quality levels; the expected plaintext is reconstructed here by construction, so the suite
|
||||
validates the decoder fully offline (no third-party 'brotli' module at test time) on Python 2.7 / 3.x.
|
||||
Positive cases exercise the static dictionary + word transforms, long overlapping copies, UTF-8, the
|
||||
low-quality (near-uniform tree) path and the repetitive content that relies on the higher insert-and-copy
|
||||
command ranges. Negative cases assert that hostile input (truncation, garbage, output bombs) is rejected
|
||||
with a BrotliError rather than silently producing corrupted output.
|
||||
"""
|
||||
|
||||
import binascii
|
||||
|
|
@ -35,12 +37,15 @@ _CASES = [
|
|||
(b"the quick brown fox " * 30,
|
||||
"1b570200047463a92ee78362f22082d628041695d90acef9a3f135e9c701"),
|
||||
(b"AB" * 400, "1b1f0300a48284a2b230b009"),
|
||||
((u"caf\xe9 na\xefve \u4f60\u597d ").encode("utf-8") * 12,
|
||||
(u"caf\xe9 na\xefve \u4f60\u597d ".encode("utf-8") * 12,
|
||||
"1bef00004427477ad6d60ac38c93200a288ab462c2a06461d22d186dbbe0263e0707"),
|
||||
(b"hello hello hello world world foo bar baz " * 6,
|
||||
"8b7d000080aaaaaaeaff74e5f355048415f8c0000c201701d0ffbbeadf736f75cfa82e6f63b82b5e2c2c2c6c6cacea654675f0e1c38fc160308e33595583c16030180ce65067442a4aa370586827d97b828968074727f5b21e97eebd045d8baeefef94c3fca4fb1e"),
|
||||
]
|
||||
|
||||
# a valid stream (~1.2 KB of text) whose truncations feed the negative corpus
|
||||
_TRUNCATION_SAMPLE = binascii.unhexlify("1b570200047463a92ee78362f22082d628041695d90acef9a3f135e9c701")
|
||||
|
||||
|
||||
class TestBrotli(unittest.TestCase):
|
||||
def test_known_fixtures(self):
|
||||
|
|
@ -50,20 +55,66 @@ class TestBrotli(unittest.TestCase):
|
|||
def test_empty_stream(self):
|
||||
self.assertEqual(decompress(binascii.unhexlify("3b")), b"")
|
||||
|
||||
def test_malformed_raises(self):
|
||||
# a hostile/truncated stream must surface as BrotliError, never a raw exception
|
||||
for blob in (b"\xff", b"\x00\x00\x00", b"\x1b\xff\xff\xff\xff", os.urandom(32)):
|
||||
def test_truncation_is_rejected(self):
|
||||
# every proper prefix of a valid stream is truncated -> must raise, never silently return
|
||||
# corrupted or zero-padded output
|
||||
for cut in range(1, len(_TRUNCATION_SAMPLE)):
|
||||
self.assertRaises(BrotliError, decompress, _TRUNCATION_SAMPLE[:cut])
|
||||
|
||||
def test_malformed_is_rejected(self):
|
||||
for blob in (b"\xff", b"\x00\x00\x00", b"\x1b\xff\xff\xff\xff",
|
||||
_TRUNCATION_SAMPLE + b"\x00\x00\x00\x00", # trailing garbage
|
||||
binascii.unhexlify("3b") + b"\xde\xad"): # data after a complete empty stream
|
||||
self.assertRaises(BrotliError, decompress, blob)
|
||||
|
||||
def test_non_brotlierror_never_escapes(self):
|
||||
# arbitrary bytes must terminate quickly and only ever raise BrotliError (never a raw exception)
|
||||
for i in range(400):
|
||||
blob = bytes(bytearray((i * 37 + j * 13) & 0xff for j in range(i % 60)))
|
||||
try:
|
||||
decompress(blob)
|
||||
except BrotliError:
|
||||
pass
|
||||
except Exception as ex:
|
||||
self.fail("non-BrotliError on malformed input: %s" % ex)
|
||||
|
||||
def test_bomb_cap(self):
|
||||
# a small stream must not be allowed to expand past the output cap
|
||||
self.assertRaises(BrotliError, decompress, binascii.unhexlify("1b1f0300a48284a2b230b009"), 16)
|
||||
|
||||
|
||||
class TestBrotliDecodePage(unittest.TestCase):
|
||||
_KB = ("pageCompress", "pageEncoding", "disableHtmlDecoding", "singleLogFlags")
|
||||
_CONF = ("encoding", "nullConnection")
|
||||
|
||||
def setUp(self):
|
||||
from lib.core.data import conf, kb
|
||||
self._kb = dict((name, kb.get(name)) for name in self._KB)
|
||||
self._conf = dict((name, conf.get(name)) for name in self._CONF)
|
||||
|
||||
def tearDown(self):
|
||||
from lib.core.data import conf, kb
|
||||
for name, value in self._kb.items():
|
||||
kb[name] = value
|
||||
for name, value in self._conf.items():
|
||||
conf[name] = value
|
||||
|
||||
def test_decodepage_br(self):
|
||||
from lib.core.data import conf, kb
|
||||
from lib.request.basic import decodePage
|
||||
from lib.core.convert import getBytes
|
||||
|
||||
conf.encoding = None
|
||||
conf.nullConnection = False
|
||||
kb.pageCompress = True
|
||||
kb.pageEncoding = None
|
||||
kb.singleLogFlags = set()
|
||||
kb.disableHtmlDecoding = True
|
||||
|
||||
body = b"<html><body>secret uid=admin</body></html>" * 25
|
||||
# brotli-compressed 'body' (reference encoder, quality 11)
|
||||
compressed = binascii.unhexlify(
|
||||
"1b190488c56d6c1ff52d8742bd820d3870892cd08016f661030e310d82f520b7a3513810bf66edf05d3500")
|
||||
self.assertEqual(getBytes(decodePage(compressed, "br", "text/html; charset=utf-8")), body)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue