From 579e559c71159e58fd98da3852ecfbead8973ade Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0tampar?= Date: Thu, 30 Jul 2026 19:06:40 +0200 Subject: [PATCH] Adding one more data correctness patch --- lib/core/settings.py | 2 +- lib/request/inject.py | 35 +++++++++++++++++++++++++++++++++-- tests/test_charset.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 3 deletions(-) diff --git a/lib/core/settings.py b/lib/core/settings.py index 0bab7f42b..93273ac4c 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -20,7 +20,7 @@ from lib.core.enums import OS from thirdparty import six # sqlmap version (...) -VERSION = "1.10.7.252" +VERSION = "1.10.7.253" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/lib/request/inject.py b/lib/request/inject.py index 0af59544f..0c46ba173 100644 --- a/lib/request/inject.py +++ b/lib/request/inject.py @@ -594,6 +594,34 @@ def _pageCharsetCorrupted(value): applyFunctionRecursively(value, _) return retVal[0] +def _looksLikeMisdecodedUtf8(value): + """ + True if a retrieved value looks like UTF-8 data shown through a single-byte (ISO-8859-1) charset - + the '' mismatch that leaves NO undecodable bytes (latin-1 accepts every byte), + so _pageCharsetCorrupted() cannot see it, yet the data is silently mojibake (a UTF-8 accent shows as + two spurious latin-1 chars). + + Precise by construction: real mojibake re-encodes to latin-1 and decodes back to a DIFFERENT valid + UTF-8 string, whereas correctly-decoded UTF-8 ('cafe' with U+00E9) and genuine latin-1 text ('resume') + do not (a lone accent is a UTF-8 lead byte with no continuation -> decode fails). So it fires only on + the corruption, not on clean data. Complements _pageCharsetCorrupted() (the too-NARROW direction). + """ + + retVal = [False] + + def _(item): + if not retVal[0] and isinstance(item, six.string_types) and any(0x80 <= ord(_) <= 0xFF for _ in item): + try: + decoded = item.encode("latin-1").decode("utf-8") + except (UnicodeEncodeError, UnicodeDecodeError): + return item + if decoded != item and any(ord(_) > 0x7F for _ in decoded): + retVal[0] = True + return item + + applyFunctionRecursively(value, _) + return retVal[0] + @lockedmethod @stackedmethod def getValue(expression, blind=True, union=True, error=True, time=True, fromUser=False, expected=None, batch=False, unpack=True, resumeValue=True, charsetType=None, firstChar=None, lastChar=None, dump=False, suppressOutput=None, expectingNone=False, safeCharEncode=True): @@ -699,7 +727,7 @@ def getValue(expression, blind=True, union=True, error=True, time=True, fromUser if (found and not conf.hexConvert and not conf.binaryFields and expected not in (EXPECTED.BOOL, EXPECTED.INT) and getTechnique() in (PAYLOAD.TECHNIQUE.UNION, PAYLOAD.TECHNIQUE.ERROR, PAYLOAD.TECHNIQUE.QUERY) and Backend.getIdentifiedDbms() and hasattr(queries[Backend.getIdentifiedDbms()], "hex") - and _pageCharsetCorrupted(value)): + and (_pageCharsetCorrupted(value) or _looksLikeMisdecodedUtf8(value))): warnMsg = "retrieved data appears corrupted because of a charset mismatch between the " warnMsg += "DBMS and the web page. Re-fetching using hexadecimal encoding" singleTimeWarnMessage(warnMsg) @@ -710,7 +738,10 @@ def getValue(expression, blind=True, union=True, error=True, time=True, fromUser finally: conf.hexConvert = False - if _value is not None: + # only adopt the hex re-fetch if it is actually cleaner: a rare mis-trigger (or a + # column whose real charset the hex decode still cannot resolve) must never replace + # the original with equal-or-worse data + if _value is not None and not _pageCharsetCorrupted(_value) and not _looksLikeMisdecodedUtf8(_value): value = _value if found and conf.dnsDomain: diff --git a/tests/test_charset.py b/tests/test_charset.py index b7930bee0..9ef15fcf8 100644 --- a/tests/test_charset.py +++ b/tests/test_charset.py @@ -22,6 +22,7 @@ from _testutils import bootstrap bootstrap() from lib.request.basic import checkCharEncoding +from lib.request.inject import _pageCharsetCorrupted, _looksLikeMisdecodedUtf8 from lib.core.common import extractRegexResult, paramToDict from lib.core.enums import PLACE from lib.core.settings import META_CHARSET_REGEX, HTML_TITLE_REGEX, META_REFRESH_REGEX @@ -58,6 +59,38 @@ class TestBodyHints(unittest.TestCase): self.assertIsNone(extractRegexResult(HTML_TITLE_REGEX, "no title here")) +class TestCharsetMismatchDetection(unittest.TestCase): + """The auto-hex recovery triggers on two complementary mismatch directions: + _pageCharsetCorrupted (page charset too NARROW -> undecodable bytes) and + _looksLikeMisdecodedUtf8 (page charset too WIDE -> UTF-8 shown as latin-1, no + undecodable byte). Both must fire on the corruption and NOT on clean data.""" + + # correctly-decoded UTF-8 and genuine latin-1 text: neither detector may fire (no wasted re-fetch). + # \u escapes keep this source pure-ASCII (py2): cafe / naive / Zurich / CJK / Cyrillic / resume / garcon + CLEAN = [u"admin", u"caf\u00e9", u"na\u00efve", u"Z\u00fcrich", u"\u65e5\u672c\u8a9e", + u"\u0417\u0434\u0440\u0430\u0432\u0435\u0439", u"r\u00e9sum\u00e9", u"gar\u00e7on"] + + def test_clean_values_trigger_neither(self): + for v in self.CLEAN: + self.assertFalse(_pageCharsetCorrupted(v), msg="narrow FP: %r" % v) + self.assertFalse(_looksLikeMisdecodedUtf8(v), msg="wide FP: %r" % v) + + def test_utf8_shown_as_latin1_is_caught(self): + # gap #1: UTF-8 column bytes decoded as latin-1 -> valid mojibake, no undecodable byte + for word in (u"caf\u00e9", u"\u65e5\u672c\u8a9e", u"\u0417\u0434\u0440\u0430\u0432\u0435\u0439", u"\u20ac"): + mojibake = word.encode("utf-8").decode("latin-1") + self.assertFalse(_pageCharsetCorrupted(mojibake), msg="narrow should miss: %r" % mojibake) + self.assertTrue(_looksLikeMisdecodedUtf8(mojibake), msg="wide should catch: %r" % mojibake) + + def test_undecodable_bytes_still_caught_by_narrow(self): + # the other direction (page charset too narrow) leaves reversible \xNN escapes + self.assertTrue(_pageCharsetCorrupted(u"foo\\xe9bar")) + + def test_list_and_nonstring_inputs(self): + self.assertTrue(_looksLikeMisdecodedUtf8([u"ok", u"caf\u00c3\u00a9"])) # 'cafe' mojibake + self.assertFalse(_looksLikeMisdecodedUtf8([u"ok", 123, None])) + + class TestParamToDict(unittest.TestCase): # NOTE: GET parsing is covered in test_urls.py; here we only cover the COOKIE place, # which uses a different (semicolon) delimiter and is a distinct code path.