From 76b75e95f091d9e8ede62fd4b9bec7422da30e4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0tampar?= Date: Fri, 17 Jul 2026 14:46:52 +0200 Subject: [PATCH] Improving dialect checks --- lib/controller/checks.py | 13 ++- lib/core/dicts.py | 6 +- lib/core/settings.py | 2 +- lib/utils/dialect.py | 196 ++++++++++++++++++-------------- tests/test_dialectdbms.py | 231 ++++++++++++++++++++++++-------------- 5 files changed, 269 insertions(+), 179 deletions(-) diff --git a/lib/controller/checks.py b/lib/controller/checks.py index 527a52e3f..f7c9d5e31 100644 --- a/lib/controller/checks.py +++ b/lib/controller/checks.py @@ -166,9 +166,11 @@ def checkSqlInjection(place, parameter, value): # keyword-free fallback: heuristicCheckDbms() above uses SELECT/quote payloads # and is skipped when the WAF/IPS is dropping requests; the operator-dialect # probes carry no SELECT/quote/schema name, so they can still narrow the DBMS in - # that case (or when it was inconclusive), using the now-calibrated boolean oracle - if not Backend.getIdentifiedDbms() and kb.heuristicDbms is None: - kb.heuristicDbms = dialectCheckDbms(injection) + # that case (or when it was inconclusive), using the now-calibrated boolean oracle. + # It feeds the lower-confidence heuristicExtendedDbms (UNION FROM / handler hint), + # deliberately NOT heuristicDbms, so it never drives reduceTests (skipping payloads) + if not Backend.getIdentifiedDbms() and kb.heuristicDbms is None and kb.heuristicExtendedDbms is None: + kb.heuristicExtendedDbms = dialectCheckDbms(injection) # If the DBMS has already been fingerprinted (via DBMS-specific # error message, simple heuristic check or via DBMS-specific @@ -730,7 +732,8 @@ def checkSqlInjection(place, parameter, value): if len(kb.dbmsFilter or []) == 1: Backend.forceDbms(kb.dbmsFilter[0]) elif not Backend.getIdentifiedDbms(): - if kb.heuristicDbms is None: + heuristicDbms = kb.heuristicDbms or kb.heuristicExtendedDbms + if heuristicDbms is None: if kb.heuristicTest == HEURISTIC_TEST.POSITIVE or injection.data: warnMsg = "using unescaped version of the test " warnMsg += "because of zero knowledge of the " @@ -738,7 +741,7 @@ def checkSqlInjection(place, parameter, value): warnMsg += "explicitly set it with option '--dbms'" singleTimeWarnMessage(warnMsg) else: - Backend.forceDbms(kb.heuristicDbms) + Backend.forceDbms(heuristicDbms) if unionExtended: infoMsg = "automatically extending ranges for UNION " diff --git a/lib/core/dicts.py b/lib/core/dicts.py index 2387a4772..821e2abef 100644 --- a/lib/core/dicts.py +++ b/lib/core/dicts.py @@ -294,13 +294,15 @@ HEURISTIC_NULL_EVAL = { DBMS.ACCESS: "CVAR(NULL)", DBMS.MAXDB: "ALPHA(NULL)", DBMS.MSSQL: "PARSENAME(NULL,NULL)", + DBMS.SYBASE: "STR_REPLACE(NULL,'x','x')", # ASE extension (MSSQL has REPLACE); doc-derived, not live-tested. Also SAP IQ / SQL Anywhere (not in DBMS) DBMS.MYSQL: "IFNULL(QUARTER(NULL),NULL XOR NULL)", # NOTE: previous form (i.e., QUARTER(NULL XOR NULL)) was bad as some optimization engines wrongly evaluate QUARTER(NULL XOR NULL) to 0 DBMS.ORACLE: "INSTR2(NULL,NULL)", DBMS.PGSQL: "QUOTE_IDENT(NULL)", DBMS.SQLITE: "JULIANDAY(NULL)", DBMS.H2: "STRINGTOUTF8(NULL)", DBMS.MONETDB: "CODE(NULL)", - DBMS.DERBY: "NULLIF(USER,SESSION_USER)", + DBMS.DERBY: "NULLIF(USER,SESSION_USER)", # not Derby-specific; safe only because DB2 (shares this dummy) is tested first + DBMS.DB2: "MULTIPLY_ALT(NULL,NULL)", # DB2-unique (Derby shares the dummy, not this function) DBMS.VERTICA: "BITSTRING_TO_BINARY(NULL)", DBMS.MCKOI: "TONUMBER(NULL)", DBMS.PRESTO: "FROM_HEX(NULL)", @@ -314,7 +316,7 @@ HEURISTIC_NULL_EVAL = { DBMS.VIRTUOSO: "__MAX_NOTNULL(NULL)", DBMS.CLICKHOUSE: "halfMD5(NULL)", DBMS.SNOWFLAKE: "BOOLNOT(NULL)", - DBMS.SPANNER: "FARM_FINGERPRINT(NULL)", + DBMS.SPANNER: "FARM_FINGERPRINT(NULL)", # also BigQuery GoogleSQL (not in DBMS) DBMS.HANA: "MAP(NULL,NULL,NULL,NULL,NULL)", } diff --git a/lib/core/settings.py b/lib/core/settings.py index d0f79826d..355655689 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -20,7 +20,7 @@ from lib.core.enums import OS from thirdparty import six # sqlmap version (...) -VERSION = "1.10.7.111" +VERSION = "1.10.7.112" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/lib/utils/dialect.py b/lib/utils/dialect.py index 47f973edc..b76fcf058 100644 --- a/lib/utils/dialect.py +++ b/lib/utils/dialect.py @@ -12,109 +12,126 @@ from lib.core.data import conf from lib.core.data import kb from lib.core.data import logger from lib.core.enums import DBMS +from lib.core.settings import SINGLE_QUOTE_MARKER from lib.request.inject import checkBooleanExpression -# Operator-dialect probes for a keyword-free back-end DBMS heuristic. -# -# Each probe is an arithmetic identity that holds only in the dialect(s) noted, using operator -# *semantics* alone - no SQL keywords, functions, quotes or schema names. It complements -# heuristicCheckDbms() (which uses (SELECT 'x')='x' string round-trips): the dialect probes carry -# no SELECT/quote, so they can narrow the back-end DBMS where those are dropped (e.g. a -# keyword-matching WAF/IPS, or when kb.droppingRequests has it skipped entirely). -# -# Each probe is evaluated through checkBooleanExpression(), i.e. as an appended boolean -# (... AND ()), which yields a clean true/false from the comparison oracle. (A value-position -# variant - replacing the value with id=2^0 etc. - was prototyped and rejected: those probes land on -# OTHER valid rows, which sqlmap's fuzzy page comparison conflates with the anchor row, producing -# false positives. See PROVE_DESIGN.md.) -# -# Signatures were measured against every SQL engine on a live OWASP-CRS platform (MySQL/MySQL5, -# MariaDB/TiDB, PostgreSQL, CockroachDB, CrateDB, Microsoft SQL Server, SQLite, Firebird, ClickHouse, -# H2, HSQLDB, Derby, MonetDB, IRIS, Trino) and encoded as an exact-signature WHITELIST in _classify() -# (only measured signatures classify; anything else -> None). With anchor value 2: -# -# * 2^0=2 -> '^' is bitwise XOR (MySQL/MSSQL/MonetDB: 2^0=2) vs exponentiation (PostgreSQL: 2^0=1) -# vs no such operator (SQLite/Oracle/... -> error, so false) -# * 2^3=8 -> '^' is exponentiation (PostgreSQL/CockroachDB/CrateDB: 2^3=8) - false for XOR dialects -# (2^3=1) and erroring dialects; a positive PostgreSQL-family marker. CAVEAT: -# '^'=exponentiation is not strictly unique to PostgreSQL - MS Access/Jet and DuckDB -# also use it (neither on the platform), so this can read as PostgreSQL there. -# * 5/2=2 -> integer division (PostgreSQL/MSSQL/SQLite/MonetDB) vs real division (MySQL/Oracle: 2.5) -# * 2|0=2 -> a bitwise OR operator exists (absent in Firebird/Oracle/ClickHouse/H2) -# * 1<<2=4 -> a bit-shift operator exists. MonetDB shares MSSQL's (xor, intdiv) = (True, True) -# signature exactly, which would misread MonetDB as SQL Server; MonetDB HAS '<<' while -# SQL Server has NO shift operator (any version) -> this probe splits that one collision. +# Operator/typing-dialect probes for a WAF-tolerant back-end DBMS heuristic, complementing +# heuristicCheckDbms() for when a WAF/IPS drops its SELECT/quote payloads. Each probe is fed to +# checkBooleanExpression() (appended as ... AND ()); all but catplus use only non-alphanumeric +# operators (WAF-friendly). Minimal set giving a collision-free signature to the classes below. DIALECT_PROBES = ( - ("xor", "2^0=2"), - ("pgpow", "2^3=8"), - ("intdiv", "5/2=2"), - ("bitor", "2|0=2"), - ("shift", "1<<2=4"), + ("pow", "2^3=8"), # '^' is exponentiation + ("intdiv", "5/2=2"), # integer division + ("mod", "5%2=1"), # '%' modulo operator + ("bitor", "2|0=2"), # '|' bitwise-OR operator + ("xeq", "1^=2"), # '^=' not-equal operator + ("bslash", "5\\2=2"), # '\' integer division + ("catplus", "%sa%s+%sb%s=%sab%s" % ((SINGLE_QUOTE_MARKER,) * 6)), # '+' concatenates strings + ("numcat", "1||1=11"), # '||' concatenates with numeric coercion ) -# Canary for the trustworthiness gate: a syntactically-invalid expression (a trailing operator) that -# a real SQL back-end can only read as FALSE - the appended clause is a parse error, the query fails, -# no row. A false-positive / noise channel (a WAF, a reflection, or a backend that ignores the -# injected tail and reads every probe the same) reads it as TRUE, which is proof the boolean oracle -# is trash, so the heuristic returns None (a true negative) rather than a bogus DBMS from a -# meaningless signature. It uses a trailing-operator form, distinct from the ' ' no-operator -# form already exercised by sqlmap's earlier false-positive check, so it adds new information. +# Trust gate: a syntactically-invalid trailing-operator expression a real back-end can only read as +# FALSE. A noise/false-positive channel reads it TRUE, proving the oracle is untrustworthy -> None. DIALECT_CANARY = "2+" -# Exact operator-dialect signature -> back-end DBMS. Strict WHITELIST re-derived from the live -# measurement above: ONLY these signatures classify; any other - an engine not measured here, or a -# false-positive / noise channel - returns None. This deliberately replaces earlier partial-condition -# rules, which would confidently mis-map physically-impossible signatures onto a DBMS (e.g. the -# all-true 'reads everything as true' noise, where '^' would be XOR and exponentiation at once). +# Reachability canaries for adversarial WAFs that selectively drop operator characters. A dropped probe +# reads FALSE, indistinguishable from a semantic FALSE, silently degrading the signature. Each canary +# embeds probe characters inside a string literal (semantically inert), so it is universally TRUE unless +# the WAF filters a character. The combined form is a fast path; on failure the per-char forms (via +# _reachCanary) locate the blocked bits, which _classify() then treats as unknown/wildcard. +_DIALECT_REACH_CHARS = (("^", (0, 4)), ("/", (1,)), ("%", (2,)), ("|", (3, 7)), ("\\", (5,)), ("+", (6,))) +_DIALECT_REACH_BODY = "a^b/c%d|e\\f+g" +DIALECT_REACH_CANARY = "%s%s%s=%s%s%s" % (SINGLE_QUOTE_MARKER, _DIALECT_REACH_BODY, SINGLE_QUOTE_MARKER, SINGLE_QUOTE_MARKER, _DIALECT_REACH_BODY, SINGLE_QUOTE_MARKER) + +# Exact operator-dialect signature -> back-end DBMS (strict whitelist). Any signature not listed - an +# unmeasured engine/version or a noise channel - returns None, so the heuristic never wrong-foots a scan. +# All rows are live-measured except Spanner (documentation-derived, tagged inline). _SIGNATURE_DBMS = { - # xor pgpow intdiv bitor shift - (True, False, False, True, True): DBMS.MYSQL, # MySQL / MariaDB / TiDB - (False, True, True, True, True): DBMS.PGSQL, # PostgreSQL - (False, True, False, True, True): DBMS.PGSQL, # CockroachDB (pgwire; has '<<' -> shift True) - (False, True, True, True, False): DBMS.PGSQL, # CrateDB - (True, False, True, True, False): DBMS.MSSQL, # Microsoft SQL Server (no bit-shift) - (True, False, True, True, True): DBMS.MONETDB, # MonetDB (as MSSQL but has '<<') - (False, False, True, True, True): DBMS.SQLITE, # SQLite + # pow intdiv mod bitor xeq bslash catplus numcat + (False, False, False, False, False, False, False, True): DBMS.INFORMIX, # Informix + (False, False, False, False, False, True, False, True): DBMS.CACHE, # InterSystems IRIS/Cache ('\' int-div) + (False, False, False, False, True, False, False, True): DBMS.ORACLE, # Oracle ('^=' not-equal) + (False, False, False, True, False, False, False, False): DBMS.SPANNER, # Google Cloud Spanner (only '|' works) - doc-derived, not live-tested + (False, False, True, False, False, False, False, True): DBMS.CLICKHOUSE, # ClickHouse (no bitwise-OR) + (False, False, True, True, False, False, True, True): DBMS.MYSQL, # MySQL / MariaDB / TiDB + (False, True, False, False, False, False, False, False): DBMS.DERBY, # Apache Derby + (False, True, False, False, False, False, True, False): DBMS.HSQLDB, # HSQLDB ('+' concat) + (False, True, False, False, True, False, False, True): DBMS.FIREBIRD, # Firebird ('^=') + (False, True, True, False, False, False, False, False): DBMS.PRESTO, # Presto / Trino + (False, True, True, False, False, False, False, True): DBMS.H2, # H2 + (False, True, True, True, False, False, False, False): DBMS.SQLITE, # SQLite + (False, True, True, True, False, False, False, True): DBMS.MONETDB, # MonetDB + (False, True, True, True, False, False, True, False): DBMS.MSSQL, # Microsoft SQL Server (2019 AND 2022) + (False, True, True, True, False, False, True, True): DBMS.CUBRID, # CUBRID (like MonetDB but '+' concat) + (False, True, True, True, True, False, False, True): DBMS.DB2, # IBM DB2 ('^=', no '<<'/'\') + (True, False, False, False, False, True, True, False): DBMS.ACCESS, # Microsoft Access (ACE/JET: '^' exp + '\' int-div + '+' concat) + (True, False, True, True, False, False, False, False): DBMS.PGSQL, # PostgreSQL + (True, False, True, True, False, False, False, True): DBMS.VERTICA, # Vertica (pg-derived but numeric '||') + (True, False, True, True, True, False, False, True): DBMS.PGSQL, # openGauss (Oracle-compat '^=') + (True, True, True, True, False, False, False, False): DBMS.PGSQL, # PostgreSQL / CrateDB variant + (True, True, True, True, False, False, False, True): DBMS.PGSQL, # PostgreSQL variant } -def _classify(signature): +def _classify(signature, unknown=()): """ - Maps an exact operator-dialect signature (xor, pgpow, intdiv, bitor, shift) to a back-end DBMS - through a strict whitelist of live-measured signatures, or returns None when the signature is not - a known DBMS fingerprint - an engine not measured, or a noise / false-positive channel - so - detection proceeds unchanged and the heuristic never wrong-foots the scan. + Maps an exact 8-bit operator/typing signature to a back-end DBMS via the strict whitelist, or None + when the signature is not a known fingerprint (unmeasured engine, or a noise/false-positive channel). - >>> _classify((True, False, False, True, True)) # MySQL / MariaDB / TiDB + 'unknown' holds bit indices whose probe character a WAF is dropping (so their FALSE is meaningless); + they are treated as wildcards and a DBMS is named only when the remaining trusted bits are unanimous, + which cannot misclassify (the true signature is always among the candidates). + + >>> _classify((False, False, True, True, False, False, True, True), unknown={2}) # MySQL, mod blocked -> still unique 'MySQL' - >>> _classify((False, True, True, True, True)) # PostgreSQL - 'PostgreSQL' - >>> _classify((False, True, False, True, True)) # CockroachDB -> PostgreSQL family - 'PostgreSQL' - >>> _classify((False, True, True, True, False)) # CrateDB -> PostgreSQL family - 'PostgreSQL' - >>> _classify((True, False, True, True, False)) # Microsoft SQL Server (no bit-shift) + >>> _classify((False, True, True, False, False, False, False, True), unknown={7}) is None # H2 vs Presto ambiguous -> None + True + >>> _classify((False, False, True, True, False, False, True, True)) # MySQL / MariaDB / TiDB + 'MySQL' + >>> _classify((False, True, True, True, False, False, True, False)) # Microsoft SQL Server (2019 and 2022) 'Microsoft SQL Server' - >>> _classify((True, False, True, True, True)) # MonetDB (as MSSQL but has '<<') + >>> _classify((False, True, True, True, False, False, False, True)) # MonetDB 'MonetDB' - >>> _classify((False, False, True, True, True)) # SQLite + >>> _classify((True, True, True, True, False, False, False, False)) # PostgreSQL / CrateDB + 'PostgreSQL' + >>> _classify((False, True, True, True, False, False, False, False)) # SQLite 'SQLite' - >>> _classify((True, True, True, True, True)) is None # 'reads everything true' noise -> None - True - >>> _classify((False, False, False, False, False)) is None # all-false (Oracle/ClickHouse/IRIS/blocked) -> None - True - >>> _classify((False, False, True, False, False)) is None # Firebird/H2/HSQLDB/Derby/Trino -> not distinctive + >>> _classify((False, False, True, False, False, False, False, True)) # ClickHouse + 'ClickHouse' + >>> _classify((False, False, False, False, True, False, False, True)) # Oracle ('^=') + 'Oracle' + >>> _classify((False, False, False, False, False, True, False, True)) # InterSystems IRIS/Cache (Oracle but no '^=') + 'InterSystems Cache' + >>> _classify((False, True, False, False, True, False, False, True)) # Firebird + 'Firebird' + >>> _classify((False, True, False, False, False, False, False, False)) # Apache Derby + 'Apache Derby' + >>> _classify((True, False, False, False, False, True, True, False)) # Microsoft Access ('^' exp + '\' int-div) + 'Microsoft Access' + >>> _classify((False, False, False, True, False, False, False, False)) # Google Cloud Spanner (doc-derived: only '|' bitwise-OR) + 'Spanner' + >>> _classify((True, True, True, True, True, True, True, True)) is None # unmeasured / noise -> None True """ - return _SIGNATURE_DBMS.get(tuple(bool(_) for _ in signature)) + signature = tuple(bool(_) for _ in signature) + + if not unknown: + return _SIGNATURE_DBMS.get(signature) + + unknown = set(unknown) + candidates = set(dbms for sig, dbms in _SIGNATURE_DBMS.items() if all(sig[i] == signature[i] for i in range(len(signature)) if i not in unknown)) + + return next(iter(candidates)) if len(candidates) == 1 else None + +def _reachCanary(char): + lit = "%sx%sx%s" % (SINGLE_QUOTE_MARKER, char, SINGLE_QUOTE_MARKER) + return "%s=%s" % (lit, lit) def dialectCheckDbms(injection): """ - Keyword-free back-end DBMS heuristic via operator-dialect differentials, evaluated through the - given (boolean-capable) injection. Complements heuristicCheckDbms() - which is skipped when the - WAF/IPS is dropping requests and otherwise relies on SELECT/quote payloads - because every probe - here is built from operator semantics alone. Returns the DBMS name or None; an ambiguous, - WAF-blocked or false-positive channel yields None, leaving the scan unchanged. + Keyword-free back-end DBMS heuristic via operator-dialect differentials, evaluated through the given + (boolean-capable) injection. Complements heuristicCheckDbms() (whose SELECT/quote payloads a WAF/IPS + may drop). Returns the DBMS name, or None for an ambiguous, WAF-blocked or false-positive channel. """ retVal = None @@ -126,14 +143,19 @@ def dialectCheckDbms(injection): kb.injection = injection try: - # Trustworthiness gate: a real boolean oracle reads a tautology TRUE, a contradiction FALSE, - # and a syntactically-invalid canary FALSE (the appended clause is a parse error -> the query - # fails). A false-positive / noise channel reads them all alike - the canary as TRUE - which - # is proof the oracle is trash, so classification is skipped (a true negative) instead of - # emitting a bogus DBMS from a meaningless signature. + # trust gate: a real oracle reads the tautology TRUE, the contradiction FALSE, the invalid + # canary FALSE. A noise channel reads them alike (canary TRUE) -> skip rather than guess. if checkBooleanExpression("2=2") and not checkBooleanExpression("2=3") and not checkBooleanExpression(DIALECT_CANARY): signature = tuple(bool(checkBooleanExpression(expr)) for _, expr in DIALECT_PROBES) - retVal = _classify(signature) + + # detect WAF-dropped probe characters (a dropped probe reads FALSE); mark their bits unknown + unknown = set() + if not checkBooleanExpression(DIALECT_REACH_CANARY): + for char, bits in _DIALECT_REACH_CHARS: + if not checkBooleanExpression(_reachCanary(char)): + unknown.update(bits) + + retVal = _classify(signature, unknown) finally: kb.injection = popValue() diff --git a/tests/test_dialectdbms.py b/tests/test_dialectdbms.py index 040d80b1a..26520ca74 100644 --- a/tests/test_dialectdbms.py +++ b/tests/test_dialectdbms.py @@ -4,13 +4,9 @@ Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org) See the file 'LICENSE' for copying permission -Operator-dialect DBMS heuristic (lib/utils/dialect.py). These lock in the empirical truth table: -the full 5-probe (2^0=2, 2^3=8, 5/2=2, 2|0=2, 1<<2=4) operator signatures measured across the live -SQL engines on an OWASP-CRS test platform, asserting _classify() maps each EXACT signature to the -expected back-end DBMS via its whitelist - and, just as importantly, that anything else (an -unmeasured engine, an ambiguous signature, or a physically-impossible / noise signature) maps to -None, so the heuristic never wrong-foots detection. The end-to-end behaviour (the probes producing -these signatures through a real boolean injection) is exercised against the live platform, not here. +Operator/typing-dialect DBMS heuristic (lib/utils/dialect.py). Locks in the empirical 8-probe truth +table: each measured signature maps to its expected back-end DBMS, and every other signature (unmeasured +engine, ambiguous, or noise) maps to None so the heuristic never wrong-foots detection. """ import os @@ -25,119 +21,186 @@ import lib.utils.dialect as dialect from lib.core.data import kb from lib.core.enums import DBMS from lib.utils.dialect import _classify +from lib.utils.dialect import _reachCanary from lib.utils.dialect import dialectCheckDbms from lib.utils.dialect import DIALECT_CANARY +from lib.utils.dialect import DIALECT_PROBES +from lib.utils.dialect import DIALECT_REACH_CANARY +from lib.utils.dialect import _DIALECT_REACH_CHARS -# Full 5-probe signature (2^0=2, 2^3=8, 5/2=2, 2|0=2, 1<<2=4) measured live -> expected DBMS. -# Every bit is significant now (whitelist): e.g. MySQL/PostgreSQL/... all have a working '<<', so -# shift=True is part of their signature; a one-bit-off variant is simply not a known fingerprint. +# Full 8-probe signature (pow, intdiv, mod, bitor, xeq, bslash, catplus, numcat) measured live -> DBMS. +# Every bit is significant (strict whitelist); a one-bit-off variant is simply not a known fingerprint. MEASURED = { - "mysql": ((True, False, False, True, True), DBMS.MYSQL), - "mysql5": ((True, False, False, True, True), DBMS.MYSQL), - "tidb": ((True, False, False, True, True), DBMS.MYSQL), # MySQL wire-compatible - "postgres": ((False, True, True, True, True), DBMS.PGSQL), - "cockroach": ((False, True, False, True, True), DBMS.PGSQL), # pgwire (exponent '^', decimal division, has '<<') - "cratedb": ((False, True, True, True, False), DBMS.PGSQL), # pgwire family (no '<<') - "mssql": ((True, False, True, True, False), DBMS.MSSQL), # '^' XOR, integer division, NO bit-shift - "monetdb": ((True, False, True, True, True), DBMS.MONETDB), # shares MSSQL base but HAS '<<' - "sqlite": ((False, False, True, True, True), DBMS.SQLITE), - # not distinctive enough -> deliberately no prior (operators alone can't safely separate these) - "firebird": ((False, False, True, False, False), None), - "hsqldb": ((False, False, True, False, False), None), # collides with firebird/derby/h2/trino - "derby": ((False, False, True, False, False), None), - "h2": ((False, False, True, False, False), None), - "trino": ((False, False, True, False, False), None), - "iris": ((False, False, False, False, False), None), # all-error, like Oracle/broken channel - "clickhouse": ((False, False, False, False, False), None), # all-error, like Oracle/broken channel + "mysql": (False, False, True , True , False, False, True , True , DBMS.MYSQL), + "mysql5": (False, False, True , True , False, False, True , True , DBMS.MYSQL), + "tidb": (False, False, True , True , False, False, True , True , DBMS.MYSQL), # MySQL wire-compatible + "postgres": (True , True , True , True , False, False, False, False, DBMS.PGSQL), + "opengauss": (True , False, True , True , True , False, False, True , DBMS.PGSQL), # Oracle-compat '^=' + "cockroach": (True , False, True , True , False, False, False, False, DBMS.PGSQL), # decimal division + "cratedb": (True , True , True , True , False, False, False, True , DBMS.PGSQL), + "mssql2019": (False, True , True , True , False, False, True , False, DBMS.MSSQL), # no '<<' + "mssql2022": (False, True , True , True , False, False, True , False, DBMS.MSSQL), # gained '<<' but shift is not a probe -> same signature + "sqlite": (False, True , True , True , False, False, False, False, DBMS.SQLITE), + "clickhouse":(False, False, True , False, False, False, False, True , DBMS.CLICKHOUSE),# no bitwise-OR + "monetdb": (False, True , True , True , False, False, False, True , DBMS.MONETDB), # like MSSQL but no '+' concat + "firebird": (False, True , False, False, True , False, False, True , DBMS.FIREBIRD), # has '^=' + "h2": (False, True , True , False, False, False, False, True , DBMS.H2), + "hsqldb": (False, True , False, False, False, False, True , False, DBMS.HSQLDB), # '+' concat + "derby": (False, True , False, False, False, False, False, False, DBMS.DERBY), + "iris": (False, False, False, False, False, True , False, True , DBMS.CACHE), # '\' int-div (Oracle-like but no '^=') + "trino": (False, True , True , False, False, False, False, False, DBMS.PRESTO), + "oracle": (False, False, False, False, True , False, False, True , DBMS.ORACLE), # '^=' not-equal + "informix": (False, False, False, False, False, False, False, True , DBMS.INFORMIX), + "cubrid": (False, True , True , True , False, False, True , True , DBMS.CUBRID), # like MonetDB but '+' concat + "db2": (False, True , True , True , True , False, False, True , DBMS.DB2), # '^=', no '<<'/'\' + "vertica": (True , False, True , True , False, False, False, True , DBMS.VERTICA), # pg-derived but numeric '||' + "access": (True , False, False, False, False, True , True , False, DBMS.ACCESS), # ACE/JET: '^' exp + '\' int-div + '+' concat, no '%'/'|'/'||' } +# Documentation-derived (vendor operator spec, not live-tested); only where the signature is a free slot. +DOCUMENTED = { + "spanner": (False, False, False, True , False, False, False, False, DBMS.SPANNER), +} + +_PROBE_COUNT = len(DIALECT_PROBES) +_ALL = dict(MEASURED); _ALL.update(DOCUMENTED) + + +def _sig(engine): + return _ALL[engine][:_PROBE_COUNT] + class TestDialectClassification(unittest.TestCase): - def test_measured_engines_map_as_expected(self): - # each engine's exact measured 5-probe signature maps to its expected DBMS (or None) - for engine, (signature, expected) in MEASURED.items(): - self.assertEqual(_classify(signature), expected, "engine %r misclassified" % engine) + def test_probe_count_matches_signature_width(self): + # the MEASURED rows and the doctested matrix must stay the same width as DIALECT_PROBES + self.assertEqual(_PROBE_COUNT, 8) - def test_shift_splits_monetdb_from_mssql(self): - # MonetDB shares MSSQL's (xor, intdiv) base exactly (a false positive before the shift probe); - # 1<<2=4 (MonetDB has it, SQL Server never does) is the sole separator. - self.assertEqual(_classify((True, False, True, True, False)), DBMS.MSSQL) - self.assertEqual(_classify((True, False, True, True, True)), DBMS.MONETDB) + def test_measured_engines_map_as_expected(self): + # each engine's exact measured 8-probe signature maps to its expected DBMS + for engine, row in MEASURED.items(): + self.assertEqual(_classify(row[:_PROBE_COUNT]), row[_PROBE_COUNT], "engine %r misclassified" % engine) + + def test_documented_engines_map_as_expected(self): + # doc-derived rows (not live-tested) still resolve to their expected DBMS via the whitelist + for engine, row in DOCUMENTED.items(): + self.assertEqual(_classify(row[:_PROBE_COUNT]), row[_PROBE_COUNT], "engine %r misclassified" % engine) + + def test_mssql_version_agnostic(self): + # SQL Server 2022 gained '<<' but 'shift' is deliberately NOT a probe (it collided MSSQL 2022 + # with MonetDB); both versions share one signature and '+' concat separates MSSQL from MonetDB. + self.assertEqual(_classify(_sig("mssql2019")), DBMS.MSSQL) + self.assertEqual(_classify(_sig("mssql2022")), DBMS.MSSQL) + self.assertEqual(_classify(_sig("monetdb")), DBMS.MONETDB) + + def test_oracle_iris_split_by_operators(self): + # Oracle and IRIS are near-identical; '^=' (Oracle) vs '\' int-div (IRIS) split them. + self.assertEqual(_classify(_sig("oracle")), DBMS.ORACLE) + self.assertEqual(_classify(_sig("iris")), DBMS.CACHE) + + def test_previously_colliding_engines_now_split(self): + # the 4 late-measured engines collided on smaller probe sets; the 8-probe matrix separates them + # (Informix != IRIS, CUBRID != MonetDB, Vertica != PostgreSQL, DB2 != all-true noise). + self.assertEqual(_classify(_sig("informix")), DBMS.INFORMIX) + self.assertEqual(_classify(_sig("cubrid")), DBMS.CUBRID) + self.assertEqual(_classify(_sig("vertica")), DBMS.VERTICA) + self.assertEqual(_classify(_sig("db2")), DBMS.DB2) def test_whitelist_is_exact_no_false_positive(self): - # only the measured classifying signatures may yield a DBMS; everything else -> None. - classifying = set(sig for sig, exp in MEASURED.values() if exp is not None) - produced = set(exp for _, exp in MEASURED.values() if exp is not None) - self.assertEqual(produced, {DBMS.MYSQL, DBMS.PGSQL, DBMS.MSSQL, DBMS.MONETDB, DBMS.SQLITE}) - # exhaustively sweep all 32 signatures: a non-None result is allowed ONLY for a measured one - for bits in range(32): - sig = tuple(bool(bits & (1 << i)) for i in range(5)) - result = _classify(sig) + # exhaustively sweep all 256 signatures: a non-None result is allowed ONLY for a known one + classifying = set(row[:_PROBE_COUNT] for row in _ALL.values()) + for bits in range(1 << _PROBE_COUNT): + sig = tuple(bool(bits & (1 << i)) for i in range(_PROBE_COUNT)) if sig not in classifying: - self.assertIsNone(result, "unmeasured signature %r wrongly mapped to %r" % (sig, result)) + self.assertIsNone(_classify(sig), "unmeasured signature %r wrongly mapped to %r" % (sig, _classify(sig))) def test_all_true_noise_is_rejected(self): - # a channel that reads EVERY probe true (a static/reflected page, or a WAF/false-positive - # oracle) produces the all-true signature - physically impossible ('^' cannot be XOR and - # exponentiation at once). It must NOT be guessed (previously it mis-read as PostgreSQL). - self.assertIsNone(_classify((True, True, True, True, True))) - - def test_all_error_signature_yields_no_prior(self): - # an all-error signature (Oracle, ClickHouse, IRIS, or a WAF-blocked channel) is not - # distinctive - it must NOT be guessed as any DBMS - self.assertIsNone(_classify((False, False, False, False, False))) - self.assertIsNone(_classify((False, False, False, False, True))) - - def test_pgpow_alone_is_not_enough(self): - # exponentiation '^' is a PostgreSQL marker, but pgpow ALONE no longer classifies: the full - # signature must match a measured PostgreSQL fingerprint (this is what stops the all-true noise - # from riding the old 'pgpow dominates' rule into a bogus PostgreSQL claim). - self.assertEqual(_classify((False, True, True, True, True)), DBMS.PGSQL) # real PostgreSQL - self.assertIsNone(_classify((True, True, False, False, False))) # pgpow set, but not a real signature + # a channel reading EVERY probe true (static/reflected page or false-positive oracle) is + # physically impossible and must NOT be guessed + self.assertIsNone(_classify((True,) * _PROBE_COUNT)) class TestDialectCheckDbmsGuard(unittest.TestCase): - """dialectCheckDbms() end-to-end with a mocked boolean oracle: correct DBMS on a good channel, - and None (no prior) whenever the channel is unreliable - the safety contract, including the - canary that turns a trashy false-positive channel into a true negative.""" + """dialectCheckDbms() end-to-end with a mocked boolean oracle: correct DBMS on a good channel, and + None whenever the channel is unreliable (including the canary that turns a trashy false-positive + channel into a true negative).""" - def _run(self, truth): - # truth: {expression: bool} simulating checkBooleanExpression through a confirmed injection + def _run(self, probeBits, gate=(True, False, False), blocked=()): + # probeBits: {probe_name: bool}; gate: (2=2, 2=3, canary); blocked: chars a WAF drops + truth = {"2=2": gate[0], "2=3": gate[1], DIALECT_CANARY: gate[2]} + for name, expr in DIALECT_PROBES: + truth[expr] = bool(probeBits.get(name, False)) + # clean channel: all reachability canaries TRUE; a blocked char reads FALSE (combined + per-char) + truth[DIALECT_REACH_CANARY] = not blocked + for char, _ in _DIALECT_REACH_CHARS: + truth[_reachCanary(char)] = char not in blocked orig = dialect.checkBooleanExpression dialect.checkBooleanExpression = lambda expr, **kwargs: bool(truth.get(expr, False)) saved = kb.get("injection") try: - return dialectCheckDbms(object()) # the injection arg is only stashed, never inspected here + return dialectCheckDbms(object()) finally: dialect.checkBooleanExpression = orig kb.injection = saved + @staticmethod + def _bits(engine): + return dict(zip((n for n, _ in DIALECT_PROBES), _sig(engine))) + def test_identifies_mysql_on_good_channel(self): - truth = {"2=2": True, "2=3": False, DIALECT_CANARY: False, - "2^0=2": True, "2^3=8": False, "5/2=2": False, "2|0=2": True, "1<<2=4": True} - self.assertEqual(self._run(truth), DBMS.MYSQL) + self.assertEqual(self._run(self._bits("mysql")), DBMS.MYSQL) def test_identifies_postgres_on_good_channel(self): - truth = {"2=2": True, "2=3": False, DIALECT_CANARY: False, - "2^0=2": False, "2^3=8": True, "5/2=2": True, "2|0=2": True, "1<<2=4": True} - self.assertEqual(self._run(truth), DBMS.PGSQL) + self.assertEqual(self._run(self._bits("postgres")), DBMS.PGSQL) + + def test_identifies_oracle_on_good_channel(self): + self.assertEqual(self._run(self._bits("oracle")), DBMS.ORACLE) def test_none_on_blocked_channel(self): # everything blocked/false -> the tautology 2=2 reads False -> sanity fails -> None - self.assertIsNone(self._run({})) + self.assertIsNone(self._run({}, gate=(False, False, False))) def test_none_on_static_channel(self): - # a static page reads everything True, so the contradiction 2=3 is True -> sanity fails -> None - self.assertIsNone(self._run({"2=2": True, "2=3": True, DIALECT_CANARY: True, - "2^0=2": True, "2^3=8": True, "5/2=2": True, "2|0=2": True, "1<<2=4": True})) + # a static page reads everything True -> the contradiction 2=3 is True -> sanity fails -> None + self.assertIsNone(self._run(self._bits("mysql"), gate=(True, True, True))) def test_none_when_canary_reads_true(self): - # THE canary contract: a channel can look like a clean oracle (2=2 true, 2=3 false) and even - # yield a DBMS-shaped signature, but if the syntactically-invalid canary also reads TRUE the - # channel accepts garbage -> it is a false positive -> return None (true negative), never a DBMS. - truth = {"2=2": True, "2=3": False, DIALECT_CANARY: True, - "2^0=2": True, "2^3=8": False, "5/2=2": False, "2|0=2": True, "1<<2=4": True} # would be MySQL - self.assertIsNone(self._run(truth)) + # THE canary contract: a channel can look clean (2=2 true, 2=3 false) and yield a DBMS-shaped + # signature, but if the invalid canary also reads TRUE the channel accepts garbage -> None. + self.assertIsNone(self._run(self._bits("mysql"), gate=(True, False, True))) + + +class TestDialectAdversarial(unittest.TestCase): + """WAF that selectively drops operator characters: a dropped probe reads FALSE, silently degrading + the signature. Reachability canaries mark those bits unknown and _classify() abstains unless the + trusted bits are unanimous - so char-blocking can never MISCLASSIFY (only answer correctly or None).""" + + def test_guard_never_misclassifies_under_single_char_block(self): + # for every droppable probe character, no known engine is ever read as a DIFFERENT DBMS + for char, bits in _DIALECT_REACH_CHARS: + for engine, row in _ALL.items(): + expected = row[_PROBE_COUNT] + got = _classify(row[:_PROBE_COUNT], unknown=set(bits)) + self.assertIn(got, (expected, None), "%s under blocked %r -> %s" % (engine, char, got)) + + def test_guard_recovers_when_trusted_bits_are_unique(self): + # MySQL stays uniquely pinned with 'mod' (%) blocked + self.assertEqual(_classify(_sig("mysql"), unknown={2}), DBMS.MYSQL) + + def test_guard_abstains_when_ambiguous(self): + # H2 vs Presto differ only in numcat; blocking '|' (kills numcat) makes them indistinguishable + self.assertIsNone(_classify(_sig("h2"), unknown={7})) + + +class TestDialectCheckDbmsReachability(TestDialectCheckDbmsGuard): + """dialectCheckDbms() end-to-end when a WAF drops a probe character.""" + + def test_blocked_char_abstains_instead_of_misclassifying(self): + # '|' dropped: H2 can no longer be told from Presto -> None (not a wrong guess) + self.assertIsNone(self._run(self._bits("h2"), blocked=("|",))) + + def test_blocked_char_still_identifies_when_unique(self): + # '%' dropped: MySQL stays uniquely identifiable + self.assertEqual(self._run(self._bits("mysql"), blocked=("%",)), DBMS.MYSQL) if __name__ == "__main__":