From 4c12dda2f7ee631a1574cb01b01484be9241ff12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0tampar?= Date: Thu, 9 Jul 2026 13:25:50 +0200 Subject: [PATCH] Fixing --eta --- lib/core/settings.py | 5 +- lib/request/inject.py | 58 +++++++++++++++- lib/utils/progress.py | 45 +++++++++--- plugins/generic/databases.py | 15 ++-- plugins/generic/entries.py | 8 ++- tests/test_progress.py | 131 +++++++++++++++++++++++++++++++++++ tests/test_techniques.py | 42 +++++++++++ 7 files changed, 285 insertions(+), 19 deletions(-) diff --git a/lib/core/settings.py b/lib/core/settings.py index 95cd32667..04460590c 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -20,7 +20,7 @@ from lib.core.enums import OS from thirdparty import six # sqlmap version (...) -VERSION = "1.10.7.55" +VERSION = "1.10.7.56" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) @@ -249,6 +249,9 @@ THREAD_FINALIZATION_TIMEOUT = 1 # Maximum number of techniques used in inject.py/getValue() per one value MAX_TECHNIQUES_PER_VALUE = 2 +# Fraction of the currently displayed progress-bar ETA kept when a fresh estimate arrives (eases the on-screen countdown toward the new value instead of snapping); 0 disables smoothing, higher is smoother but laggier +ETA_DISPLAY_SMOOTHING = 0.5 + # In case of missing piece of partial union dump, buffered array must be flushed after certain size MAX_BUFFERED_PARTIAL_UNION_LENGTH = 1024 diff --git a/lib/request/inject.py b/lib/request/inject.py index 731a47b0c..8948a07cc 100644 --- a/lib/request/inject.py +++ b/lib/request/inject.py @@ -8,6 +8,7 @@ See the file 'LICENSE' for copying permission from __future__ import print_function import re +import threading import time from lib.core.agent import agent @@ -16,6 +17,7 @@ from lib.core.common import applyFunctionRecursively from lib.core.common import dataToStdout from lib.core.common import unArrayizeValue from lib.core.datatype import AttribDict +from lib.utils.progress import ProgressBar from lib.utils.safe2bin import safecharencode from lib.core.common import Backend from lib.core.common import calculateDeltaSeconds @@ -58,12 +60,14 @@ from lib.core.exception import SqlmapDataException from lib.core.exception import SqlmapNotVulnerableException from lib.core.exception import SqlmapUserQuitException from lib.core.settings import GET_VALUE_UPPERCASE_KEYWORDS +from lib.core.settings import IS_TTY from lib.core.settings import INFERENCE_MARKER from lib.core.settings import MAX_TECHNIQUES_PER_VALUE from lib.core.settings import SQL_SCALAR_REGEX from lib.core.settings import UNICODE_ENCODING from lib.core.threads import getCurrentThreadData from lib.core.threads import runThreads +from lib.core.threads import setDaemon from lib.core.unescaper import unescaper from lib.request.connect import Connect as Request from lib.request.direct import direct @@ -405,6 +409,25 @@ def _verifyInferredValue(expression, value): except Exception: return True +def valueParallelEligible(): + """ + Whether blind enumeration/dumping should take the value-parallel path (one whole value per worker) + rather than the classic char loop. It is chosen for concurrency ('--threads') and, independently, to + render a single whole-job progress bar/ETA ('--eta'). A concurrency-safe channel - boolean or the + HTTP/2 timeless oracle - qualifies with either. Classic time-based qualifies only single-threaded under + '--eta': concurrent SLEEP measurements interfere, so it must stay sequential (where it is safe, and + still gets the whole-job ETA that matters most for the slowest channel). Callers add their own extra + guards (e.g. '--dns-domain', per-column comments). + """ + + concurrencySafe = isTechniqueAvailable(PAYLOAD.TECHNIQUE.BOOLEAN) or kb.get("timeless") is not None + + if conf.threads > 1: + return concurrencySafe + if conf.eta: + return concurrencySafe or isTechniqueAvailable(PAYLOAD.TECHNIQUE.TIME) + return False + def _threadedInferenceValues(exprBuilder, indices, context=None, charsetType=None, dump=False): """ Value-parallel blind retrieval. @@ -446,6 +469,13 @@ def _threadedInferenceValues(exprBuilder, indices, context=None, charsetType=Non results = [None] * len(indices) cursor = iter(xrange(len(indices))) + # With '--eta' show a single value-level bar (values completed / total) instead of the per-value + # stream: it is monotonic and carries a meaningful ETA across the whole set, and - unlike the + # per-char bar drawn inside bisection() - it survives the worker's disableStdOut (drawn forced, + # below), so '--eta' is honoured under '--threads' too. Skipped for trivial single-value sets. + etaProgress = ProgressBar(maxValue=len(indices)) if (conf.eta and len(indices) > 1) else None + completed = [0] + def inferenceThread(): threadData = getCurrentThreadData() # Each per-value bisection streams its characters to stdout and mirrors them into @@ -489,13 +519,36 @@ def _threadedInferenceValues(exprBuilder, indices, context=None, charsetType=Non with kb.locks.value: results[slot] = value + if etaProgress is not None: + with kb.locks.io: + completed[0] += 1 + threadData.disableStdOut = False # let the value-level bar through (per-char streaming stays muted) + try: + etaProgress.progress(completed[0]) + finally: + threadData.disableStdOut = True + # Stream each retrieved value as it completes (they arrive out of order under threads, exactly # like the error/union dumps), so a dump shows its data live rather than a silent counter. - if conf.verbose >= 1 and not kb.bruteMode and not isNoneValue(value): + elif conf.verbose >= 1 and not kb.bruteMode and not isNoneValue(value): with kb.locks.io: rendered = safecharencode(unArrayizeValue(value)) dataToStdout("[%s] [INFO] retrieved: %s\n" % (time.strftime("%X"), "'%s'" % rendered if dump else rendered), forceOutput=True) + # Keep the '--eta' countdown live between value updates: a value (esp. time-based) can take many + # seconds, so a daemon redraws the bar every second with the ETA decremented by elapsed time (it runs + # on its own thread, so it is not muted by the workers' disableStdOut, and shares kb.locks.io with them). + etaTickerStop = threading.Event() + etaTicker = None + if etaProgress is not None and IS_TTY: + def _etaTicker(): + while not etaTickerStop.wait(1.0): + with kb.locks.io: + etaProgress.tick() + etaTicker = threading.Thread(target=_etaTicker, name="eta-ticker") + setDaemon(etaTicker) + etaTicker.start() + # Save/restore the calling thread's state: with a single thread runThreads runs the worker # INLINE on this thread, so the worker's disableStdOut/shared mutations must not leak out. savedPartRun = kb.partRun @@ -505,6 +558,9 @@ def _threadedInferenceValues(exprBuilder, indices, context=None, charsetType=Non try: runThreads(min(conf.threads or 1, len(indices)) or 1, inferenceThread) finally: + etaTickerStop.set() + if etaTicker is not None: + etaTicker.join(timeout=2) kb.partRun = savedPartRun mainThreadData.disableStdOut = savedStdOut mainThreadData.shared = savedShared diff --git a/lib/utils/progress.py b/lib/utils/progress.py index 1bfb10656..97e58a9bb 100644 --- a/lib/utils/progress.py +++ b/lib/utils/progress.py @@ -13,6 +13,8 @@ from lib.core.common import dataToStdout from lib.core.convert import getUnicode from lib.core.data import conf from lib.core.data import kb +from lib.core.settings import ETA_DISPLAY_SMOOTHING +from lib.core.settings import IS_TTY class ProgressBar(object): """ @@ -26,7 +28,9 @@ class ProgressBar(object): self._span = max(self._max - self._min, 0.001) self._width = totalWidth if totalWidth else conf.progressWidth self._amount = 0 - self._start = None + self._start = time.time() # begin timing at construction, so the first completed item already yields an estimate + self._eta = None # last estimated seconds-remaining and when it was computed, so tick() + self._etaAt = None # can keep the countdown live between (possibly slow) item updates self.update() def _convertSeconds(self, value): @@ -73,17 +77,39 @@ class ProgressBar(object): def progress(self, newAmount): """ - This method saves item delta time and shows updated progress bar with calculated eta + Redraw the bar with an ETA from the average time per completed item so far, applied to the items + still remaining: (elapsed / done) * (max - newAmount). The remaining-item count is (max - newAmount) + - i.e. at 1/3 it estimates the 2 items left, at 2/3 the 1 left - not just the current item. """ - if self._start is None or newAmount > self._max: - self._start = time.time() - eta = None - else: - delta = time.time() - self._start - eta = (self._max - self._min) * (1.0 * delta / newAmount) - delta + now = time.time() + if newAmount > self._max: # counter rollover/reset -> restart timing + self._start = now + self._eta = None + done = newAmount - self._min + elapsed = now - self._start + target = (elapsed / done) * (self._max - newAmount) if (done > 0 and elapsed > 0) else None + + if target is None: + self._eta = None + elif self._eta is None: + self._eta = target # first estimate: nothing to ease from + else: + current = max(0, self._eta - (now - self._etaAt)) # what is on screen now (already decremented by tick()) + self._eta = ETA_DISPLAY_SMOOTHING * current + (1 - ETA_DISPLAY_SMOOTHING) * target # ease into the fresh estimate + + self._etaAt = now self.update(newAmount) + self.draw(self._eta) + + def tick(self): + """ + Redraw the current bar with its ETA decremented by real elapsed time, so the countdown stays + live between updates (e.g. during a long time-based wait) instead of freezing at the last estimate + """ + + eta = None if self._eta is None else max(0, self._eta - (time.time() - self._etaAt)) self.draw(eta) def draw(self, eta=None): @@ -91,6 +117,9 @@ class ProgressBar(object): This method draws the progress bar if it has changed """ + if not IS_TTY: # a progress bar is a terminal animation; suppress it when piped/redirected (as done for other '\r' output) + return + dataToStdout("\r%s %d/%d%s" % (self._progBar, self._amount, self._max, (" (ETA %s)" % (self._convertSeconds(int(eta)) if eta is not None else "??:??")))) if self._amount >= self._max: dataToStdout("\r%s\r" % (" " * self._width)) diff --git a/plugins/generic/databases.py b/plugins/generic/databases.py index c439863b9..70f210b15 100644 --- a/plugins/generic/databases.py +++ b/plugins/generic/databases.py @@ -403,12 +403,13 @@ class Databases(object): # Value-parallel, prediction-assisted name enumeration for the DBMSes using the # generic ", " blind template. Retrieves whole names concurrently (one per - # worker, each decoded sequentially so wordlist prediction applies). Used only with - # '--threads' (like getColumns below); single-thread stays on the classic getValue loop, - # which still gets predictive inference via getPartRun. The special templates stay serial. + # worker, each decoded sequentially so wordlist prediction applies). Used with '--threads' + # and under '--eta' (one whole-job bar/ETA) per valueParallelEligible(); plain single-thread + # stays on the classic getValue loop, which still gets predictive inference via getPartRun. + # The special templates stay serial. genericTemplate = Backend.getIdentifiedDbms() not in (DBMS.SYBASE, DBMS.MAXDB, DBMS.ACCESS, DBMS.MCKOI, DBMS.EXTREMEDB, DBMS.SQLITE, DBMS.FIREBIRD, DBMS.HSQLDB, DBMS.INFORMIX, DBMS.FRONTBASE, DBMS.VIRTUOSO, DBMS.SPANNER) - if genericTemplate and conf.threads > 1 and isTechniqueAvailable(PAYLOAD.TECHNIQUE.BOOLEAN): + if genericTemplate and inject.valueParallelEligible(): for table in (inject._threadedInferenceValues(lambda index: _query % (unsafeSQLIdentificatorNaming(db), index), indexRange, context="Tables") or []): if not isNoneValue(table): kb.hintValue = table @@ -899,9 +900,11 @@ class Databases(object): # Value-parallel column-NAME enumeration: the same axis/mechanism as getTables (one name per # worker, decoded sequentially - no length probe, predictive inference applies, names stream - # live). Serial fallback for single-thread and when also fetching per-column comments. + # live, and under '--eta' a single whole-job bar/ETA is drawn). Eligibility is shared via + # valueParallelEligible(); serial fallback only when also fetching per-column comments (those + # need an extra per-column query). columnNames = None - if conf.threads > 1 and not conf.getComments and isTechniqueAvailable(PAYLOAD.TECHNIQUE.BOOLEAN): + if not conf.getComments and inject.valueParallelEligible(): columnNames = inject._threadedInferenceValues(columnNameQuery, indexList, context="Columns") for position, index in enumerate(indexList): diff --git a/plugins/generic/entries.py b/plugins/generic/entries.py index 568a0eb53..4e8ef5a7d 100644 --- a/plugins/generic/entries.py +++ b/plugins/generic/entries.py @@ -444,9 +444,11 @@ class Entries(object): # value's characters across threads) and the per-column Huffman model + low-cardinality # guessing engage under concurrency. Used for the boolean channel with '--threads', and # for the HTTP/2 timeless oracle (its per-thread connections make concurrency safe and - # deterministic - unlike classic time-based, whose char-parallel loop interleaves its - # per-thread output). Single-thread and classic time-based keep the char-parallel loop. - if conf.threads > 1 and not conf.dnsDomain and (isTechniqueAvailable(PAYLOAD.TECHNIQUE.BOOLEAN) or kb.get("timeless") is not None): + # deterministic). Also used under '--eta' - including single-threaded time-based, the + # slowest channel - to drive one whole-job progress bar/ETA (how long the dump takes) + # instead of a per-cell counter. Eligibility (channel x threads/eta) is valueParallelEligible(); + # '--dns-domain' keeps the classic loop (its OOB fast path bypasses bisection). + if not conf.dnsDomain and inject.valueParallelEligible(): # One value-parallel pass over every (non-empty) cell, so there is a single # thread pool and values stream live as they complete - out of order, exactly # like the error/union dumps - instead of a silent progress counter. diff --git a/tests/test_progress.py b/tests/test_progress.py index dbc007f01..5690763d1 100644 --- a/tests/test_progress.py +++ b/tests/test_progress.py @@ -56,7 +56,9 @@ class TestProgressBar(unittest.TestCase): def test_progress_draws_eta_after_second_call(self): captured = [] real = progress_mod.dataToStdout + realTty = progress_mod.IS_TTY progress_mod.dataToStdout = lambda data, *a, **k: captured.append(data) + progress_mod.IS_TTY = True # draw() only animates on a terminal try: pb = ProgressBar(0, 10, 78) pb.progress(0) # first call only seeds the timer (eta None) @@ -64,6 +66,7 @@ class TestProgressBar(unittest.TestCase): pb.progress(5) # second call computes and draws a real ETA finally: progress_mod.dataToStdout = real + progress_mod.IS_TTY = realTty self.assertTrue(captured, msg="progress() never wrote to stdout") last = captured[-1] @@ -73,6 +76,134 @@ class TestProgressBar(unittest.TestCase): self.assertTrue(re.search(r"\(ETA \d{2}:\d{2}\)", last), msg="ETA token missing an mm:ss timer: %r" % last) + def test_eta_available_from_first_completed_item(self): + captured = [] + real = progress_mod.dataToStdout + realTty = progress_mod.IS_TTY + progress_mod.dataToStdout = lambda data, *a, **k: captured.append(data) + progress_mod.IS_TTY = True + try: + pb = ProgressBar(0, 5, 78) + pb._start = pb._lastTime = time.time() - 2.0 # one item took ~2s -> estimate must appear now, at 1/5 + pb.progress(1) + finally: + progress_mod.dataToStdout = real + progress_mod.IS_TTY = realTty + + last = captured[-1] + self.assertNotIn("??:??", last, msg="no ETA at the first item: %r" % last) + m = re.search(r"\(ETA (\d{2}):(\d{2})\)", last) + secs = int(m.group(1)) * 60 + int(m.group(2)) + self.assertTrue(7 <= secs <= 9, msg="ETA not ~8s (2s/item x 4 remaining): %r" % last) # not the 2x-optimistic ~4s + + def test_eta_reflects_remaining_item_count(self): + # at a constant 10s/item pace: 1/3 must estimate the 2 items left (~20s), 2/3 the 1 left (~10s) - + # i.e. (max - done) items, not just the current one. Fresh bars so each is a first (unsmoothed) estimate. + captured = [] + real = progress_mod.dataToStdout + realTty = progress_mod.IS_TTY + progress_mod.dataToStdout = lambda data, *a, **k: captured.append(data) + progress_mod.IS_TTY = True + + def drawnEta(): + m = re.search(r"\(ETA (\d{2}):(\d{2})\)", captured[-1]) + self.assertTrue(m, msg="no ETA drawn: %r" % captured[-1]) + return int(m.group(1)) * 60 + int(m.group(2)) + + try: + a = ProgressBar(0, 3, 78) + a._start = time.time() - 10.0 # 1 done in 10s -> 10s/item, 2 remaining -> ~20s + a.progress(1) + eta1 = drawnEta() + + b = ProgressBar(0, 3, 78) + b._start = time.time() - 20.0 # 2 done in 20s -> 10s/item, 1 remaining -> ~10s + b.progress(2) + eta2 = drawnEta() + finally: + progress_mod.dataToStdout = real + progress_mod.IS_TTY = realTty + + self.assertTrue(18 <= eta1 <= 22, msg="1/3 should estimate 2 remaining (~20s): %ds" % eta1) + self.assertTrue(8 <= eta2 <= 12, msg="2/3 should estimate 1 remaining (~10s): %ds" % eta2) + + def test_new_estimate_is_eased_not_snapped(self): + # when a fresh estimate is far from the value currently on screen, the drawn ETA must land + # between the two (smoothed), not snap straight to the new target + captured = [] + real = progress_mod.dataToStdout + realTty = progress_mod.IS_TTY + progress_mod.dataToStdout = lambda data, *a, **k: captured.append(data) + progress_mod.IS_TTY = True + try: + pb = ProgressBar(0, 10, 78) + pb._eta = 8.0 # currently showing ~8s... + pb._etaAt = time.time() + pb._start = time.time() - 2.0 # ...but the fresh target is (2/1)*(10-1) = 18s + pb.progress(1) + finally: + progress_mod.dataToStdout = real + progress_mod.IS_TTY = realTty + + m = re.search(r"\(ETA (\d{2}):(\d{2})\)", captured[-1]) + secs = int(m.group(1)) * 60 + int(m.group(2)) + self.assertTrue(10 <= secs <= 16, msg="ETA snapped instead of easing between 8 and 18: %ds" % secs) + + def test_tick_counts_down_from_stored_eta(self): + captured = [] + real = progress_mod.dataToStdout + realTty = progress_mod.IS_TTY + progress_mod.dataToStdout = lambda data, *a, **k: captured.append(data) + progress_mod.IS_TTY = True + try: + pb = ProgressBar(0, 10, 78) + pb.update(5) + pb._eta = 100 # a 100s estimate... + pb._etaAt = time.time() - 30 # ...taken 30s ago -> tick() must show ~70s + pb.tick() + finally: + progress_mod.dataToStdout = real + progress_mod.IS_TTY = realTty + + m = re.search(r"\(ETA (\d{2}):(\d{2})\)", captured[-1]) + self.assertTrue(m, msg="no mm:ss ETA drawn: %r" % captured[-1]) + secs = int(m.group(1)) * 60 + int(m.group(2)) + self.assertTrue(66 <= secs <= 71, msg="ETA not decremented to ~70s: %r" % captured[-1]) + + def test_tick_clamps_at_zero_when_overdue(self): + captured = [] + real = progress_mod.dataToStdout + realTty = progress_mod.IS_TTY + progress_mod.dataToStdout = lambda data, *a, **k: captured.append(data) + progress_mod.IS_TTY = True + try: + pb = ProgressBar(0, 10, 78) + pb.update(5) + pb._eta = 5 + pb._etaAt = time.time() - 60 # long overdue -> must clamp to 00:00, never negative + pb.tick() + finally: + progress_mod.dataToStdout = real + progress_mod.IS_TTY = realTty + + self.assertIn("(ETA 00:00)", captured[-1], msg=captured[-1]) + + def test_no_draw_when_not_tty(self): + captured = [] + real = progress_mod.dataToStdout + realTty = progress_mod.IS_TTY + progress_mod.dataToStdout = lambda data, *a, **k: captured.append(data) + progress_mod.IS_TTY = False # piped/redirected: no animated bar should reach the stream + try: + pb = ProgressBar(0, 10, 78) + for i in range(1, 11): + pb.progress(i) + finally: + progress_mod.dataToStdout = real + progress_mod.IS_TTY = realTty + + self.assertEqual(captured, [], msg="progress bar leaked to a non-TTY stream: %r" % captured) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_techniques.py b/tests/test_techniques.py index 6ab50de7e..239bc33f8 100644 --- a/tests/test_techniques.py +++ b/tests/test_techniques.py @@ -48,6 +48,7 @@ from lib.core.agent import agent from lib.core.unescaper import unescaper from lib.request.connect import Connect from lib.request.connect import Connect as Request +from lib.request import inject from lib.utils.hashdb import HashDB import lib.techniques.union.use as uu @@ -1516,6 +1517,47 @@ class TestConfigUnion(unittest.TestCase): self.assertEqual(kb.uChar, "SENTINEL") +class TestValueParallelEligibility(unittest.TestCase): + """ + inject.valueParallelEligible() picks the value-parallel path (job-level '--eta' bar / concurrency). + Safety invariant under test: classic time-based must never run concurrently (interfering SLEEP + measurements), so it qualifies only single-threaded under '--eta'; a concurrency-safe channel + (boolean or the timeless oracle) may run under either '--threads' or '--eta'. + """ + + def setUp(self): + self._avail = set() + self._realAvail = inject.isTechniqueAvailable + inject.isTechniqueAvailable = lambda t: t in self._avail + self._saved = (conf.threads, conf.eta, kb.get("timeless")) + + def tearDown(self): + inject.isTechniqueAvailable = self._realAvail + conf.threads, conf.eta, kb.timeless = self._saved + + def _elig(self, threads, eta, techniques, timeless=None): + conf.threads, conf.eta, kb.timeless = threads, eta, timeless + self._avail = set(techniques) + return inject.valueParallelEligible() + + def test_single_thread_eta_time_based_qualifies(self): + self.assertTrue(self._elig(1, True, {PAYLOAD.TECHNIQUE.TIME})) + + def test_multi_thread_time_based_never_parallel(self): + self.assertFalse(self._elig(8, True, {PAYLOAD.TECHNIQUE.TIME})) + self.assertFalse(self._elig(8, False, {PAYLOAD.TECHNIQUE.TIME})) + + def test_boolean_qualifies_under_threads_or_eta(self): + self.assertTrue(self._elig(8, False, {PAYLOAD.TECHNIQUE.BOOLEAN})) + self.assertTrue(self._elig(1, True, {PAYLOAD.TECHNIQUE.BOOLEAN})) + + def test_plain_single_thread_no_eta_stays_classic(self): + self.assertFalse(self._elig(1, False, {PAYLOAD.TECHNIQUE.BOOLEAN})) + + def test_timeless_is_concurrency_safe(self): + self.assertTrue(self._elig(8, True, {PAYLOAD.TECHNIQUE.TIME}, timeless=object())) + + if __name__ == "__main__": unittest.main(verbosity=2)