Fixing --eta

lib/core/settings.py

@@ -20,7 +20,7 @@ from lib.core.enums import OS
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.7.55"
+VERSION = "1.10.7.56"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -249,6 +249,9 @@ THREAD_FINALIZATION_TIMEOUT = 1
 # Maximum number of techniques used in inject.py/getValue() per one value
 MAX_TECHNIQUES_PER_VALUE = 2
 
+# Fraction of the currently displayed progress-bar ETA kept when a fresh estimate arrives (eases the on-screen countdown toward the new value instead of snapping); 0 disables smoothing, higher is smoother but laggier
+ETA_DISPLAY_SMOOTHING = 0.5
+
 # In case of missing piece of partial union dump, buffered array must be flushed after certain size
 MAX_BUFFERED_PARTIAL_UNION_LENGTH = 1024
 

lib/request/inject.py

@@ -8,6 +8,7 @@ See the file 'LICENSE' for copying permission
 from __future__ import print_function
 
 import re
+import threading
 import time
 
 from lib.core.agent import agent
@@ -16,6 +17,7 @@ from lib.core.common import applyFunctionRecursively
 from lib.core.common import dataToStdout
 from lib.core.common import unArrayizeValue
 from lib.core.datatype import AttribDict
+from lib.utils.progress import ProgressBar
 from lib.utils.safe2bin import safecharencode
 from lib.core.common import Backend
 from lib.core.common import calculateDeltaSeconds
@@ -58,12 +60,14 @@ from lib.core.exception import SqlmapDataException
 from lib.core.exception import SqlmapNotVulnerableException
 from lib.core.exception import SqlmapUserQuitException
 from lib.core.settings import GET_VALUE_UPPERCASE_KEYWORDS
+from lib.core.settings import IS_TTY
 from lib.core.settings import INFERENCE_MARKER
 from lib.core.settings import MAX_TECHNIQUES_PER_VALUE
 from lib.core.settings import SQL_SCALAR_REGEX
 from lib.core.settings import UNICODE_ENCODING
 from lib.core.threads import getCurrentThreadData
 from lib.core.threads import runThreads
+from lib.core.threads import setDaemon
 from lib.core.unescaper import unescaper
 from lib.request.connect import Connect as Request
 from lib.request.direct import direct
@@ -405,6 +409,25 @@ def _verifyInferredValue(expression, value):
     except Exception:
         return True
 
+def valueParallelEligible():
+    """
+    Whether blind enumeration/dumping should take the value-parallel path (one whole value per worker)
+    rather than the classic char loop. It is chosen for concurrency ('--threads') and, independently, to
+    render a single whole-job progress bar/ETA ('--eta'). A concurrency-safe channel - boolean or the
+    HTTP/2 timeless oracle - qualifies with either. Classic time-based qualifies only single-threaded under
+    '--eta': concurrent SLEEP measurements interfere, so it must stay sequential (where it is safe, and
+    still gets the whole-job ETA that matters most for the slowest channel). Callers add their own extra
+    guards (e.g. '--dns-domain', per-column comments).
+    """
+
+    concurrencySafe = isTechniqueAvailable(PAYLOAD.TECHNIQUE.BOOLEAN) or kb.get("timeless") is not None
+
+    if conf.threads > 1:
+        return concurrencySafe
+    if conf.eta:
+        return concurrencySafe or isTechniqueAvailable(PAYLOAD.TECHNIQUE.TIME)
+    return False
+
 def _threadedInferenceValues(exprBuilder, indices, context=None, charsetType=None, dump=False):
     """
     Value-parallel blind retrieval.
@@ -446,6 +469,13 @@ def _threadedInferenceValues(exprBuilder, indices, context=None, charsetType=Non
     results = [None] * len(indices)
     cursor = iter(xrange(len(indices)))
 
+    # With '--eta' show a single value-level bar (values completed / total) instead of the per-value
+    # stream: it is monotonic and carries a meaningful ETA across the whole set, and - unlike the
+    # per-char bar drawn inside bisection() - it survives the worker's disableStdOut (drawn forced,
+    # below), so '--eta' is honoured under '--threads' too. Skipped for trivial single-value sets.
+    etaProgress = ProgressBar(maxValue=len(indices)) if (conf.eta and len(indices) > 1) else None
+    completed = [0]
+
     def inferenceThread():
         threadData = getCurrentThreadData()
         # Each per-value bisection streams its characters to stdout and mirrors them into
@@ -489,13 +519,36 @@ def _threadedInferenceValues(exprBuilder, indices, context=None, charsetType=Non
             with kb.locks.value:
                 results[slot] = value
 
+            if etaProgress is not None:
+                with kb.locks.io:
+                    completed[0] += 1
+                    threadData.disableStdOut = False        # let the value-level bar through (per-char streaming stays muted)
+                    try:
+                        etaProgress.progress(completed[0])
+                    finally:
+                        threadData.disableStdOut = True
+
             # Stream each retrieved value as it completes (they arrive out of order under threads, exactly
             # like the error/union dumps), so a dump shows its data live rather than a silent counter.
-            if conf.verbose >= 1 and not kb.bruteMode and not isNoneValue(value):
+            elif conf.verbose >= 1 and not kb.bruteMode and not isNoneValue(value):
                 with kb.locks.io:
                     rendered = safecharencode(unArrayizeValue(value))
                     dataToStdout("[%s] [INFO] retrieved: %s\n" % (time.strftime("%X"), "'%s'" % rendered if dump else rendered), forceOutput=True)
 
+    # Keep the '--eta' countdown live between value updates: a value (esp. time-based) can take many
+    # seconds, so a daemon redraws the bar every second with the ETA decremented by elapsed time (it runs
+    # on its own thread, so it is not muted by the workers' disableStdOut, and shares kb.locks.io with them).
+    etaTickerStop = threading.Event()
+    etaTicker = None
+    if etaProgress is not None and IS_TTY:
+        def _etaTicker():
+            while not etaTickerStop.wait(1.0):
+                with kb.locks.io:
+                    etaProgress.tick()
+        etaTicker = threading.Thread(target=_etaTicker, name="eta-ticker")
+        setDaemon(etaTicker)
+        etaTicker.start()
+
     # Save/restore the calling thread's state: with a single thread runThreads runs the worker
     # INLINE on this thread, so the worker's disableStdOut/shared mutations must not leak out.
     savedPartRun = kb.partRun
@@ -505,6 +558,9 @@ def _threadedInferenceValues(exprBuilder, indices, context=None, charsetType=Non
     try:
         runThreads(min(conf.threads or 1, len(indices)) or 1, inferenceThread)
     finally:
+        etaTickerStop.set()
+        if etaTicker is not None:
+            etaTicker.join(timeout=2)
         kb.partRun = savedPartRun
         mainThreadData.disableStdOut = savedStdOut
         mainThreadData.shared = savedShared

lib/utils/progress.py

@@ -13,6 +13,8 @@ from lib.core.common import dataToStdout
 from lib.core.convert import getUnicode
 from lib.core.data import conf
 from lib.core.data import kb
+from lib.core.settings import ETA_DISPLAY_SMOOTHING
+from lib.core.settings import IS_TTY
 
 class ProgressBar(object):
     """
@@ -26,7 +28,9 @@ class ProgressBar(object):
         self._span = max(self._max - self._min, 0.001)
         self._width = totalWidth if totalWidth else conf.progressWidth
         self._amount = 0
-        self._start = None
+        self._start = time.time()   # begin timing at construction, so the first completed item already yields an estimate
+        self._eta = None            # last estimated seconds-remaining and when it was computed, so tick()
+        self._etaAt = None          # can keep the countdown live between (possibly slow) item updates
         self.update()
 
     def _convertSeconds(self, value):
@@ -73,17 +77,39 @@ class ProgressBar(object):
 
     def progress(self, newAmount):
         """
-        This method saves item delta time and shows updated progress bar with calculated eta
+        Redraw the bar with an ETA from the average time per completed item so far, applied to the items
+        still remaining: (elapsed / done) * (max - newAmount). The remaining-item count is (max - newAmount)
+        - i.e. at 1/3 it estimates the 2 items left, at 2/3 the 1 left - not just the current item.
         """
 
-        if self._start is None or newAmount > self._max:
-            self._start = time.time()
-            eta = None
-        else:
-            delta = time.time() - self._start
-            eta = (self._max - self._min) * (1.0 * delta / newAmount) - delta
+        now = time.time()
+        if newAmount > self._max:               # counter rollover/reset -> restart timing
+            self._start = now
+            self._eta = None
 
+        done = newAmount - self._min
+        elapsed = now - self._start
+        target = (elapsed / done) * (self._max - newAmount) if (done > 0 and elapsed > 0) else None
+
+        if target is None:
+            self._eta = None
+        elif self._eta is None:
+            self._eta = target                  # first estimate: nothing to ease from
+        else:
+            current = max(0, self._eta - (now - self._etaAt))   # what is on screen now (already decremented by tick())
+            self._eta = ETA_DISPLAY_SMOOTHING * current + (1 - ETA_DISPLAY_SMOOTHING) * target   # ease into the fresh estimate
+
+        self._etaAt = now
         self.update(newAmount)
+        self.draw(self._eta)
+
+    def tick(self):
+        """
+        Redraw the current bar with its ETA decremented by real elapsed time, so the countdown stays
+        live between updates (e.g. during a long time-based wait) instead of freezing at the last estimate
+        """
+
+        eta = None if self._eta is None else max(0, self._eta - (time.time() - self._etaAt))
         self.draw(eta)
 
     def draw(self, eta=None):
@@ -91,6 +117,9 @@ class ProgressBar(object):
         This method draws the progress bar if it has changed
         """
 
+        if not IS_TTY:      # a progress bar is a terminal animation; suppress it when piped/redirected (as done for other '\r' output)
+            return
+
         dataToStdout("\r%s %d/%d%s" % (self._progBar, self._amount, self._max, ("  (ETA %s)" % (self._convertSeconds(int(eta)) if eta is not None else "??:??"))))
         if self._amount >= self._max:
             dataToStdout("\r%s\r" % (" " * self._width))

plugins/generic/databases.py

@@ -403,12 +403,13 @@ class Databases(object):
 
                     # Value-parallel, prediction-assisted name enumeration for the DBMSes using the
                     # generic "<db>, <index>" blind template. Retrieves whole names concurrently (one per
-                    # worker, each decoded sequentially so wordlist prediction applies). Used only with
-                    # '--threads' (like getColumns below); single-thread stays on the classic getValue loop,
-                    # which still gets predictive inference via getPartRun. The special templates stay serial.
+                    # worker, each decoded sequentially so wordlist prediction applies). Used with '--threads'
+                    # and under '--eta' (one whole-job bar/ETA) per valueParallelEligible(); plain single-thread
+                    # stays on the classic getValue loop, which still gets predictive inference via getPartRun.
+                    # The special templates stay serial.
                     genericTemplate = Backend.getIdentifiedDbms() not in (DBMS.SYBASE, DBMS.MAXDB, DBMS.ACCESS, DBMS.MCKOI, DBMS.EXTREMEDB, DBMS.SQLITE, DBMS.FIREBIRD, DBMS.HSQLDB, DBMS.INFORMIX, DBMS.FRONTBASE, DBMS.VIRTUOSO, DBMS.SPANNER)
 
-                    if genericTemplate and conf.threads > 1 and isTechniqueAvailable(PAYLOAD.TECHNIQUE.BOOLEAN):
+                    if genericTemplate and inject.valueParallelEligible():
                         for table in (inject._threadedInferenceValues(lambda index: _query % (unsafeSQLIdentificatorNaming(db), index), indexRange, context="Tables") or []):
                             if not isNoneValue(table):
                                 kb.hintValue = table
@@ -899,9 +900,11 @@ class Databases(object):
 
                 # Value-parallel column-NAME enumeration: the same axis/mechanism as getTables (one name per
                 # worker, decoded sequentially - no length probe, predictive inference applies, names stream
-                # live). Serial fallback for single-thread and when also fetching per-column comments.
+                # live, and under '--eta' a single whole-job bar/ETA is drawn). Eligibility is shared via
+                # valueParallelEligible(); serial fallback only when also fetching per-column comments (those
+                # need an extra per-column query).
                 columnNames = None
-                if conf.threads > 1 and not conf.getComments and isTechniqueAvailable(PAYLOAD.TECHNIQUE.BOOLEAN):
+                if not conf.getComments and inject.valueParallelEligible():
                     columnNames = inject._threadedInferenceValues(columnNameQuery, indexList, context="Columns")
 
                 for position, index in enumerate(indexList):

plugins/generic/entries.py

@@ -444,9 +444,11 @@ class Entries(object):
                             # value's characters across threads) and the per-column Huffman model + low-cardinality
                             # guessing engage under concurrency. Used for the boolean channel with '--threads', and
                             # for the HTTP/2 timeless oracle (its per-thread connections make concurrency safe and
-                            # deterministic - unlike classic time-based, whose char-parallel loop interleaves its
-                            # per-thread output). Single-thread and classic time-based keep the char-parallel loop.
-                            if conf.threads > 1 and not conf.dnsDomain and (isTechniqueAvailable(PAYLOAD.TECHNIQUE.BOOLEAN) or kb.get("timeless") is not None):
+                            # deterministic). Also used under '--eta' - including single-threaded time-based, the
+                            # slowest channel - to drive one whole-job progress bar/ETA (how long the dump takes)
+                            # instead of a per-cell counter. Eligibility (channel x threads/eta) is valueParallelEligible();
+                            # '--dns-domain' keeps the classic loop (its OOB fast path bypasses bisection).
+                            if not conf.dnsDomain and inject.valueParallelEligible():
                                 # One value-parallel pass over every (non-empty) cell, so there is a single
                                 # thread pool and values stream live as they complete - out of order, exactly
                                 # like the error/union dumps - instead of a silent progress counter.

tests/test_progress.py

@@ -56,7 +56,9 @@ class TestProgressBar(unittest.TestCase):
     def test_progress_draws_eta_after_second_call(self):
         captured = []
         real = progress_mod.dataToStdout
+        realTty = progress_mod.IS_TTY
         progress_mod.dataToStdout = lambda data, *a, **k: captured.append(data)
+        progress_mod.IS_TTY = True    # draw() only animates on a terminal
         try:
             pb = ProgressBar(0, 10, 78)
             pb.progress(0)               # first call only seeds the timer (eta None)
@@ -64,6 +66,7 @@ class TestProgressBar(unittest.TestCase):
             pb.progress(5)               # second call computes and draws a real ETA
         finally:
             progress_mod.dataToStdout = real
+            progress_mod.IS_TTY = realTty
 
         self.assertTrue(captured, msg="progress() never wrote to stdout")
         last = captured[-1]
@@ -73,6 +76,134 @@ class TestProgressBar(unittest.TestCase):
         self.assertTrue(re.search(r"\(ETA \d{2}:\d{2}\)", last),
                         msg="ETA token missing an mm:ss timer: %r" % last)
 
+    def test_eta_available_from_first_completed_item(self):
+        captured = []
+        real = progress_mod.dataToStdout
+        realTty = progress_mod.IS_TTY
+        progress_mod.dataToStdout = lambda data, *a, **k: captured.append(data)
+        progress_mod.IS_TTY = True
+        try:
+            pb = ProgressBar(0, 5, 78)
+            pb._start = pb._lastTime = time.time() - 2.0   # one item took ~2s -> estimate must appear now, at 1/5
+            pb.progress(1)
+        finally:
+            progress_mod.dataToStdout = real
+            progress_mod.IS_TTY = realTty
+
+        last = captured[-1]
+        self.assertNotIn("??:??", last, msg="no ETA at the first item: %r" % last)
+        m = re.search(r"\(ETA (\d{2}):(\d{2})\)", last)
+        secs = int(m.group(1)) * 60 + int(m.group(2))
+        self.assertTrue(7 <= secs <= 9, msg="ETA not ~8s (2s/item x 4 remaining): %r" % last)   # not the 2x-optimistic ~4s
+
+    def test_eta_reflects_remaining_item_count(self):
+        # at a constant 10s/item pace: 1/3 must estimate the 2 items left (~20s), 2/3 the 1 left (~10s) -
+        # i.e. (max - done) items, not just the current one. Fresh bars so each is a first (unsmoothed) estimate.
+        captured = []
+        real = progress_mod.dataToStdout
+        realTty = progress_mod.IS_TTY
+        progress_mod.dataToStdout = lambda data, *a, **k: captured.append(data)
+        progress_mod.IS_TTY = True
+
+        def drawnEta():
+            m = re.search(r"\(ETA (\d{2}):(\d{2})\)", captured[-1])
+            self.assertTrue(m, msg="no ETA drawn: %r" % captured[-1])
+            return int(m.group(1)) * 60 + int(m.group(2))
+
+        try:
+            a = ProgressBar(0, 3, 78)
+            a._start = time.time() - 10.0      # 1 done in 10s -> 10s/item, 2 remaining -> ~20s
+            a.progress(1)
+            eta1 = drawnEta()
+
+            b = ProgressBar(0, 3, 78)
+            b._start = time.time() - 20.0      # 2 done in 20s -> 10s/item, 1 remaining -> ~10s
+            b.progress(2)
+            eta2 = drawnEta()
+        finally:
+            progress_mod.dataToStdout = real
+            progress_mod.IS_TTY = realTty
+
+        self.assertTrue(18 <= eta1 <= 22, msg="1/3 should estimate 2 remaining (~20s): %ds" % eta1)
+        self.assertTrue(8 <= eta2 <= 12, msg="2/3 should estimate 1 remaining (~10s): %ds" % eta2)
+
+    def test_new_estimate_is_eased_not_snapped(self):
+        # when a fresh estimate is far from the value currently on screen, the drawn ETA must land
+        # between the two (smoothed), not snap straight to the new target
+        captured = []
+        real = progress_mod.dataToStdout
+        realTty = progress_mod.IS_TTY
+        progress_mod.dataToStdout = lambda data, *a, **k: captured.append(data)
+        progress_mod.IS_TTY = True
+        try:
+            pb = ProgressBar(0, 10, 78)
+            pb._eta = 8.0                       # currently showing ~8s...
+            pb._etaAt = time.time()
+            pb._start = time.time() - 2.0       # ...but the fresh target is (2/1)*(10-1) = 18s
+            pb.progress(1)
+        finally:
+            progress_mod.dataToStdout = real
+            progress_mod.IS_TTY = realTty
+
+        m = re.search(r"\(ETA (\d{2}):(\d{2})\)", captured[-1])
+        secs = int(m.group(1)) * 60 + int(m.group(2))
+        self.assertTrue(10 <= secs <= 16, msg="ETA snapped instead of easing between 8 and 18: %ds" % secs)
+
+    def test_tick_counts_down_from_stored_eta(self):
+        captured = []
+        real = progress_mod.dataToStdout
+        realTty = progress_mod.IS_TTY
+        progress_mod.dataToStdout = lambda data, *a, **k: captured.append(data)
+        progress_mod.IS_TTY = True
+        try:
+            pb = ProgressBar(0, 10, 78)
+            pb.update(5)
+            pb._eta = 100                       # a 100s estimate...
+            pb._etaAt = time.time() - 30        # ...taken 30s ago -> tick() must show ~70s
+            pb.tick()
+        finally:
+            progress_mod.dataToStdout = real
+            progress_mod.IS_TTY = realTty
+
+        m = re.search(r"\(ETA (\d{2}):(\d{2})\)", captured[-1])
+        self.assertTrue(m, msg="no mm:ss ETA drawn: %r" % captured[-1])
+        secs = int(m.group(1)) * 60 + int(m.group(2))
+        self.assertTrue(66 <= secs <= 71, msg="ETA not decremented to ~70s: %r" % captured[-1])
+
+    def test_tick_clamps_at_zero_when_overdue(self):
+        captured = []
+        real = progress_mod.dataToStdout
+        realTty = progress_mod.IS_TTY
+        progress_mod.dataToStdout = lambda data, *a, **k: captured.append(data)
+        progress_mod.IS_TTY = True
+        try:
+            pb = ProgressBar(0, 10, 78)
+            pb.update(5)
+            pb._eta = 5
+            pb._etaAt = time.time() - 60        # long overdue -> must clamp to 00:00, never negative
+            pb.tick()
+        finally:
+            progress_mod.dataToStdout = real
+            progress_mod.IS_TTY = realTty
+
+        self.assertIn("(ETA 00:00)", captured[-1], msg=captured[-1])
+
+    def test_no_draw_when_not_tty(self):
+        captured = []
+        real = progress_mod.dataToStdout
+        realTty = progress_mod.IS_TTY
+        progress_mod.dataToStdout = lambda data, *a, **k: captured.append(data)
+        progress_mod.IS_TTY = False   # piped/redirected: no animated bar should reach the stream
+        try:
+            pb = ProgressBar(0, 10, 78)
+            for i in range(1, 11):
+                pb.progress(i)
+        finally:
+            progress_mod.dataToStdout = real
+            progress_mod.IS_TTY = realTty
+
+        self.assertEqual(captured, [], msg="progress bar leaked to a non-TTY stream: %r" % captured)
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_techniques.py

@@ -48,6 +48,7 @@ from lib.core.agent import agent
 from lib.core.unescaper import unescaper
 from lib.request.connect import Connect
 from lib.request.connect import Connect as Request
+from lib.request import inject
 from lib.utils.hashdb import HashDB
 
 import lib.techniques.union.use as uu
@@ -1516,6 +1517,47 @@ class TestConfigUnion(unittest.TestCase):
         self.assertEqual(kb.uChar, "SENTINEL")
 
 
+class TestValueParallelEligibility(unittest.TestCase):
+    """
+    inject.valueParallelEligible() picks the value-parallel path (job-level '--eta' bar / concurrency).
+    Safety invariant under test: classic time-based must never run concurrently (interfering SLEEP
+    measurements), so it qualifies only single-threaded under '--eta'; a concurrency-safe channel
+    (boolean or the timeless oracle) may run under either '--threads' or '--eta'.
+    """
+
+    def setUp(self):
+        self._avail = set()
+        self._realAvail = inject.isTechniqueAvailable
+        inject.isTechniqueAvailable = lambda t: t in self._avail
+        self._saved = (conf.threads, conf.eta, kb.get("timeless"))
+
+    def tearDown(self):
+        inject.isTechniqueAvailable = self._realAvail
+        conf.threads, conf.eta, kb.timeless = self._saved
+
+    def _elig(self, threads, eta, techniques, timeless=None):
+        conf.threads, conf.eta, kb.timeless = threads, eta, timeless
+        self._avail = set(techniques)
+        return inject.valueParallelEligible()
+
+    def test_single_thread_eta_time_based_qualifies(self):
+        self.assertTrue(self._elig(1, True, {PAYLOAD.TECHNIQUE.TIME}))
+
+    def test_multi_thread_time_based_never_parallel(self):
+        self.assertFalse(self._elig(8, True, {PAYLOAD.TECHNIQUE.TIME}))
+        self.assertFalse(self._elig(8, False, {PAYLOAD.TECHNIQUE.TIME}))
+
+    def test_boolean_qualifies_under_threads_or_eta(self):
+        self.assertTrue(self._elig(8, False, {PAYLOAD.TECHNIQUE.BOOLEAN}))
+        self.assertTrue(self._elig(1, True, {PAYLOAD.TECHNIQUE.BOOLEAN}))
+
+    def test_plain_single_thread_no_eta_stays_classic(self):
+        self.assertFalse(self._elig(1, False, {PAYLOAD.TECHNIQUE.BOOLEAN}))
+
+    def test_timeless_is_concurrency_safe(self):
+        self.assertTrue(self._elig(8, True, {PAYLOAD.TECHNIQUE.TIME}, timeless=object()))
+
+
 if __name__ == "__main__":
     unittest.main(verbosity=2)