Fixing --eta

lib/core/settings.py

@@ -20,7 +20,7 @@ from lib.core.enums import OS
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.7.55"
+VERSION = "1.10.7.56"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -249,6 +249,9 @@ THREAD_FINALIZATION_TIMEOUT = 1
 # Maximum number of techniques used in inject.py/getValue() per one value
 MAX_TECHNIQUES_PER_VALUE = 2
 
+# Fraction of the currently displayed progress-bar ETA kept when a fresh estimate arrives (eases the on-screen countdown toward the new value instead of snapping); 0 disables smoothing, higher is smoother but laggier
+ETA_DISPLAY_SMOOTHING = 0.5
+
 # In case of missing piece of partial union dump, buffered array must be flushed after certain size
 MAX_BUFFERED_PARTIAL_UNION_LENGTH = 1024
 

lib/request/inject.py

@@ -8,6 +8,7 @@ See the file 'LICENSE' for copying permission
 from __future__ import print_function
 
 import re
+import threading
 import time
 
 from lib.core.agent import agent
@@ -16,6 +17,7 @@ from lib.core.common import applyFunctionRecursively
 from lib.core.common import dataToStdout
 from lib.core.common import unArrayizeValue
 from lib.core.datatype import AttribDict
+from lib.utils.progress import ProgressBar
 from lib.utils.safe2bin import safecharencode
 from lib.core.common import Backend
 from lib.core.common import calculateDeltaSeconds
@@ -58,12 +60,14 @@ from lib.core.exception import SqlmapDataException
 from lib.core.exception import SqlmapNotVulnerableException
 from lib.core.exception import SqlmapUserQuitException
 from lib.core.settings import GET_VALUE_UPPERCASE_KEYWORDS
+from lib.core.settings import IS_TTY
 from lib.core.settings import INFERENCE_MARKER
 from lib.core.settings import MAX_TECHNIQUES_PER_VALUE
 from lib.core.settings import SQL_SCALAR_REGEX
 from lib.core.settings import UNICODE_ENCODING
 from lib.core.threads import getCurrentThreadData
 from lib.core.threads import runThreads
+from lib.core.threads import setDaemon
 from lib.core.unescaper import unescaper
 from lib.request.connect import Connect as Request
 from lib.request.direct import direct
@@ -405,6 +409,25 @@ def _verifyInferredValue(expression, value):
     except Exception:
         return True
 
+def valueParallelEligible():
+    """
+    Whether blind enumeration/dumping should take the value-parallel path (one whole value per worker)
+    rather than the classic char loop. It is chosen for concurrency ('--threads') and, independently, to
+    render a single whole-job progress bar/ETA ('--eta'). A concurrency-safe channel - boolean or the
+    HTTP/2 timeless oracle - qualifies with either. Classic time-based qualifies only single-threaded under
+    '--eta': concurrent SLEEP measurements interfere, so it must stay sequential (where it is safe, and
+    still gets the whole-job ETA that matters most for the slowest channel). Callers add their own extra
+    guards (e.g. '--dns-domain', per-column comments).
+    """
+
+    concurrencySafe = isTechniqueAvailable(PAYLOAD.TECHNIQUE.BOOLEAN) or kb.get("timeless") is not None
+
+    if conf.threads > 1:
+        return concurrencySafe
+    if conf.eta:
+        return concurrencySafe or isTechniqueAvailable(PAYLOAD.TECHNIQUE.TIME)
+    return False
+
 def _threadedInferenceValues(exprBuilder, indices, context=None, charsetType=None, dump=False):
     """
     Value-parallel blind retrieval.
@@ -446,6 +469,13 @@ def _threadedInferenceValues(exprBuilder, indices, context=None, charsetType=Non
     results = [None] * len(indices)
     cursor = iter(xrange(len(indices)))
 
+    # With '--eta' show a single value-level bar (values completed / total) instead of the per-value
+    # stream: it is monotonic and carries a meaningful ETA across the whole set, and - unlike the
+    # per-char bar drawn inside bisection() - it survives the worker's disableStdOut (drawn forced,
+    # below), so '--eta' is honoured under '--threads' too. Skipped for trivial single-value sets.
+    etaProgress = ProgressBar(maxValue=len(indices)) if (conf.eta and len(indices) > 1) else None
+    completed = [0]
+
     def inferenceThread():
         threadData = getCurrentThreadData()
         # Each per-value bisection streams its characters to stdout and mirrors them into
@@ -489,13 +519,36 @@ def _threadedInferenceValues(exprBuilder, indices, context=None, charsetType=Non
             with kb.locks.value:
                 results[slot] = value
 
+            if etaProgress is not None:
+                with kb.locks.io:
+                    completed[0] += 1
+                    threadData.disableStdOut = False        # let the value-level bar through (per-char streaming stays muted)
+                    try:
+                        etaProgress.progress(completed[0])
+                    finally:
+                        threadData.disableStdOut = True
+
             # Stream each retrieved value as it completes (they arrive out of order under threads, exactly
             # like the error/union dumps), so a dump shows its data live rather than a silent counter.
-            if conf.verbose >= 1 and not kb.bruteMode and not isNoneValue(value):
+            elif conf.verbose >= 1 and not kb.bruteMode and not isNoneValue(value):
                 with kb.locks.io:
                     rendered = safecharencode(unArrayizeValue(value))
                     dataToStdout("[%s] [INFO] retrieved: %s\n" % (time.strftime("%X"), "'%s'" % rendered if dump else rendered), forceOutput=True)
 
+    # Keep the '--eta' countdown live between value updates: a value (esp. time-based) can take many
+    # seconds, so a daemon redraws the bar every second with the ETA decremented by elapsed time (it runs
+    # on its own thread, so it is not muted by the workers' disableStdOut, and shares kb.locks.io with them).
+    etaTickerStop = threading.Event()
+    etaTicker = None
+    if etaProgress is not None and IS_TTY:
+        def _etaTicker():
+            while not etaTickerStop.wait(1.0):
+                with kb.locks.io:
+                    etaProgress.tick()
+        etaTicker = threading.Thread(target=_etaTicker, name="eta-ticker")
+        setDaemon(etaTicker)
+        etaTicker.start()
+
     # Save/restore the calling thread's state: with a single thread runThreads runs the worker
     # INLINE on this thread, so the worker's disableStdOut/shared mutations must not leak out.
     savedPartRun = kb.partRun
@@ -505,6 +558,9 @@ def _threadedInferenceValues(exprBuilder, indices, context=None, charsetType=Non
     try:
         runThreads(min(conf.threads or 1, len(indices)) or 1, inferenceThread)
     finally:
+        etaTickerStop.set()
+        if etaTicker is not None:
+            etaTicker.join(timeout=2)
         kb.partRun = savedPartRun
         mainThreadData.disableStdOut = savedStdOut
         mainThreadData.shared = savedShared

lib/utils/progress.py

@@ -13,6 +13,8 @@ from lib.core.common import dataToStdout
 from lib.core.convert import getUnicode
 from lib.core.data import conf
 from lib.core.data import kb
+from lib.core.settings import ETA_DISPLAY_SMOOTHING
+from lib.core.settings import IS_TTY
 
 class ProgressBar(object):
     """
@@ -26,7 +28,9 @@ class ProgressBar(object):
         self._span = max(self._max - self._min, 0.001)
         self._width = totalWidth if totalWidth else conf.progressWidth
         self._amount = 0
-        self._start = None
+        self._start = time.time()   # begin timing at construction, so the first completed item already yields an estimate
+        self._eta = None            # last estimated seconds-remaining and when it was computed, so tick()
+        self._etaAt = None          # can keep the countdown live between (possibly slow) item updates
         self.update()
 
     def _convertSeconds(self, value):
@@ -73,17 +77,39 @@ class ProgressBar(object):
 
     def progress(self, newAmount):
         """
-        This method saves item delta time and shows updated progress bar with calculated eta
+        Redraw the bar with an ETA from the average time per completed item so far, applied to the items
+        still remaining: (elapsed / done) * (max - newAmount). The remaining-item count is (max - newAmount)
+        - i.e. at 1/3 it estimates the 2 items left, at 2/3 the 1 left - not just the current item.
         """
 
-        if self._start is None or newAmount > self._max:
-            self._start = time.time()
-            eta = None
-        else:
-            delta = time.time() - self._start
-            eta = (self._max - self._min) * (1.0 * delta / newAmount) - delta
+        now = time.time()
+        if newAmount > self._max:               # counter rollover/reset -> restart timing
+            self._start = now
+            self._eta = None
 
+        done = newAmount - self._min
+        elapsed = now - self._start
+        target = (elapsed / done) * (self._max - newAmount) if (done > 0 and elapsed > 0) else None
+
+        if target is None:
+            self._eta = None
+        elif self._eta is None:
+            self._eta = target                  # first estimate: nothing to ease from
+        else:
+            current = max(0, self._eta - (now - self._etaAt))   # what is on screen now (already decremented by tick())
+            self._eta = ETA_DISPLAY_SMOOTHING * current + (1 - ETA_DISPLAY_SMOOTHING) * target   # ease into the fresh estimate
+
+        self._etaAt = now
         self.update(newAmount)
+        self.draw(self._eta)
+
+    def tick(self):
+        """
+        Redraw the current bar with its ETA decremented by real elapsed time, so the countdown stays
+        live between updates (e.g. during a long time-based wait) instead of freezing at the last estimate
+        """
+
+        eta = None if self._eta is None else max(0, self._eta - (time.time() - self._etaAt))
         self.draw(eta)
 
     def draw(self, eta=None):
@@ -91,6 +117,9 @@ class ProgressBar(object):
         This method draws the progress bar if it has changed
         """
 
+        if not IS_TTY:      # a progress bar is a terminal animation; suppress it when piped/redirected (as done for other '\r' output)
+            return
+
         dataToStdout("\r%s %d/%d%s" % (self._progBar, self._amount, self._max, ("  (ETA %s)" % (self._convertSeconds(int(eta)) if eta is not None else "??:??"))))
         if self._amount >= self._max:
             dataToStdout("\r%s\r" % (" " * self._width))