From fabe4ce04d6efa923e8c5a9f32574511f8003565 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0tampar?= Date: Sun, 19 Jul 2026 12:22:23 +0200 Subject: [PATCH] Minor fix --- lib/core/settings.py | 2 +- lib/utils/crawler.py | 39 +++++++++++++++++--------- tests/test_crawler.py | 65 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 14 deletions(-) create mode 100644 tests/test_crawler.py diff --git a/lib/core/settings.py b/lib/core/settings.py index 5cc0106f0..750d36bd6 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -20,7 +20,7 @@ from lib.core.enums import OS from thirdparty import six # sqlmap version (...) -VERSION = "1.10.7.122" +VERSION = "1.10.7.123" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/lib/utils/crawler.py b/lib/utils/crawler.py index ff8d7bdd6..787f0a15e 100644 --- a/lib/utils/crawler.py +++ b/lib/utils/crawler.py @@ -223,22 +223,35 @@ def crawl(target, post=None, cookie=None): kb.normalizeCrawlingChoice = readInput(message, default='Y', boolean=True) if kb.normalizeCrawlingChoice: - seen = set() - results = OrderedSet() - - for target in kb.targets: - value = "%s%s%s" % (target[0], '&' if '?' in target[0] else '?', target[2] or "") - match = re.search(r"/[^/?]*\?.+\Z", value) - if match: - key = re.sub(r"=[^=&]*", "=", match.group(0)).strip("&?") - if '=' in key and key not in seen: - results.add(target) - seen.add(key) - - kb.targets = results + kb.targets = normalizeCrawlingResults(kb.targets) storeResultsToFile(kb.targets) +def normalizeCrawlingResults(targets): + """ + Collapses crawled targets that differ only in their parameter values (e.g. ?id=1 vs ?id=2), + keeping one representative per distinct endpoint+parameter-name shape + + >>> sorted(_[0] for _ in normalizeCrawlingResults([("http://h/users/edit?id=1", None, None, None, None), ("http://h/users/edit?id=2", None, None, None, None), ("http://h/products/edit?id=1", None, None, None, None)])) + ['http://h/products/edit?id=1', 'http://h/users/edit?id=1'] + """ + + seen = set() + results = OrderedSet() + + for target in targets: + value = "%s%s%s" % (target[0], '&' if '?' in target[0] else '?', target[2] or "") + # Note: key on the full path (not just the last segment) so distinct endpoints sharing an + # action name and parameters (e.g. /users/edit?id= vs /products/edit?id=) are not collapsed + match = re.search(r"\A[^?]+\?.+\Z", value) + if match: + key = re.sub(r"=[^=&]*", "=", match.group(0)).strip("&?") + if '=' in key and key not in seen: + results.add(target) + seen.add(key) + + return results + def storeResultsToFile(results): if not results: return diff --git a/tests/test_crawler.py b/tests/test_crawler.py new file mode 100644 index 000000000..709e25896 --- /dev/null +++ b/tests/test_crawler.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python + +""" +Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org) +See the file 'LICENSE' for copying permission + +Crawler result normalization (lib/utils/crawler.py normalizeCrawlingResults). + +--crawl can surface thousands of near-identical URLs; normalization keeps one +representative per distinct endpoint+parameter shape so the scan is not flooded +with value-only variants. The key must span the full path: collapsing on the +last path segment alone silently drops distinct endpoints that share an action +name (e.g. /users/edit vs /products/edit), losing real attack surface. +""" + +import os +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from _testutils import bootstrap +bootstrap() + +from lib.utils.crawler import normalizeCrawlingResults + + +def _t(url, data=None): + # kb.targets tuple shape: (url, method, data, ...) + return (url, None, data, None, None) + + +class TestNormalizeCrawlingResults(unittest.TestCase): + def _urls(self, targets): + return [t[0] for t in normalizeCrawlingResults(targets)] + + def test_value_only_variants_collapse(self): + kept = self._urls([_t("http://h/item?id=1"), _t("http://h/item?id=2"), _t("http://h/item?id=3")]) + self.assertEqual(kept, ["http://h/item?id=1"]) + + def test_distinct_endpoints_sharing_action_are_kept(self): + # the regression: /users/edit and /products/edit must not collapse on the shared last segment + kept = self._urls([_t("http://h/users/edit?id=1"), + _t("http://h/products/edit?id=1"), + _t("http://h/orders/edit?id=1"), + _t("http://h/users/edit?id=2")]) + self.assertEqual(set(kept), {"http://h/users/edit?id=1", + "http://h/products/edit?id=1", + "http://h/orders/edit?id=1"}) + + def test_different_parameter_names_are_kept(self): + kept = self._urls([_t("http://h/p?id=1"), _t("http://h/p?name=x")]) + self.assertEqual(set(kept), {"http://h/p?id=1", "http://h/p?name=x"}) + + def test_different_hosts_are_kept(self): + kept = self._urls([_t("http://a.tld/edit?id=1"), _t("http://b.tld/edit?id=1")]) + self.assertEqual(set(kept), {"http://a.tld/edit?id=1", "http://b.tld/edit?id=1"}) + + def test_post_data_folded_into_shape(self): + # POST body params participate in the shape, and value-only POST variants collapse + kept = self._urls([_t("http://h/login", "user=a&pass=b"), _t("http://h/login", "user=c&pass=d")]) + self.assertEqual(kept, ["http://h/login"]) + + +if __name__ == "__main__": + unittest.main(verbosity=2)