diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 367bec21405..a6bedc1e686 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,6 +17,10 @@ jobs: runs-on: ${{ matrix.os }} timeout-minutes: 30 + env: + # deterministic dict/set iteration order run-to-run (guards against hash-order flakiness in CI) + PYTHONHASHSEED: "0" + strategy: matrix: include: diff --git a/data/txt/common-outputs.txt b/data/txt/common-outputs.txt index 1df3cd36f81..5df11be3dcb 100644 --- a/data/txt/common-outputs.txt +++ b/data/txt/common-outputs.txt @@ -1364,3 +1364,113 @@ username visible zip zip_code + +# --- real-world application / CMS / framework values (repeated section headers are merged on load) --- +[Databases] +wordpress +wp +drupal +joomla +magento +prestashop +opencart +moodle +mediawiki +phpbb +typo3 +laravel +symfony +django +app +application +webapp +web +website +main +backend +api +cms +shop +store +ecommerce +blog +forum +wiki +crm +erp +billing +sales +accounts +inventory +catalog +orders +payments +customers +members +users +data +db +mydb +appdb +prod +production +dev +staging +qa +demo +sample +employees +sakila +world +classicmodels +dvwa +bwapp +mutillidae +dashboard +defaultdb + +[Users] +admin +administrator +root +sa +postgres +oracle +system +dbadmin +dba +dbo +webadmin +web +www +www-data +apache +nginx +app +appuser +application +service +svc +user +dbuser +guest +test +demo +backup +replication +monitor +readonly +superuser +wordpress +drupal +joomla +magento +laravel +django +symfony +'admin'@'localhost' +'admin'@'%' +'app'@'localhost' +'app'@'%' +'web'@'%' +'wordpress'@'localhost' diff --git a/data/txt/sha256sums.txt b/data/txt/sha256sums.txt index 956c8865d93..a894ed05419 100644 --- a/data/txt/sha256sums.txt +++ b/data/txt/sha256sums.txt @@ -25,7 +25,7 @@ c52c17f3344707cae4c3694a979e073202bd46866fcc51d99f7e4d0c21cf335b data/shell/sta af4e1f87ec7afd12b7ddb39ff07bf24cd31be2b1de11e1be064e1dd96ff43eac data/shell/stagers/stager.php_ eb86f6ad21e597f9283bb4360129ebc717bc8f063d7ab2298f31118275790484 data/txt/common-columns.txt 63ba15f2ba3df6e55600a2749752c82039add43ed61129febd9221eb1115f240 data/txt/common-files.txt -852b420157bbffb56947e4b201a7df5242e75443ab161049a50235eb4e8e9aae data/txt/common-outputs.txt +4d6a32155dd6b570e5cdae8036efd69d8f8ebab79cb82a4d094c15f35af8b13d data/txt/common-outputs.txt 44047281263ef297f27fdd8fa98a0b0438a25989f897ce184cb0e2e442fb6c11 data/txt/common-tables.txt ccba96624a0176b4c5acd8824db62a8c6856dafa7d32424807f38efed22a6c29 data/txt/keywords.txt 522cce0327de8a5dfb5ade505e8a23bbd37bcabcbb2993f4f787ccdecf24997e data/txt/smalldict.txt @@ -84,12 +84,12 @@ c8d467837c8567b61a11e2dfd75a2d8305a8b317041ee81eda6d0e47609dabb7 data/xml/paylo 0648264166455010921df1ec431e4c973809f37ef12cbfea75f95029222eb689 data/xml/payloads/stacked_queries.xml 379fc92f2dadd948f401e17490d8a8f03a1988d817323cbe1feff5fe87726079 data/xml/payloads/time_blind.xml 40a4878669f318568097719d07dc906a19b8520bc742be3583321fc1e8176089 data/xml/payloads/union_query.xml -45aa5280edc0412a217498bd229651ff9c55afab44d555507ee5bdc27531de82 data/xml/queries.xml +ff99497d2f04a872e16e799183e6c8f2e16f3e69cddb336e29162f1e92ae45c7 data/xml/queries.xml 127799739f9aeabca367027197f3c0240f141303bd7499928ccfa1443bf148c7 doc/ARCHITECTURE.md 0f5a9c84cb57809be8759f483c7d05f54847115e715521ac0ecf390c0aa68465 doc/AUTHORS ce20a4b452f24a97fde7ec9ed816feee12ac148e1fde5f1722772cc866b12740 doc/CHANGELOG.md 233fb10dff24a2436eb24496db7fadb46659da6745a0d53c744db701188041ef doc/THANKS.md -b6fcc489c6eaca2a7d0d031bd04fe28e6790ffe4dfd4bdf055b6dc83b992dc86 doc/THIRD-PARTY.md +8d9c49ac2c05b594c1c36a03c41cf9e3641626a94fe11d86787df4125064b6a0 doc/THIRD-PARTY.md 2af9b7a8c5f24de68f9b8b1bcf3a7f2b0e55fdb48b6545e1fc8b13f406ac97c2 doc/translations/README-ar-AR.md c25f7d7f0cc5e13db71994d2b34ada4965e06c87778f1d6c1a103063d25e2c89 doc/translations/README-bg-BG.md e85c82df1a312d93cd282520388c70ecb48bfe8692644fe8dbbf7d43244cda41 doc/translations/README-bn-BD.md @@ -163,36 +163,36 @@ df768bcb9838dc6c46dab9b4a877056cb4742bd6cfaaf438c4a3712c5cc0d264 extra/shutils/ 9af5fdfa8b2425d404d86ab08d3644caa95bcf77605551f5da482a59d1e54a22 extra/vulnserver/vulnserver.py a2bf70d7f87c3a4e0675c0bad54119a4e04efa6ea2730a8338d5aebcd995630e lib/controller/action.py 736715a73941a06e5d3d349dd01a1f1b171f54eb4c374c6752b2cc44b0977ffe lib/controller/checks.py -666935b658074dc9c42153622b75d4ec7bfe56fbe0742de827a5d30a1a0f9d96 lib/controller/controller.py +2086100cd7a78a4e8c12d72bd4f5b414ec6b3f49926e83285494534140e60ce7 lib/controller/controller.py d69e84f1648cdb907f5d2dd454f03874a4613752b07867510145d51d84b3c56f lib/controller/handler.py 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 lib/controller/__init__.py -9c5764c92ce536d1f0f96200359ee5ef1f37f9128769bf990cb77f1d1f8e17b1 lib/core/agent.py +48ffe93d61734e16c3b20153b51595853d9ac1fbcf0b537e0e61e957b0c0bfa6 lib/core/agent.py c51c33501cc905586a9aaac93b06f2ac6f71628d032a7dc39fd0ef05d7ee3856 lib/core/bigarray.py -751c3bf178e91e60b25e3b01ce7636029804dd78f64e9ee0418bdb126889a7bc lib/core/common.py +c230a214023a6556648e6af485b42fbcd10f23d2cb9018ad7bc68e36f7241328 lib/core/common.py 8f1272487e1adfcc8c755a2f56f0c6d21eac5e685a73a9a159482f9dc9142bc5 lib/core/compat.py 5301ba2204404d086e9a67271cde00fc10214c63b018a95fc5aa90ff9e0b2ad9 lib/core/convert.py c03dc585f89642cfd81b087ac2723e3e1bb3bfa8c60e6f5fe58ef3b0113ebfe6 lib/core/data.py -d9ec034a6d51ab4ddde0b6aa7ed306f9e0b1336557f77d7939ba547600f9b3ae lib/core/datatype.py +771ef50ebfa72a1019f819071dcfcd249ea6bb533051e9388c14917823e1f4f3 lib/core/datatype.py f8de57606325456928e46ae2896f5f8bbec9ad18b1c644b492a566fa992216f6 lib/core/decorators.py 147823c37596bd6a56d677697781f34b8d1d1671d5a2518fbc9468d623c6d07d lib/core/defaults.py 8e4f4b5ea37a49d445bb0df83bf04b34f61035ec33fd8acf598ebcf371cb19a7 lib/core/dicts.py -10d8bb671a64cc787fc2fbf2c641560b7797fccd62c4792e55dffe5efab9f544 lib/core/dump.py -6dd47f52082e98dc0cda6969b277b7d81c6f7c68dac4688821f873a1c65c6edf lib/core/enums.py +b14628a6c9327d110afe50b01f3171f64f61823343b8de89596e854b00b74928 lib/core/dump.py +c2db614a3ce7dda889152bea8bd6d709e5d8c2b556741fdbfe44469f27ce266b lib/core/enums.py 5387168e5dfedd94ae22af7bb255f27d6baaca50b24179c6b98f4f325f5cc7b4 lib/core/exception.py 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 lib/core/__init__.py 914a13ee21fd610a6153a37cbe50830fcbd1324c7ebc1e7fc206d5e598b0f7ad lib/core/log.py -4fe3ac4c0d354d1ac42ad3f5dc1b308993588f8a249ff880d273f5031d6b52b0 lib/core/optiondict.py -98d3d61278794705c7039e40fab66a626e8d6ab765383c5379cec7a066b09301 lib/core/option.py +47c9828bdfa606a02f07925539d7af55c5eaf1fda61d05ecc40f73d77df036f9 lib/core/optiondict.py +3ac60716cf1c619b80038acb8b213c728cc607e7c5a387911e01635a23fbc92b lib/core/option.py 21b2b1745107c211fc7593923a3da7a808d40763c00091c28de5f7c129bcf3bc lib/core/patch.py 49c0fa7e3814dfda610d665ee02b12df299b28bc0b6773815b4395514ddf8dec lib/core/profiling.py 0c36a65b6237732eb001d333f80f0c58c088ff01ae80cf07e4dcc6da2a806364 lib/core/readlineng.py 9bf174058f15d14e24e94f9aaf42df045119d3617c6c54bd2f3af79b462f331d lib/core/replication.py 0b8c38a01bb01f843d94a6c5f2075ee47520d0c4aa799cecea9c3e2c5a4a23a6 lib/core/revision.py 888daba83fd4a34e9503fe21f01fef4cc730e5cde871b1d40e15d4cbc847d56c lib/core/session.py -61024490352e898a43f1cb001fb79276d185ef3579b6230df46badf573336833 lib/core/settings.py +2f2411c91cab0ee8b337c9672bd510e408e1ab44b83ec0eaf0763604f4f99926 lib/core/settings.py c7804223319e18eb0b8e2cbf0a8b6896d1cefb7b0b1a2e9f1cf826a8a3b56750 lib/core/shell.py a2e98a94b231432736d6b304fc75525c8b5fdb4768c418387c5b4c1a610dad64 lib/core/subprocessng.py -19f1e3c5e3ba703d28d510cd7a9ab8284d5fbe9df5ce7e77c86e5931571364b7 lib/core/target.py +15d36cdac9389d0a54a6c33fbb89f32bb65e303f50de573773dcb6d4618bca64 lib/core/target.py 96d107a31bb9647a9b7c26f10beac528bf4edc6e607c8b776c624d494332c7f8 lib/core/testing.py 95656c44bab1771f4808030dd6a17eae5b129cb1234443f00b19695c7b712b86 lib/core/threads.py b9aacb840310173202f79c2ba125b0243003ee6b44c92eca50424f2bdfc83c02 lib/core/unescaper.py @@ -200,26 +200,27 @@ b9aacb840310173202f79c2ba125b0243003ee6b44c92eca50424f2bdfc83c02 lib/core/unesc 2400e465fa4d13e4c32795910878c71ff212e4361b46428d57ce43983f5e997c lib/core/wordlist.py 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 lib/__init__.py 54bfd31ebded3ffa5848df1c644f196eb704116517c7a3d860b5d081e984d821 lib/parse/banner.py -d6ba23b8f3d40cb021de1ebe50eabf891f060df77e9643838ff8fd3850b507d0 lib/parse/cmdline.py -02d82e4069bd98c52755417f8b8e306d79945672656ac24f1a45e7a6eff4b158 lib/parse/configfile.py +fef119c6f3f2fe6a092112fd832d645c58e4c3c2af0bd97ace4487372c1e3574 lib/parse/cmdline.py +925a068efa1885fa40671414a887c088f2aafbe8cb76f01286e6bde3f624dac1 lib/parse/configfile.py c5b258be7485089fac9d9cd179960e774fbd85e62836dc67cce76cc028bb6aeb lib/parse/handler.py 5c9a9caee948843d5537745640cc7b98d70a0412cc0949f59d4ebe8b2907c06c lib/parse/headers.py ea9b195e5f5030b96d1993c106c1e13fb5c7faaf6bdc5daacfd06ec984e7f323 lib/parse/html.py 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 lib/parse/__init__.py +9cb95cc5136d5ac624860578099929fdb335face41026f79f49df4f52da9805d lib/parse/openapi.py d2e771cdacef25ee3fdc0e0355b92e7cd1b68f5edc2756ffc19f75d183ba2c73 lib/parse/payloads.py c2f34e27578742e729c2fa9c1d4f0a0d8f8f7f4cf0fc14c62ec817a260c71dec lib/parse/sitemap.py 1be3da334411657461421b8a26a0f2ff28e1af1e28f1e963c6c92768f9b0847c lib/request/basicauthhandler.py -369484a2999d29f49bf839a329d1686ed94f6ea27c695e027fe08c8da51f30a3 lib/request/basic.py +a988c659e0c642e4f3dc4034118b5a6e138a522394ff2eda5bdc3c8495ea2207 lib/request/basic.py bc61bc944b81a7670884f82231033a6ac703324b34b071c9834886a92e249d0e lib/request/chunkedhandler.py -9c0dccc1cee66d38478aaf75a7c513d0d136d50a90b15fed146faa1653899fe1 lib/request/comparison.py -c96deaa69743d2cf4ae48f2ae0036f7e11b838f97a0e8c7f1205c61e9dd36bc1 lib/request/connect.py +4fd1957e31b14e7670b09d85a634fa6772a1cd90babe149f39a1c945fe306f0a lib/request/comparison.py +4a3b997a83b1724e8bd025be95ec5d84c6bf41d533ba097fcab1eab763352111 lib/request/connect.py 8e06682280fce062eef6174351bfebcb6040e19976acff9dc7b3699779783498 lib/request/direct.py a6b37b436838caeb197fea858d0a39fadbff4736256e741b5fcec1f28fcf1ce0 lib/request/dns.py -21e8e2d44788b124f741b76a483ce9528ca53ff6da6691808ee679fe91128050 lib/request/http2.py +7344978ac1c52060716b7837c88a62768c6a445eafe189ea3232b8a498fdd038 lib/request/http2.py 92c81cc31ff4a396723242058fb2152c9e9745f8412d01ea74480b048a53af6c lib/request/httpshandler.py 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 lib/request/__init__.py 7a0ac2522213e756348fd871a7af74cc963bdc82f9d7ade57be5de42b5bf7cab lib/request/inject.py -d1c5e4bda94394b5bb42c3b48b41b73ecb6069c3971af2c54394c9b35c2fed6e lib/request/keepalive.py +ff15723c82e343eb95f4599d251165d478ca720afc8f5daaed3da44ea923df44 lib/request/keepalive.py ada4d305d6ce441f79e52ec3f2fc23869ee2fa87c017723e8f3ed0dfa61cdab4 lib/request/methodrequest.py 43a7fdf64e7ba63c6b2d641c9f999a63c12ac23b43b64fedfce4e05b863de568 lib/request/pkihandler.py b90feeb16e89a844427df42373b0139eb6f6cf3c48ccec32b3e3a3f540c2451e lib/request/rangehandler.py @@ -250,11 +251,11 @@ bde75d41ac3e5747b96d2af4c33922573158cb43b48714a28490d6720dd85d89 lib/techniques 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 lib/techniques/ssti/__init__.py 14637b64878248e5965887b07aa68e62615dac88e2ffc6c3a581430bdd4e309e lib/techniques/ssti/inject.py 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 lib/techniques/union/__init__.py -ceec65f8cb7c3254c4671351c837418c76ac5bc55ccbc40779f67231b54d7085 lib/techniques/union/test.py -c65766f71e285fc85cdf58e7448c4c1d015af2a9dbb44fa3b665a9f13362fbcc lib/techniques/union/use.py +f6678ac1342f8d234ed32ae69be5ac5d7837393e9348929ec029c9764c030e82 lib/techniques/union/test.py +c68f8259e0a89a556d049f227041849df584313bd1b5349b02f74a47778c901c lib/techniques/union/use.py 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 lib/techniques/xpath/__init__.py c61816c9dba9f6cc2223aed1a923f95130979e5f0a88ec254ee667d955ed2734 lib/techniques/xpath/inject.py -aeefb42ea0c68f72744bc1bfd7194ec1bc06480d8a7e23f4b8d3d23fbba2b014 lib/utils/api.py +c5850075861bd5f172e191a0e48dd1d636d7c6af53bb471a44d56e7cef4e79c5 lib/utils/api.py 442555ab85277aff7c9e0cf465ea5b0d28395c326f68363449b2d3941f4b6de2 lib/utils/brute.py da5bcbcda3f667582adf5db8c1b5d511b469ac61b55d387cec66de35720ed718 lib/utils/crawler.py 51deedec3d3e869b067824caa51406d2ef396c188f82013ca60777006a821e27 lib/utils/deps.py @@ -263,10 +264,11 @@ bd9267d94390ba87d6c5a35c90f2406d6a4135a7c8ea01db76dd9e6519eee2ed lib/utils/dial 3c4ad819589fe4fca303706dc87969273a07a04dee85e23f064b39caf1fb80e9 lib/utils/gui.py 972c5db9c9e30ac0f91c0f8d4df4531d0304e151dac99f1399c37c952ba9f935 lib/utils/har.py 0cd3860c03e39bacd1d0fe4cf1a0c605de48ff82f70441319f21d47e38e7e3a9 lib/utils/hashdb.py -71a66ff766a2921106770b26acff380de469222dc893816a7b970b384c927666 lib/utils/hash.py +0c4ffffbf873bfc6981da6c92697331ce8d985025982ad7c6d52f2c26639df73 lib/utils/hash.py 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 lib/utils/__init__.py 1bbf57e43f921d4132e6e5a336ff39454a9506b36de94ebcc45879d0abcac56a lib/utils/keysetdump.py -04b28ad98340a589eb9b21d014c435e8193c2bea3a21af9875b6f23c9b270f1f lib/utils/pivotdumptable.py +b57aa20b7a6fd8afd07bae773fd03f8acb05655ee605362b220e65a0664dc38d lib/utils/library.py +dd30ef67da30b666c53013ee32253cd9396ed0e5d0a44d509680742e06ebcd23 lib/utils/pivotdumptable.py c1dfc3bed0fed9b181f612d1d747955dd2b506dbe99bc9fd481495602371473a lib/utils/progress.py c442e9ef8324fd6fdf7bc334d765f0a6ce4037397eb3d79d59b5ce3e9a043855 lib/utils/prove.py 2cd84db16edef8c9948e197a51d870cf1c338f4a89037b4d422de990f4a45237 lib/utils/purge.py @@ -509,7 +511,7 @@ cedf45d33461bd7e5400d06611a63c8a4ffae1a4510030c5696b9d46ed6a9883 plugins/generi 46517f1444c202710e388873960130850ed092e17bd6f4dd5f2fedea3dbb8ffc sqlmapapi.py f09d1b06901e7e02d0dbf4de607f6a4a9889acc322ae9353b98ea9101fb9548a sqlmapapi.yaml 627d90f1194335b800cbc9cc78db6697cf9e02e193a83598e0d4d0abb55b63b8 sqlmap.conf -d375c77f1f4270ec0967e67963fe410f14b5d2e51ed6483593dc1aaa4e8e106e sqlmap.py +80d66407453d34d672c389f6d9ab059d925528615429f2e6e9f286ce03d2c5d6 sqlmap.py eb37a88357522fd7ad00d90cdc5da6b57442b4fec49366aadb2944c4fbf8b804 tamper/0eunion.py a9785a4c111d6fee2e6d26466ba5efb3b229c00520b26e8024b041553b53efba tamper/apostrophemask.py cf26bc8006519bd25ce06d347f72770cd75b61575cf65e5812274e8ab9392eb4 tamper/apostrophenullencode.py @@ -584,83 +586,86 @@ dcdeed9ee285e63cf06baf8347e3db7f210ef25a63869bab78ce1ec6898ae191 tamper/unional 0694e721b07b8242245688be5c7951a3a22f512ed73776a998885e4b1bc82bc7 tamper/versionedmorekeywords.py ce1b6bf8f296de27014d6f21aa8b3df9469d418740cd31c93d1f5e36d6c509cf tamper/xforwardedfor.py 44401cad3e39ae9fb899ed5d0e2fdd0879561de05c3117f17f3b0db54f4e3724 tests/__init__.py -d16977d057c28888aa41500f79a19789cadef693cb8b7d9a3bca55b983ce2266 tests/test_agent.py -138381e05a860272fedab780e6c38ab74c59c879048b11b909d23f8df654352a tests/test_api.py -feb763ddcbf4f32822372ca53f8c71c754af7b72510ef06e1e9c77927fc90b10 tests/test_bigarray.py -36bcb68483d824db5d05870fab62f1907221bf256826b734302fbc15a9231c42 tests/test_brute.py +0e9054da5d1fed1ddfc982b8f559914237f65d9be5e595c3218fcd236dfa7212 tests/test_agent.py +9dc0ce7a038e7ac67c7f992b478a58492dad335d14761fa0600eec1f5a339c76 tests/test_api.py +694d8c87b2b98d7de6bc09fd634a2d32c436c7955c793cca6fa8790d3868f701 tests/test_bigarray.py +aeefe699f477e77ec4fb46c2692a1ea04cd89ad9cce62e8857d13e3bc0606e9d tests/test_brute.py 27ad87c0ea377e0657bd6f6a4eaa0e9756aa9d28ec0483bdadeb3f66dcc4660d tests/test_charset.py -7596fc69678304923b5c945c0fd9b8ee62a2dfc7fb14ccb6dc7af30893dc8012 tests/test_checks.py +9cc73e06ba3b4c07e0d8f5fd1962f8f25ba6b7ab7278cfb094bfff76fe5e7328 tests/test_checks.py 9e678a56e16211c49ab4995b6c658d3f122bfa3b357d9e17ff38f5a489ace6ad tests/test_cloak.py 2ec894f49ca9bd750a23ead16dae176bcbc57d18ec5847fa4a5eeb886d75c1bd tests/test_common_helpers.py -cdacb37cbe5667fded00abe62a822e11c917e9cb5c3f664b7aa1a8d738412ed4 tests/test_common.py +886754f39804a4f3f7157124b21ce08d9bad83d156dcd81bc942521bb42c4a29 tests/test_common.py 899bc085e96d68f8a8cbe0d7e55863e98ef37b73ab0e4234f7d969e31ea2d23a tests/test_comparison_json.py 7b72d4f850bbd059b8e95fceb45a58470354cb7270c99b0e9981aaa189af20d1 tests/test_comparison.py a7c3cf9f7820f377ebfdecf9383ebebc2932dd4a2a531a2b4496071f9d973c1c tests/test_compat.py 75357efd92f3f57cc05244a0f40985108077479fd192caaaa81e14f61c13783d tests/test_convert.py -2bd0faeaf7db1d73dd0caab3bde9900fdaa1f38fd736a6e238cd56ff9bc67b66 tests/test_databases_enum.py +6e3c08e1f76dd6c782d2ddc505b4e1a751b381c88ad91f79a95bf49f9c28a28f tests/test_databases_enum.py c17544be5e945dc8c4fbb5c3b922da8eceec30b0fb239c32fb5f40e1660a197f tests/test_datafiles.py 9c240d4f796e56376374d4ce46f358ceb7d48cc6a7427760c5bfb89ff01cb545 tests/test_datatypes.py -8a1edb6dbc000e412ba5cc598e024b669fc76ec0a8fc32136808e6325a018f70 tests/test_dbms_enum.py +7cf63166206d543ff4423e1b5bda3ec3212805b0aeaf95d877117df7eb79c8ec tests/test_dbms_enum.py 3804eb2d730220360f9dc07d5994eb64e9f65acf3b0d8648df8df2a2177ba8fd tests/test_decodepage.py 180e5fd3f75fadf7ac1135f99797314e2cf1f8ae6dced02edfb18ccba43c0148 tests/test_deps.py fa85881aa8d082a65aeacb2b03fcb5d2abb1daa9a02ee24ff048d54fbc904b90 tests/test_dialectdbms.py -e40a49cfa73c45b3c3c6d1d1d00738861e270cb7a07b28f5a5356f9c7c800cf2 tests/test_dialect.py +41bb0981cb7372753dbaa328c8be3678d328b736e6b97f7bd2573b465753af01 tests/test_dialect.py 993a2d4d87c4fbaf261663b069629acc95ee4405aa0c42cf5a8f39649fdb0fff tests/test_dicts.py -7f9180a53dbf0bb3e52801fdbfffd31f365a0bff77bf90e58d2ef63a0c23026f tests/test_dns_engine.py +62a4386524d0ef269cba3bd6dcadc5a2a11c0d2bdd198773b79bcd8589324328 tests/test_dns_engine.py ec58ba0849d90d2bb7580fe2b8b96cd8299ddfc25f14dc27d9de9d41f152c78a tests/test_dns_server.py -4556bb0bfa6fcd5b98552426c57c99942ee8274eaefec7c316fd64247e4fcd6a tests/test_dump_format.py -9cd5841349bc4db818658d12184929a96f7f279eff1f53ad18a54dbefbd6b276 tests/test_dump_jsonl.py +3dc788fd3adba8b6f766281e0a50025b1ee9150d80ab9a738c6c43f2eaf805b3 tests/test_dump_format.py +118d1987861ed0df978474329adce8c23009b3964210c13fbaf667e0019bbd15 tests/test_dump_jsonl.py 2bbe4b01f79992cfa8884651fc0a28dbd0e3abb0cbea9eb7eadf1f98ca3c3420 tests/test_encoding.py -fe1211ce43a51cd8ec7dd3395aafda8d7313ff60e2ef013072ce9fa49ca4a242 tests/test_entries.py -bb6991260a994fcbe79e05febaa34affd5631d02299fbc626820addd5f6ea4f4 tests/test_error_engine.py -26730151abea598f193131c5d64ef92b531941972f3d6236f9951c3116030b1c tests/test_filesystem.py -16fba97cba6afe8af11aa30bcc4266f53b00f2530161e010af10b51db1509703 tests/test_fingerprint.py -20844dfc758e99b2f757906c51ef32aca0f699283ec5aa629158d3dc0fd279ea tests/test_generic_takeover.py -f1f38f8b8ca667caadcb027d1a20eb895be4ef0935511114db235e66903bb463 tests/test_graphql.py +f4c54b19a294bf392b23dc627781d50894c8e44ca4fe5d7315c98984a3e196a4 tests/test_entries.py +ed7df24ce154e4cbb4462874a38202794664d12b083845bbee9f80481ec9cf52 tests/test_error_engine.py +950527f0abaffdc031e34336a870cd0f89723ee8589bf77763f5978f5e4c0be8 tests/test_filesystem.py +31fa778c7ee318169961d04ea7b93afc539c24b4114a6a3eaf45698fef57bb4b tests/test_fingerprint.py +abb6eef3d2d08b87b6210dde6dd1333d39da64f5abe5574240fa47efce7528f3 tests/test_generic_takeover.py +b7d59fe68af29d47dda1d7ad77e9b5c91ed50e9efbb976e62e0dc67dd11b3e17 tests/test_graphql.py 50b71422ee91b9a4864f4d5ce6c9bdf169dc5f57ed1db05c152eb010c282136b tests/test_gui_helpers.py 92648f2fe81e22c5726b198bbbda14961cd4d3294a0d9139dcea808b324142ac tests/test_har.py cc7677bc6c568c395112c1aa7d01e1d664e4d5940c86cb4d44987172864bae6f tests/test_hash_crack.py 0336c875dd2b6554bff6eafd746229e38c69ca8070cd933d45cf27c82ef3e05f tests/test_hashdb.py c04e8358fb6df45f69f2f26435c971acde280535bf304e84d30cf2681158c6a7 tests/test_hash.py -d539d0ae758b5bb91e314ab82ab4fe03d6fb2f8b377d16aefa6d7d1d77a7d5a9 tests/test_identifiers_output.py -5372270b7ed82b62f273c2e9bd1f7ecd8605371e66cd0ad70663762cb08d42f1 tests/test_inference_engine.py -0fc7bd9bae4fbd09f51027780b7a8e72eab73810dccdfdf87ed9e489e6e671c9 tests/test_ldap.py -caa06fed7323b2bb6d0f2443ce343de94f75bf8ad012c055d5e07741d908ebad tests/test_misc.py -790b78c600b61eb0bdd6e07e14b1db3eb2ddd5fc5d4edb9e975f85ced38558c7 tests/test_nosql.py +b23bf934dafe54c241761517a7b8c139159aa4b941db10832a626a51fea81e35 tests/test_http2.py +139dcedb9093eb0404ce497549eb6ab7e83ae1e70df8eb42da74ab5a3e7d2a85 tests/test_identifiers_output.py +0a5736b86a47e66d47d44ecf7b8c7531417453fc3e976cd64e9865d3afba78f4 tests/test_inference_engine.py +22629df783f75a88c2a30ffb8e37af095e761b771322fefbd69bdd7a5c9348fb tests/test_ldap.py +571d7761d60a2919985d065893af68eac5d12286f491eaba434c1d8587f913a0 tests/test_library.py +d2f701f4c3a8621b937ddd322343df91e102af5424ab58675dec4dc7781035b4 tests/test_misc.py +2f6d2270b26f68b3c9b511364c57eb5eb7b010ff716346fe2b320df30280f94c tests/test_nosql.py 88a8c7ce0ba0ca721dffbcf9351cd07f7e471ad2fe667a10608c18952b09868d tests/test_openapi_drift.py +a0d173bb595ffbd2b49ee7fb1519d9898aefc262f2565923c4fe41bbc06f57e0 tests/test_openapi.py 6e63ed05db0490148d1c8428d785a23b0d5d5a0f566cd397c9c4a8fe8a6ed7dc tests/test_option.py -cde0bea1263ae857561f91ed2bd515e972b716743f017d31b1718a8546c72759 tests/test_pagecontent.py -7554a918309cf0f2cd8a63a3bb7659708f13beffbcd5ce498ece9f9167d55c97 tests/test_parse_modules.py -0d52bf4b96eea2330553fdf7f875ed571e596d2f7a4b3648a2b53e44666f0c70 tests/test_payload_marking.py +fc698e34b53e95c2cc190dadb087d5873711202b2c5eef9db9fc6de5f9c88063 tests/test_pagecontent.py +7297b791aed9278d9252a3ade688e67796eb5c9cc4d6b29e1d2b56d83aa20295 tests/test_parse_modules.py +6cfe189c49749a2e0bc551173f5d2c4eb5aad8cbb1f9584ecc60958b9a842725 tests/test_payload_marking.py 6bfc8201724078bd9d6d559916ef73c9ff97e19b0f2948f37e588a49b027795f tests/test_payloads_structure.py d6ffa83bd56ae98e7f55307b72dd7ea4802bccea9a85bb8f062619fb0a88913e tests/test_progress.py -a6d013104601c0414628aff3d8b5b69bee3e6733781d8f8da880457d8b44bd3a tests/test_property.py -c4c6f500bb71c3e430da343a49e8c8b8b3c919f438b6e6130597ce68dd856487 tests/test_purge.py +2d135eba3ad0fd091962d84742ebf67314fd3f89dcaaa1252b3e3d76fae7c9fd tests/test_property.py +9a0915f34e1f80a2989238fcce940734cd886020c549711a8444e7ee62eab812 tests/test_purge.py 2dfefb4bfaee3868152835502ec43da317c4f274b1d55cd2ef21e4f7390c9bea tests/test_replication.py -67a5241aeebc20eb1c20cfc490422a59af5179040824e5731bd785db2e6bf750 tests/test_report.py +427a543e17dfede42b9fbccc916fa0aecd93fb7bfb5c280de4c2bca87c5d8de5 tests/test_report.py 4723d3bdf9623a49972e1d7378168ae8efbeaa31fb11c35d83bb40cc135fa0a8 tests/test_request_basic.py cec98d72992c0799229a780fa7f0d7f3fb01ec2d708187ce0e4a05c8612f291b tests/test_safe2bin.py -5b6ce95dddbd07d0126224f4f066643938476e536e18b700ea5d916e1052a715 tests/test_search_enum.py +575ebc336be598858279094072cde1ac9b124109cd7397bd805decd1b0a616d4 tests/test_search_enum.py a1c6cda1e5b483f61e6a4f8ddd0b06a15ddaa3fd2119bfb9dbd9cc970d7a751d tests/test_settings_regex.py -29d0278e3718b0fee422d3f6bb85ca02560138d48cd76f9fe1f35ac19d96071b tests/test_sgmllib.py +295581435c4dbf7fe6c291bbf0163c43ccb6ee610e6f3f2609bfeed734c91a1a tests/test_sgmllib.py d3d991331096e16e5019de3d652e9fff92c09bd9f97c50b1c2c3ceb0ed49b17e tests/test_sqlparse.py -412a61053c2531cc0380b34dfd01d52bd118f6a6473728c069c467054c7e3c8e tests/test_ssti.py +19e1e17d7a94e42cf75a37901c3468c79807a2d423bd1988b6f4a2566b864f3b tests/test_ssti.py 8bcbf1091134dd0a62f6201f8b3645ed87b5ff2f7ba40a87231a29dac412591f tests/test_strings.py 8f1c5f0f337ecd26d35c5551060034e0aa33a62cce5385fc1227fdc485f6383e tests/test_tamper.py -67472bd71c20782cc0f738e2c2e674c29d6985669e14d15b69baef7d0e33de62 tests/test_target_parsing.py -b3e13febe9e0ff6f97334f2868655bfdbaa18755e464a6dc4c6d424f513bad02 tests/test_targeturl.py -0e644bb7b25c183d0d689ea7be542d7a2ce780cc68067f89afb2ee095a79f762 tests/test_techniques.py -639851dc68f62b559b200b09c308e64e453f414969940005bac75dc0ab07a6b6 tests/test_texthelpers.py -f49bcce1df533ffa1acfd02af43faf6687b21eebda9362ceb1e5871b8cb37fd4 tests/test_threads.py -708b3c040f8b677a84020dd6f7c4242f77260b3c6d2697fe8189e1881b0e1365 tests/test_union_engine.py +b2b3a00254301e5e880e2e77351ebc47eed2c5280477915feedf780ea8cbd34f tests/test_target_parsing.py +cc67045d60472913eca574d601077e5111a95f4563c66caf361b8deaa2bed03c tests/test_targeturl.py +d7d8aaba1d22ee690c8da2c6e28cea0ab45b0d7a6915a5ae7f581c44d7121aab tests/test_techniques.py +61769e1d6c4429659ebfb2de696b506821e3c6f3ca81b4318ce790b9553ca6a3 tests/test_texthelpers.py +095a889a6274f0f8e437bf9a23e4b073ab6c4b60aba582e6d1e2099645f1d883 tests/test_threads.py +8d23cb42cde68e0da2c4b47db367139d0c53363fef7493ae70b7f6636a1bbbc7 tests/test_union_engine.py 48b0ae4abe0fdde8ce4975c5cbf4c3514a2815021cb2e3a490a189bea5edfe78 tests/test_unpickle_security.py 4b646f513c6da1e33200184ed6eabe0aa345eb2e2a19598dc123e191168591bf tests/test_urls.py -eca021208e388b4d14c53f1e9f8a6e7d685e54ba572fb2a8487e6b620a20bcb5 tests/test_users_enum.py -045f05f958100adc883b3f56613c5f8002dd19d0752225397a1f771775cb2779 tests/_testutils.py +b03689c4dcca0e88a62a88784c61418f963c031d338a357dcc223560c8f9bd22 tests/test_users_enum.py +729b3a5e00fff2e2b6c3acd3fd3e970ac1985c0a6ad1829b23c4099bd409afa1 tests/_testutils.py 2364db35025a53ea4e5a0a80c034997642785f7e6d1566d0d0f1db959fe3c82e tests/test_utils.py 93ef9944effc62d4f744c57bd643137c90fd92205c6a6cbe891e0e99efb80a7f tests/test_wafbypass.py 81bb6d7449f224fa337734ae361c1a340bf9a51768a854d6a1a6e718ed1263ca tests/test_wordlist.py -2698060e7f001e054e345512ce95be458d9902b913afa769398b53145475738a tests/test_xpath.py +9d6dd551b751ab38200ab190c744ec0a9afa798b37f83b0078a4325ab3f80aec tests/test_xpath.py 55eaefc664bd8598329d535370612351ec8443c52465f0a37172ea46a97c458a thirdparty/ansistrm/ansistrm.py e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 thirdparty/ansistrm/__init__.py f597b49ef445bfbfb8f98d1f1a08dcfe4810de5769c0abfab7cdce4eebbfcae7 thirdparty/beautifulsoup/beautifulsoup.py @@ -725,8 +730,6 @@ e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 thirdparty/mag 4d89a52f809c28ce1dc17bb0c00c775475b8ce01c2165942877596a6180a2fd8 thirdparty/magic/magic.py e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 thirdparty/multipart/__init__.py 2574a2027b4a63214bad8bd71f28cac66b5748159bf16d63eb2a3e933985b0a5 thirdparty/multipart/multipartpost.py -ef70b88cc969a3e259868f163ad822832f846196e3f7d7eccb84958c80b7f696 thirdparty/odict/__init__.py -9a8186aeb9553407f475f59d1fab0346ceab692cf4a378c15acd411f271c8fdb thirdparty/odict/ordereddict.py 3739db672154ad4dfa05c9ac298b0440f3f1500c6a3697c2b8ac759479426b84 thirdparty/pydes/__init__.py 4c9d2c630064018575611179471191914299992d018efdc861a7109f3ec7de5e thirdparty/pydes/pyDes.py c51c91f703d3d4b3696c923cb5fec213e05e75d9215393befac7f2fa6a3904df thirdparty/six/__init__.py diff --git a/data/xml/queries.xml b/data/xml/queries.xml index 449b6cb9be0..61dc69d9ed5 100644 --- a/data/xml/queries.xml +++ b/data/xml/queries.xml @@ -35,8 +35,8 @@ - - + + diff --git a/doc/THIRD-PARTY.md b/doc/THIRD-PARTY.md index f2c10e27255..b20e1630996 100644 --- a/doc/THIRD-PARTY.md +++ b/doc/THIRD-PARTY.md @@ -270,8 +270,6 @@ be bound by the terms and conditions of this License Agreement. Copyright (C) 2024, Marcel Hellkamp. * The `identYwaf` library located under `thirdparty/identywaf/`. Copyright (C) 2019-2021, Miroslav Stampar. -* The `ordereddict` library located under `thirdparty/odict/`. - Copyright (C) 2009, Raymond Hettinger. * The `six` Python 2 and 3 compatibility library located under `thirdparty/six/`. Copyright (C) 2010-2024, Benjamin Peterson. * The `Termcolor` library located under `thirdparty/termcolor/`. diff --git a/lib/controller/controller.py b/lib/controller/controller.py index 67b9278b1be..ba27f49aad1 100644 --- a/lib/controller/controller.py +++ b/lib/controller/controller.py @@ -561,9 +561,10 @@ def start(): checkNullConnection() if (len(kb.injections) == 0 or (len(kb.injections) == 1 and kb.injections[0].place is None)) and (kb.injection.place is None or kb.injection.parameter is None): - if not any((conf.string, conf.notString, conf.regexp)) and PAYLOAD.TECHNIQUE.BOOLEAN in conf.technique: - # NOTE: this is not needed anymore, leaving only to display - # a warning message to the user in case the page is not stable + if not any((conf.string, conf.notString, conf.regexp)) and any(_ in conf.technique for _ in (PAYLOAD.TECHNIQUE.BOOLEAN, PAYLOAD.TECHNIQUE.UNION)): + # NOTE: besides the not-stable warning, this marks dynamic content for removal, which + # UNION column-count detection relies on too (it compares pages) - so it must run when + # UNION is tested even if BOOLEAN is excluded (e.g. '--technique=U' on a dynamic page) checkStability() # Do a little prioritization reorder of a testable parameter list diff --git a/lib/core/agent.py b/lib/core/agent.py index ec781a43e58..ad67ade14ae 100644 --- a/lib/core/agent.py +++ b/lib/core/agent.py @@ -958,12 +958,19 @@ def _collate(value): if not infoFile: query = _collate(query) + # A fuzzy-discovered per-column type template (kb.unionTemplate, e.g. ['1234', '%s', '5678']) + # forces type-compatible fillers on strict DBMSes (e.g. Apache Derby, which rejects bare NULL + # and demands UNION column-type parity); '%s' marks the slot carrying the injected expression. + template = kb.unionTemplate if isinstance(kb.unionTemplate, (list, tuple)) and len(kb.unionTemplate) == count else None + for element in xrange(0, count): if element > 0: unionQuery += ',' if conf.uValues and conf.uValues.count(',') + 1 == count: unionQuery += conf.uValues.split(',')[element] + elif template is not None: + unionQuery += query if template[element] == "%s" else template[element] elif element == position: unionQuery += query else: @@ -985,7 +992,9 @@ def _collate(value): if element > 0: unionQuery += ',' - if element == position: + if template is not None: + unionQuery += _collate(multipleUnions) if template[element] == "%s" else template[element] + elif element == position: unionQuery += _collate(multipleUnions) else: unionQuery += char diff --git a/lib/core/common.py b/lib/core/common.py index ec7db6ff96b..9a86af8cda3 100644 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -200,7 +200,7 @@ from thirdparty.clientform.clientform import ParseError from thirdparty.colorama.initialise import init as coloramainit from thirdparty.magic import magic -from thirdparty.odict import OrderedDict +from collections import OrderedDict from thirdparty.six import unichr as _unichr from thirdparty.six.moves import collections_abc as _collections from thirdparty.six.moves import configparser as _configparser @@ -2099,7 +2099,9 @@ def getFileType(filePath): desc = getText(desc) if desc == getText(magic.MAGIC_UNKNOWN_FILETYPE): - content = openFile(filePath, "rb", encoding=None).read() + _ = openFile(filePath, "rb", encoding=None) + content = _.read() + _.close() try: content.decode() @@ -2624,6 +2626,17 @@ def initCommonOutputs(): if line not in kb.commonOutputs[key]: kb.commonOutputs[key].add(line) + # The curated '--common-tables'/'--common-columns' brute-force wordlists are far larger and much + # more app-focused than the built-in [Tables]/[Columns] prediction sections (which are mostly + # system objects), so fold them into the good-samaritan prediction to raise its real-world hit rate. + # The mechanism only reorders the charset, so extra coverage never penalizes a miss. + for _key, _path in (("Tables", paths.COMMON_TABLES), ("Columns", paths.COMMON_COLUMNS)): + try: + for _ in getFileItems(_path): + kb.commonOutputs.setdefault(_key, set()).add(_) + except SqlmapSystemException: + pass + def getFileItems(filename, commentPrefix='#', unicoded=True, lowercase=False, unique=False): """ Returns newline delimited items contained inside file diff --git a/lib/core/datatype.py b/lib/core/datatype.py index 11b45878a6f..f667c0cd9e1 100644 --- a/lib/core/datatype.py +++ b/lib/core/datatype.py @@ -8,7 +8,7 @@ import copy import threading -from thirdparty.odict import OrderedDict +from collections import OrderedDict from thirdparty.six.moves import collections_abc as _collections class AttribDict(dict): diff --git a/lib/core/dump.py b/lib/core/dump.py index c81f5251916..8b8feec0bd7 100644 --- a/lib/core/dump.py +++ b/lib/core/dump.py @@ -62,7 +62,7 @@ from lib.utils.safe2bin import safechardecode from thirdparty import six from thirdparty.magic import magic -from thirdparty.odict import OrderedDict +from collections import OrderedDict class Dump(object): """ diff --git a/lib/core/enums.py b/lib/core/enums.py index 479b9f6826b..727eaed88fc 100644 --- a/lib/core/enums.py +++ b/lib/core/enums.py @@ -180,6 +180,8 @@ class HASH(object): MYSQL = r'(?i)\A\*[0-9a-f]{40}\Z' MYSQL_OLD = r'(?i)\A(?![0-9]+\Z)[0-9a-f]{16}\Z' POSTGRES = r'(?i)\Amd5[0-9a-f]{32}\Z' + POSTGRES_SCRAM = r'\ASCRAM-SHA-256\$\d+:[A-Za-z0-9+/]+={0,2}\$[A-Za-z0-9+/]+={0,2}:[A-Za-z0-9+/]+={0,2}\Z' + MYSQL_SHA2 = r'\A\$mysql\$A\$[0-9A-Fa-f]{3}\*[0-9A-Fa-f]{40}\*[0-9A-Fa-f]{86}\Z' MSSQL = r'(?i)\A0x0100[0-9a-f]{8}[0-9a-f]{40}\Z' MSSQL_OLD = r'(?i)\A0x0100[0-9a-f]{8}[0-9a-f]{80}\Z' MSSQL_NEW = r'(?i)\A0x0200[0-9a-f]{8}[0-9a-f]{128}\Z' @@ -192,6 +194,8 @@ class HASH(object): SHA384_GENERIC = r'(?i)\A[0-9a-f]{96}\Z' SHA512_GENERIC = r'(?i)\A(0x)?[0-9a-f]{128}\Z' CRYPT_GENERIC = r'\A(?!\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\Z)(?![0-9]+\Z)[./0-9A-Za-z]{13}\Z' + SHA256_UNIX_CRYPT = r'\A\$5\$(?:rounds=\d+\$)?[./0-9A-Za-z]{1,16}\$[./0-9A-Za-z]{43}\Z' + SHA512_UNIX_CRYPT = r'\A\$6\$(?:rounds=\d+\$)?[./0-9A-Za-z]{1,16}\$[./0-9A-Za-z]{86}\Z' JOOMLA = r'\A[0-9a-f]{32}:\w{32}\Z' PHPASS = r'\A\$[PHQS]\$[./0-9a-zA-Z]{31}\Z' APACHE_MD5_CRYPT = r'\A\$apr1\$.{1,8}\$[./a-zA-Z0-9]+\Z' @@ -205,6 +209,13 @@ class HASH(object): SSHA512 = r'\A\{SSHA512\}[a-zA-Z0-9+/]+={0,2}\Z' DJANGO_MD5 = r'\Amd5\$[^$]*\$[0-9a-f]{32}\Z' DJANGO_SHA1 = r'\Asha1\$[^$]*\$[0-9a-f]{40}\Z' + DJANGO_PBKDF2_SHA256 = r'\Apbkdf2_sha256\$\d+\$[^$]+\$[A-Za-z0-9+/]+={0,2}\Z' + WERKZEUG_PBKDF2 = r'\Apbkdf2:(?:sha1|sha256|sha512):\d+\$[^$]+\$[0-9a-f]+\Z' + WERKZEUG_SCRYPT = r'\Ascrypt:\d+:\d+:\d+\$[^$]+\$[0-9a-f]+\Z' + BCRYPT = r'\A\$2[abxy]\$\d{2}\$[./A-Za-z0-9]{53}\Z' + WORDPRESS_BCRYPT = r'\A\$wp\$2[abxy]\$\d{2}\$[./A-Za-z0-9]{53}\Z' + ARGON2 = r'\A\$argon2(?:id|i|d)\$v=\d+\$m=\d+,t=\d+,p=\d+\$[A-Za-z0-9+/]+={0,2}\$[A-Za-z0-9+/]+={0,2}\Z' + ASPNET_IDENTITY = r'\AAQAAAA[A-Za-z0-9+/]{76}==\Z' MD5_BASE64 = r'\A[a-zA-Z0-9+/]{22}==\Z' SHA1_BASE64 = r'\A[a-zA-Z0-9+/]{27}=\Z' SHA256_BASE64 = r'\A[a-zA-Z0-9+/]{43}=\Z' diff --git a/lib/core/option.py b/lib/core/option.py index f7d26907483..f828e4cf916 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -492,6 +492,70 @@ def _setBulkMultipleTargets(): warnMsg = "no usable links found (with GET parameters)" logger.warning(warnMsg) +def _setOpenApiTargets(): + if not conf.openApiFile: + return + + from lib.parse.openapi import openApiTargets + + if conf.method: + warnMsg = "option '--method' will override the HTTP method(s) derived from the OpenAPI/Swagger specification" + logger.warning(warnMsg) + + # origin resolves a spec's relative 'servers' to absolute target URLs: an explicit '--openapi-base' + # (needed for a host-less local spec) or, when fetched by URL, the fetch URL itself. + origin = conf.openApiBase.rstrip('/') if conf.openApiBase else None + if re.match(r"(?i)\Ahttps?://", conf.openApiFile): + infoMsg = "fetching OpenAPI/Swagger specification from '%s'" % conf.openApiFile + logger.info(infoMsg) + from lib.request.connect import Connect as Request + content = Request.getPage(url=conf.openApiFile, raise404=True)[0] + if not origin: + match = re.match(r"(?i)(https?://[^/]+)", conf.openApiFile) + origin = match.group(1) if match else None + else: + conf.openApiFile = safeExpandUser(conf.openApiFile) + checkFile(conf.openApiFile) + infoMsg = "parsing OpenAPI/Swagger specification from '%s'" % conf.openApiFile + logger.info(infoMsg) + content = openFile(conf.openApiFile).read() + + try: + targets = openApiTargets(content, origin) + except ValueError as ex: + errMsg = "unable to parse the OpenAPI/Swagger specification ('%s')" % getSafeExString(ex) + raise SqlmapSyntaxException(errMsg) + + if re.search(r"(?i)securitySchemes|securityDefinitions", content) and not any((conf.authType, conf.authCred, conf.authFile)) and not any((_[0] or "").lower() == HTTP_HEADER.AUTHORIZATION.lower() for _ in (conf.httpHeaders or [])): + warnMsg = "the OpenAPI/Swagger specification declares authentication (security schemes) but no credentials were provided. " + warnMsg += "If the API requires authentication, requests are likely to be rejected. Provide credentials with " + warnMsg += "'--auth-type'/'--auth-cred' or a header (e.g. --headers=\"Authorization: Bearer ...\")" + logger.warning(warnMsg) + + before = len(kb.targets) # openapi carries per-target bodies -> no conf.data fallback + mutating = 0 + for url, method, data, headers in targets: + if conf.scope and not re.search(conf.scope, url, re.I): + continue + if method not in ("GET", "HEAD", "OPTIONS"): + mutating += 1 + kb.targets.add((url, method, data, conf.cookie, tuple(headers) if headers else None)) + + added = len(kb.targets) - before + if added: + conf.multipleTargets = True + infoMsg = "derived %d target(s) from the OpenAPI/Swagger specification" % added + logger.info(infoMsg) + if mutating: + warnMsg = "%d of the derived target(s) use state-changing HTTP methods (e.g. POST/PUT/PATCH/DELETE). " % mutating + warnMsg += "Scanning them may create, modify or delete server-side data" + logger.warning(warnMsg) + else: + warnMsg = "no usable targets derived from the OpenAPI/Swagger specification" + if not conf.openApiBase: + warnMsg += " (if it uses relative 'servers', provide a base with '--openapi-base' or fetch it by URL)" + logger.warning(warnMsg) + def _findPageForms(): if not conf.forms or conf.crawlDepth: return @@ -870,6 +934,15 @@ def _setTamperingFunctions(): warnMsg += "a good idea" logger.warning(warnMsg) + # tamper scripts rewrite SQL injection payloads; the self-contained non-SQL engines + # (--graphql/--nosql/--ldap/--xpath/--ssti) do not run payloads through the tampering hook, so + # warn instead of silently ignoring the user's '--tamper' + if kb.tamperFunctions and any((conf.graphql, conf.nosql, conf.ldap, conf.xpath, conf.ssti)): + engine = next(_ for _ in ("graphql", "nosql", "ldap", "xpath", "ssti") if conf.get(_)) + warnMsg = "tamper scripts are applied to SQL injection payloads only and " + warnMsg += "will be ignored by the '--%s' engine" % engine + logger.warning(warnMsg) + if resolve_priorities and priorities: priorities.sort(key=functools.cmp_to_key(lambda a, b: cmp(a[0], b[0])), reverse=True) kb.tamperFunctions = [] @@ -1249,10 +1322,12 @@ def _setHTTPHandlers(): handlers.append(_urllib.request.HTTPCookieProcessor(conf.cj)) # Reference: http://www.w3.org/Protocols/rfc2616/rfc2616-sec8.html - # Note: persistent (Keep-Alive) connections are used by default; '--no-keep-alive' opts out, - # and they are automatically disabled when incompatible (HTTP(s) proxy, authentication methods, - # or chunked transfer-encoding of the request body - handled by a dedicated, non-pooling handler) - conf.keepAlive = not conf.noKeepAlive and not conf.proxy and not conf.authType and not conf.chunked + # Note: persistent (Keep-Alive) connections are used by default (including through an HTTP(s) + # proxy - the keep-alive handler pools the proxy socket for plain HTTP and the CONNECT-tunnelled + # socket per origin for HTTPS); '--no-keep-alive' opts out, and they are automatically disabled + # when incompatible (authentication methods, or chunked transfer-encoding of the request body - + # handled by a dedicated, non-pooling handler) + conf.keepAlive = not conf.noKeepAlive and not conf.authType and not conf.chunked if conf.keepAlive: # persistent connections for both HTTP and HTTPS; the keep-alive HTTPS @@ -1261,8 +1336,8 @@ def _setHTTPHandlers(): handlers.remove(httpsHandler) handlers.append(keepAliveHandler) handlers.append(keepAliveHandlerHTTPS) - elif not conf.noKeepAlive and (conf.proxy or conf.authType or conf.chunked): - reason = "an HTTP(s) proxy" if conf.proxy else ("authentication methods" if conf.authType else "chunked transfer-encoding") + elif not conf.noKeepAlive and (conf.authType or conf.chunked): + reason = "authentication methods" if conf.authType else "chunked transfer-encoding" debugMsg = "persistent (Keep-Alive) connections were disabled (incompatible with %s)" % reason logger.debug(debugMsg) @@ -1841,7 +1916,7 @@ def _cleanupOptions(): if conf.tmpPath: conf.tmpPath = ntToPosixSlashes(normalizePath(conf.tmpPath)) - if any((conf.googleDork, conf.logFile, conf.bulkFile, conf.forms, conf.crawlDepth, conf.stdinPipe)): + if any((conf.googleDork, conf.logFile, conf.bulkFile, conf.forms, conf.crawlDepth, conf.stdinPipe, conf.openApiFile)): conf.multipleTargets = True if conf.optimize: @@ -2717,8 +2792,8 @@ def _basicOptionValidation(): errMsg += "'SQLMAP_UNSAFE_EVAL=1' to be explicitly set" raise SqlmapSystemException(errMsg) - if conf.chunked and not any((conf.data, conf.requestFile, conf.forms)): - errMsg = "switch '--chunked' requires usage of (POST) options/switches '--data', '-r' or '--forms'" + if conf.chunked and not any((conf.data, conf.requestFile, conf.forms, conf.openApiFile)): + errMsg = "switch '--chunked' requires usage of (POST) options/switches '--data', '-r', '--forms' or '--openapi'" raise SqlmapSyntaxException(errMsg) if conf.api and not conf.configFile: @@ -3011,7 +3086,7 @@ def init(): parseTargetDirect() - if any((conf.url, conf.logFile, conf.bulkFile, conf.requestFile, conf.googleDork, conf.stdinPipe)): + if any((conf.url, conf.logFile, conf.bulkFile, conf.requestFile, conf.googleDork, conf.stdinPipe, conf.openApiFile)): _setHostname() _setHTTPTimeout() _setHTTPExtraHeaders() @@ -3027,6 +3102,7 @@ def init(): _doSearch() _setStdinPipeTargets() _setBulkMultipleTargets() + _setOpenApiTargets() _checkTor() _setCrawler() _findPageForms() diff --git a/lib/core/optiondict.py b/lib/core/optiondict.py index 21c6cfa37fd..8ead4860487 100644 --- a/lib/core/optiondict.py +++ b/lib/core/optiondict.py @@ -19,6 +19,8 @@ "sessionFile": "string", "googleDork": "string", "configFile": "string", + "openApiFile": "string", + "openApiBase": "string", }, "Request": { diff --git a/lib/core/settings.py b/lib/core/settings.py index 0c7de36ad88..4600eb3d176 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -20,7 +20,7 @@ from thirdparty import six # sqlmap version (...) -VERSION = "1.10.7.4" +VERSION = "1.10.7.21" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) @@ -141,6 +141,9 @@ # Upper threshold for starting the fuzz(y) UNION test FUZZ_UNION_MAX_COLUMNS = 10 +# Maximum number of probe requests the fuzz(y) UNION test may issue (bounds its otherwise exponential type-combination search when run automatically) +FUZZ_UNION_MAX_REQUESTS = 80 + # Regular expression used for recognition of generic maximum connection messages MAX_CONNECTIONS_REGEX = r"\bmax.{1,100}\bconnection" @@ -713,7 +716,10 @@ CRAWL_EXCLUDE_EXTENSIONS = frozenset(("3ds", "3g2", "3gp", "7z", "DS_Store", "a", "aac", "accdb", "access", "adp", "ai", "aif", "aiff", "apk", "ar", "asf", "au", "avi", "bak", "bin", "bk", "bkp", "bmp", "btif", "bz2", "c", "cab", "caf", "cfg", "cgm", "cmx", "com", "conf", "config", "cpio", "cpp", "cr2", "cue", "dat", "db", "dbf", "deb", "debug", "djvu", "dll", "dmg", "dmp", "dng", "doc", "docx", "dot", "dotx", "dra", "dsk", "dts", "dtshd", "dvb", "dwg", "dxf", "dylib", "ear", "ecelp4800", "ecelp7470", "ecelp9600", "egg", "elf", "env", "eol", "eot", "epub", "error", "exe", "f4v", "fbs", "fh", "fla", "flac", "fli", "flv", "fpx", "fst", "fvt", "g3", "gif", "go", "gz", "h", "h261", "h263", "h264", "ico", "ief", "img", "ini", "ipa", "iso", "jar", "java", "jpeg", "jpg", "jpgv", "jpm", "js", "jxr", "ktx", "lock", "log", "lvp", "lz", "lzma", "lzo", "m3u", "m4a", "m4v", "mar", "mdb", "mdi", "mid", "mj2", "mka", "mkv", "mmr", "mng", "mov", "movie", "mp3", "mp4", "mp4a", "mpeg", "mpg", "mpga", "msi", "mxu", "nef", "npx", "nrg", "o", "oga", "ogg", "ogv", "old", "otf", "ova", "ovf", "pbm", "pcx", "pdf", "pea", "pgm", "pic", "pid", "pkg", "png", "pnm", "ppm", "pps", "ppt", "pptx", "ps", "psd", "py", "pya", "pyc", "pyo", "pyv", "qt", "rar", "ras", "raw", "rb", "rgb", "rip", "rlc", "rs", "run", "rz", "s3m", "s7z", "scm", "scpt", "service", "sgi", "shar", "sil", "smv", "so", "sock", "socket", "sqlite", "sqlitedb", "sub", "svc", "swf", "swo", "swp", "sys", "tar", "tbz2", "temp", "tga", "tgz", "tif", "tiff", "tlz", "tmp", "toast", "torrent", "ts", "ttf", "uvh", "uvi", "uvm", "uvp", "uvs", "uvu", "vbox", "vdi", "vhd", "vhdx", "viv", "vmdk", "vmx", "vob", "vxd", "war", "wav", "wax", "wbmp", "wdp", "weba", "webm", "webp", "whl", "wm", "wma", "wmv", "wmx", "woff", "woff2", "wvx", "xbm", "xif", "xls", "xlsx", "xlt", "xm", "xpi", "xpm", "xwd", "xz", "yaml", "yml", "z", "zip", "zipx")) # Patterns often seen in HTTP headers containing custom injection marking character '*' -PROBLEMATIC_CUSTOM_INJECTION_PATTERNS = r"(;q=[^;']+)|(\*/\*)" +# Note: the ';q=' quality-value class excludes '*' so a user-placed injection mark right after a +# quality value (e.g. 'Accept: ...;q=0.9*') is not swallowed (ref: #5357 - header injection was then +# missed on a GET lacking a Content-Length header, which is otherwise what forces params detection) +PROBLEMATIC_CUSTOM_INJECTION_PATTERNS = r"(;q=[^;'*]+)|(\*/\*)" # Template used for common table existence check BRUTE_TABLE_EXISTS_TEMPLATE = "EXISTS(SELECT %d FROM %s)" diff --git a/lib/core/target.py b/lib/core/target.py index b6666807fd3..a74955b717c 100644 --- a/lib/core/target.py +++ b/lib/core/target.py @@ -79,7 +79,7 @@ from lib.core.threads import getCurrentThreadData from lib.utils.hashdb import HashDB from thirdparty import six -from thirdparty.odict import OrderedDict +from collections import OrderedDict from thirdparty.six.moves import urllib as _urllib def _setRequestParams(): diff --git a/lib/parse/cmdline.py b/lib/parse/cmdline.py index b12f05281af..9081fe27d69 100644 --- a/lib/parse/cmdline.py +++ b/lib/parse/cmdline.py @@ -144,6 +144,12 @@ def cmdLineParser(argv=None): target.add_argument("-c", dest="configFile", help="Load options from a configuration INI file") + target.add_argument("--openapi", dest="openApiFile", + help="Derive targets from OpenAPI/Swagger (file/URL)") + + target.add_argument("--openapi-base", dest="openApiBase", + help="Base URL for a host-less OpenAPI/Swagger spec") + # Request options request = parser.add_argument_group("Request", "These options can be used to specify how to connect to the target URL") @@ -153,7 +159,7 @@ def cmdLineParser(argv=None): request.add_argument("-H", "--header", dest="header", help="Extra header (e.g. \"X-Forwarded-For: 127.0.0.1\")") - request.add_argument("--method", dest="method", + request.add_argument("-X", "--method", dest="method", help="Force usage of given HTTP method (e.g. PUT)") request.add_argument("--data", dest="data", @@ -523,7 +529,7 @@ def cmdLineParser(argv=None): enumeration.add_argument("-C", dest="col", help="DBMS database table column(s) to enumerate") - enumeration.add_argument("-X", dest="exclude", + enumeration.add_argument("--exclude", dest="exclude", help="DBMS database identifier(s) to not enumerate") enumeration.add_argument("-U", dest="user", @@ -1172,7 +1178,7 @@ def _format_action_invocation(self, action): else: args.stdinPipe = None - if not any((args.direct, args.url, args.logFile, args.bulkFile, args.googleDork, args.configFile, args.requestFile, args.updateAll, args.smokeTest, args.vulnTest, args.fpTest, args.apiTest, args.wizard, args.dependencies, args.purge, args.listTampers, args.hashFile, args.stdinPipe)): + if not any((args.direct, args.url, args.logFile, args.bulkFile, args.googleDork, args.configFile, args.requestFile, args.openApiFile, args.updateAll, args.smokeTest, args.vulnTest, args.fpTest, args.apiTest, args.wizard, args.dependencies, args.purge, args.listTampers, args.hashFile, args.stdinPipe)): errMsg = "missing a mandatory option (-d, -u, -l, -m, -r, -g, -c, --wizard, --shell, --update, --purge, --list-tampers or --dependencies). " errMsg += "Use -h for basic and -hh for advanced help\n" parser.error(errMsg) diff --git a/lib/parse/configfile.py b/lib/parse/configfile.py index a3bd3786b4f..88f91ce7828 100644 --- a/lib/parse/configfile.py +++ b/lib/parse/configfile.py @@ -75,6 +75,8 @@ def configFileParser(configFile): except Exception as ex: errMsg = "you have provided an invalid and/or unreadable configuration file ('%s')" % getSafeExString(ex) raise SqlmapSyntaxException(errMsg) + finally: + configFP.close() if not config.has_section("Target"): errMsg = "missing a mandatory section 'Target' in the configuration file" diff --git a/lib/parse/openapi.py b/lib/parse/openapi.py new file mode 100644 index 00000000000..996b5ece6a4 --- /dev/null +++ b/lib/parse/openapi.py @@ -0,0 +1,361 @@ +#!/usr/bin/env python + +""" +Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org) +See the file 'LICENSE' for copying permission +""" + +import json +import re + +from lib.core.common import getSafeExString +from lib.core.data import logger +from lib.core.enums import HTTP_HEADER +from lib.core.settings import CUSTOM_INJECTION_MARK_CHAR +from thirdparty import six +from thirdparty.six.moves.urllib.parse import quote as _quote + +try: + import yaml # optional (only needed for YAML specs) +except ImportError: + yaml = None + +# Best-effort extraction of concrete request targets from an OpenAPI (v3) / Swagger (v2) document. The +# document is treated as a request generator, NOT a contract to validate: for every operation a single +# concrete request is synthesized (base URL + filled path + example query/body from the schema) and any +# operation that cannot be built is skipped with a warning, so a loose/incomplete spec degrades gracefully. + +MAX_REF_DEPTH = 25 + +def _loadSpec(content): + try: + return json.loads(content) + except ValueError: + if yaml is None: + errMsg = "the provided OpenAPI/Swagger specification is not JSON and the optional " + errMsg += "'pyyaml' module (needed for YAML specifications) is not available" + raise ValueError(errMsg) + try: + return yaml.safe_load(content) + except Exception as ex: + raise ValueError("not valid JSON nor YAML (%s)" % getSafeExString(ex)) + +def _resolve(spec, node, seen=None, depth=0): + seen = seen or set() + if isinstance(node, dict) and "$ref" in node: + ref = node["$ref"] + if not isinstance(ref, six.string_types): # malformed '$ref' (non-string) -> treat as no ref + return {} + if ref in seen or depth > MAX_REF_DEPTH: + return {} + if not ref.startswith("#/"): + logger.warning("skipping external OpenAPI $ref '%s'" % ref) + return {} + seen = seen | set([ref]) + current = spec + for part in ref[2:].split('/'): + part = part.replace("~1", "/").replace("~0", "~") + if not isinstance(current, dict) or part not in current: + logger.warning("skipping dangling OpenAPI $ref '%s'" % ref) + return {} + current = current[part] + return _resolve(spec, current, seen, depth + 1) + return node + +EXAMPLE_MAX_DEPTH = 8 # request examples do not need deep nesting; caps runaway synthesis on large specs + +def _example(spec, schema, seen=None, depth=0, cache=None): + # 'cache' memoizes the synthesized example per $ref across the whole run - big real-world specs + # (Stripe/GitHub/k8s) reuse the same large schemas across thousands of operations, so without this + # the extraction is exponential. 'depth' caps recursion for deeply nested / self-referential schemas. + seen = seen or set() + if cache is None: + cache = {} + if depth > EXAMPLE_MAX_DEPTH: + return "1" + ref = schema.get("$ref") if isinstance(schema, dict) else None + if not isinstance(ref, six.string_types): # only a string $ref is a valid (hashable) cache key + ref = None + if ref is not None and ref in cache: + return cache[ref] + + schema = _resolve(spec, schema or {}, seen, depth) + if not isinstance(schema, dict): + return "1" + + value = None + if "example" in schema: + value = schema["example"] + elif "const" in schema: # JSON Schema 2020-12 (OpenAPI 3.1) + value = schema["const"] + elif "default" in schema: + value = schema["default"] + elif isinstance(schema.get("examples"), list) and schema["examples"]: + value = schema["examples"][0] + elif isinstance(schema.get("enum"), list) and schema["enum"]: + value = schema["enum"][0] + else: + combinator = next((_ for _ in ("allOf", "oneOf", "anyOf") if schema.get(_)), None) + if combinator: + if combinator == "allOf": + merged = {} + for sub in schema[combinator]: + part = _example(spec, sub, seen, depth + 1, cache) + if isinstance(part, dict): + merged.update(part) + value = merged if merged else _example(spec, schema[combinator][0], seen, depth + 1, cache) + else: + value = _example(spec, schema[combinator][0], seen, depth + 1, cache) + else: + _type = schema.get("type") + if isinstance(_type, list): # OpenAPI 3.1 allows a list of types (e.g. ["string", "null"]) + _type = next((_ for _ in _type if _ != "null"), None) + if _type == "object" or ("properties" in schema and not _type): + properties = schema.get("properties") + value = dict((name, _example(spec, sub, seen, depth + 1, cache)) for name, sub in (properties if isinstance(properties, dict) else {}).items()) + elif _type == "array": + value = [_example(spec, schema.get("items") or {}, seen, depth + 1, cache)] + elif _type in ("integer", "number"): + value = 1 + elif _type == "boolean": + value = True + elif _type == "string": + formats = {"uuid": "11111111-1111-1111-1111-111111111111", "date": "2020-01-01", "date-time": "2020-01-01T00:00:00Z", "email": "a@b.co", "byte": "MQ=="} + value = formats.get(schema.get("format"), "1") + else: + value = "1" + + if ref is not None: + cache[ref] = value + return value + +def _scalar(value): + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, (int, float)): + return str(value) + if isinstance(value, six.string_types): + return value + try: + return json.dumps(value) + except TypeError: # e.g. datetime.date from a YAML 'example: 2020-01-01' + return str(value) + +_NO_EXAMPLE = object() + +def _explicitExample(spec, container): + # a concrete 'example'/'examples' declared on a parameter or media-type object - preferred over a + # schema-synthesized value (real specs carry the canonical, validation-passing sample here). 'examples' + # is a map of name -> {"value": ...} (each entry possibly a $ref). + if not isinstance(container, dict): + return _NO_EXAMPLE + if container.get("example") is not None: # 'null' -> treat as absent, fall back to schema synthesis + return container["example"] + examples = container.get("examples") + if isinstance(examples, dict) and examples: + first = _resolve(spec, next(iter(examples.values()))) + if isinstance(first, dict) and first.get("value") is not None: + return first["value"] + return _NO_EXAMPLE + +def _noMark(text): + # strip any custom injection mark already present in a synthesized value so only the intentionally + # appended mark (if any) survives (avoids a stray/second injection point) + return text.replace(CUSTOM_INJECTION_MARK_CHAR, "") + +def _headerClean(text): + # remove characters that can not legally appear in an HTTP header name/value (CR, LF, NUL and other + # C0 controls) so a spec-supplied header can not inject extra headers or corrupt the request line + return re.sub(r"[\x00-\x1f\x7f]", "", text) + +_HEADER_NAME_RE = re.compile(r"\A[!#$%&'*+.^_`|~0-9A-Za-z-]+\Z") # RFC 7230 header field-name token (no spaces / ':' / separators) + +def _urlSafe(value, safe=""): + # percent-encode a synthesized value/name so it can not break the URL/body structure (spaces, '&', + # '=', '/', '?', '#', ...); py2/py3-safe (py2 urllib.quote needs bytes for non-ASCII). 'safe' keeps + # selected chars unescaped (e.g. "[]" for deep-object parameter names like filter[status]). + try: + return _quote(value.encode("utf-8") if isinstance(value, six.text_type) else str(value), safe=safe) + except Exception: + return value + +def _baseUrl(spec, origin=None, servers=None): + # defensive throughout: a hostile/loose spec must not crash here (this runs outside the per-operation + # try/except, so an exception would abort the whole extraction). 'servers' overrides the spec-level + # 'servers' (used for per-path / per-operation 'servers'). + basePath = spec.get("basePath") if isinstance(spec.get("basePath"), six.string_types) else "" + if basePath and not basePath.startswith("/"): # Swagger v2 basePath is a path -> ensure it is slash-prefixed + basePath = "/" + basePath + servers = servers if servers is not None else spec.get("servers") + if isinstance(servers, list) and servers and isinstance(servers[0], dict): + url = servers[0].get("url") + url = url if isinstance(url, six.string_types) else "" + variables = servers[0].get("variables") + if isinstance(variables, dict): + for name, meta in variables.items(): + default = meta.get("default", "1") if isinstance(meta, dict) else "1" + url = url.replace("{%s}" % name, str(default)) + if re.match(r"(?i)[a-z][a-z0-9+.-]*://", url): # absolute server URL -> used as declared (the host is NOT rewritten to the spec's own origin) + return url.rstrip('/') + return ((origin.rstrip('/') if origin else "") + "/" + url.lstrip('/')).rstrip('/') # relative server URL -> resolved against origin + if spec.get("host"): # Swagger v2 with an explicit host + schemes = spec.get("schemes") + scheme = schemes[0] if isinstance(schemes, list) and schemes else "https" + return "%s://%s%s" % (scheme, spec["host"], basePath.rstrip('/')) + return (origin.rstrip('/') if origin else "") + basePath.rstrip('/') # no servers/host -> spec's own origin + +_METHODS = ("get", "post", "put", "delete", "patch", "options", "head") + +def openApiTargets(content, origin=None): + """ + Returns a list of (url, method, data, headers) request tuples derived from an OpenAPI/Swagger + specification. 'headers' is a list of (name, value) tuples (matching conf.httpHeaders). 'origin' + (scheme://host[:port] of the specification's own location) is used only to resolve RELATIVE 'servers' + entries - absolute server URLs are used as declared. Path parameters and header/cookie values carry + the custom injection mark so they become testable injection points. + """ + + spec = _loadSpec(content) + if not isinstance(spec, dict) or not isinstance(spec.get("paths"), dict) or not spec.get("paths"): + errMsg = "no valid 'paths' object found in the provided OpenAPI/Swagger specification" + raise ValueError(errMsg) + + try: + rootBase = _baseUrl(spec, origin) + except Exception: # never let base-URL synthesis abort the whole run + rootBase = origin.rstrip('/') if isinstance(origin, six.string_types) else "" + isV2 = "swagger" in spec and "openapi" not in spec + retVal = [] + cache = {} # $ref -> synthesized example, shared across all operations (large specs reuse schemas) + + for path, item in (spec.get("paths") or {}).items(): + item = _resolve(spec, item) # a Path Item object may itself be a $ref + if not isinstance(item, dict): + continue + shared = item.get("parameters") or [] # 'or []': a present-but-null 'parameters' must not break concatenation + for method, operation in item.items(): + if str(method).lower() not in _METHODS or not isinstance(operation, dict): # str(): YAML keys can be non-string (e.g. 404, 'on'->bool) + continue + try: + # effective base URL with OpenAPI precedence: operation 'servers' > path-item 'servers' > root + opServers = operation.get("servers") or item.get("servers") + base = rootBase + if opServers: + try: + base = _baseUrl(spec, origin, opServers) + except Exception: + base = rootBase + + # merge path-level + operation-level parameters, de-duplicated by (in, name); operation wins + params, seen = [], {} + for raw in ((shared if isinstance(shared, list) else []) + (operation.get("parameters") or [])): + resolved = _resolve(spec, raw) + if isinstance(resolved, dict) and resolved.get("name"): + key = (resolved.get("in"), resolved.get("name")) + if key in seen: + params[seen[key]] = resolved + continue + seen[key] = len(params) + params.append(resolved) + + urlPath = path if isinstance(path, six.string_types) else str(path) + query, headers, form, cookies = [], [], [], [] + + for param in params: + if not isinstance(param, dict): + continue + location, name = param.get("in"), param.get("name") + if not name: + continue + if not isinstance(name, six.string_types): # YAML can yield a non-string param name (e.g. 5) + name = str(name) + explicit = _explicitExample(spec, param) # parameter-level example/examples wins over schema synthesis + if explicit is not _NO_EXAMPLE: + value = _scalar(explicit) + else: + schema = param.get("schema") or {"type": param.get("type", "string")} + value = _scalar(_example(spec, schema, cache=cache)) + if location == "path": + # mark the filled path segment as a (custom) URI injection point - path parameters are + # prime REST injection targets; the value is encoded first so its own chars add no mark + urlPath = urlPath.replace("{%s}" % name, _urlSafe(value) + CUSTOM_INJECTION_MARK_CHAR) + elif location == "query": + # best-effort: array/object query params are scalarized (single value), NOT expanded per + # OpenAPI style/explode (repeated keys, comma/space/pipe delimited, deepObject) - the goal + # is one testable request per operation, not faithful serialization + query.append("%s=%s" % (_urlSafe(name, "[]"), _urlSafe(value))) + elif location == "header": + # append the custom injection mark so the header value becomes a testable (custom) + # injection point (non-exclusive: query/body params are still auto-tested); skip names + # that are not valid HTTP field-name tokens + headerName = _headerClean(name) + if headerName and _HEADER_NAME_RE.match(headerName): + headers.append((headerName, "%s%s" % (_headerClean(_noMark(value)), CUSTOM_INJECTION_MARK_CHAR))) + elif location == "cookie": + # a cookie name is a token; the value must not contain cookie-structure chars ('; ,' + # and whitespace) or a spec could smuggle extra cookie pairs + cookieName = _headerClean(name) + if cookieName and _HEADER_NAME_RE.match(cookieName): + cookieValue = re.sub(r"[;,\s]", "", _headerClean(_noMark(value))) + cookies.append("%s=%s%s" % (cookieName, cookieValue, CUSTOM_INJECTION_MARK_CHAR)) + elif location == "formData": # Swagger v2 in:"formData" -> urlencoded body field + form.append("%s=%s" % (_urlSafe(name, "[]"), _urlSafe(value))) + + if cookies: # aggregate all cookie params into a single Cookie header + headers.append((HTTP_HEADER.COOKIE, "; ".join(cookies))) + + urlPath = urlPath.replace(" ", "%20").replace("?", "%3F").replace("#", "%23") # keep a literal path key from breaking the URL (filled values are already encoded) + if urlPath and not urlPath.startswith("/"): # OpenAPI path keys start with '/'; harden a loose spec so base+path is not glued (/v1pets) + urlPath = "/" + urlPath + + url = base + urlPath + if query: + url += "?" + "&".join(query) + + url = re.sub(r"\{[^}]+\}", "1", url) # any leftover template var (undefined path OR server variable) -> "1" + + if not re.match(r"(?i)[a-z][a-z0-9+.-]*://", url): # no scheme/host -> unscannable relative URL + logger.warning("skipping OpenAPI operation '%s %s' (unable to resolve an absolute target URL; provide the specification by URL or add a 'servers'/'host' entry)" % (str(method).upper(), path)) + continue + + data = None + body = _resolve(spec, operation.get("requestBody") or {}) + content_ = body.get("content") if isinstance(body, dict) else None + if isinstance(content_, dict) and content_: + mediaTypes = [_ for _ in content_ if isinstance(_, six.string_types)] # media-type keys must be strings + picked = next((_ for _ in mediaTypes if _ == "application/json" or _.endswith("+json") or "json" in _), None) \ + or ("application/x-www-form-urlencoded" if "application/x-www-form-urlencoded" in mediaTypes else None) \ + or (mediaTypes[0] if mediaTypes else None) + if picked: + mediaType = content_[picked] if isinstance(content_[picked], dict) else {} + example = _explicitExample(spec, mediaType) # media-type-level example/examples wins over schema synthesis + if example is _NO_EXAMPLE: + example = _example(spec, mediaType.get("schema") or {}, cache=cache) + if "json" in picked: + data = _noMark(json.dumps(example, default=str)) + headers.append((HTTP_HEADER.CONTENT_TYPE, "application/json")) + elif picked == "application/x-www-form-urlencoded" and isinstance(example, dict): + data = "&".join("%s=%s" % (_urlSafe(name, "[]"), _urlSafe(_scalar(value))) for name, value in example.items()) + headers.append((HTTP_HEADER.CONTENT_TYPE, "application/x-www-form-urlencoded")) + elif isinstance(example, six.string_types): + # raw (text / xml / ...) body -> mark it so the whole body becomes a testable point + data = _noMark(example) + CUSTOM_INJECTION_MARK_CHAR + headers.append((HTTP_HEADER.CONTENT_TYPE, picked)) + else: # e.g. multipart/form-data or a structured non-JSON body (no safe serialization) + logger.debug("not synthesizing a '%s' request body for '%s %s'" % (picked, str(method).upper(), path)) + elif isinstance(operation.get("parameters"), list) or isV2: + for param in params: # Swagger v2 in:"body" + if isinstance(param, dict) and param.get("in") == "body": + example = _example(spec, param.get("schema") or {}, cache=cache) + data = _noMark(json.dumps(example, default=str)) + headers.append((HTTP_HEADER.CONTENT_TYPE, "application/json")) + + if data is None and form: # Swagger v2 in:"formData" fields -> urlencoded body + data = "&".join(form) + headers.append((HTTP_HEADER.CONTENT_TYPE, "application/x-www-form-urlencoded")) + + retVal.append((url, str(method).upper(), data, headers or None)) + except Exception as ex: + logger.warning("skipping OpenAPI operation '%s %s' (%s)" % (str(method).upper(), path, getSafeExString(ex))) + + return retVal diff --git a/lib/request/basic.py b/lib/request/basic.py index 2d72a3242ff..5cddbd98338 100644 --- a/lib/request/basic.py +++ b/lib/request/basic.py @@ -57,7 +57,7 @@ from thirdparty import six from thirdparty.chardet import detect from thirdparty.identywaf import identYwaf -from thirdparty.odict import OrderedDict +from collections import OrderedDict from thirdparty.six import unichr as _unichr from thirdparty.six.moves import http_client as _http_client diff --git a/lib/request/comparison.py b/lib/request/comparison.py index e3278297395..d2e8bac0790 100644 --- a/lib/request/comparison.py +++ b/lib/request/comparison.py @@ -39,15 +39,18 @@ def _isJsonResponse(headers): """ - Returns True if the response Content-Type indicates a JSON document (e.g. 'application/json' - or a structured suffix like 'application/vnd.api+json') + Returns True if the response Content-Type plausibly indicates a JSON document - i.e. the canonical + 'application/json', the common misservings ('text/json', 'application/javascript', ...), or a + structured suffix like 'application/vnd.api+json'. Being liberal here is safe: jsonMinimize() returns + None for anything that is not actually parseable JSON, so a mislabelled body simply falls back to the + normal text comparison. """ retVal = False if headers: contentType = (headers.get(HTTP_HEADER.CONTENT_TYPE) or "").split(';')[0].strip().lower() - retVal = contentType == "application/json" or contentType.endswith("+json") + retVal = contentType in ("application/json", "text/json", "application/javascript", "text/javascript", "application/x-javascript") or contentType.endswith("+json") return retVal diff --git a/lib/request/connect.py b/lib/request/connect.py index a14309fa80f..ce59eae0cba 100644 --- a/lib/request/connect.py +++ b/lib/request/connect.py @@ -140,7 +140,7 @@ class WebSocketException(Exception): from lib.request.methodrequest import MethodRequest from lib.utils.safe2bin import safecharencode from thirdparty import six -from thirdparty.odict import OrderedDict +from collections import OrderedDict from thirdparty.six import unichr as _unichr from thirdparty.six.moves import http_client as _http_client from thirdparty.six.moves import urllib as _urllib @@ -508,7 +508,10 @@ def getPage(**kwargs): for key, value in list(headers.items()): if key.upper() == HTTP_HEADER.ACCEPT_ENCODING.upper(): - value = ','.join(_ for _ in re.split(r"\s*,\s*", value) if _.split(';', 1)[0].lower() != "br") or "identity" + # keep only content-codings sqlmap can actually decode (see decodePage): a browser-pasted + # 'Accept-Encoding' (e.g. "gzip, deflate, br, zstd") must not make the server return a body + # we cannot read. Anything else (br, zstd, *, ...) is dropped, falling back to "identity". + value = ','.join(_ for _ in re.split(r"\s*,\s*", value) if _.split(';', 1)[0].strip().lower() in ("gzip", "x-gzip", "deflate", "identity")) or "identity" del headers[key] if isinstance(value, six.string_types): diff --git a/lib/request/http2.py b/lib/request/http2.py index 2af00c69ec3..c885f75cfb6 100644 --- a/lib/request/http2.py +++ b/lib/request/http2.py @@ -14,6 +14,7 @@ import socket import ssl import struct +import threading try: from http.client import responses as _HTTP_RESPONSES @@ -154,6 +155,11 @@ CONNECTION_PREFACE = b"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" def encode_frame(ftype, flags, stream_id, payload=b""): + """Serialize an HTTP/2 frame (RFC 7540 s4.1): 24-bit length + type + flags + 31-bit stream id. + + >>> decode_frame_header(encode_frame(HEADERS, FLAG_END_HEADERS, 1, b'abc')[:9]) + (3, 1, 4, 1) + """ if len(payload) > 0xffffff: raise ValueError("frame payload exceeds 24-bit length") header = struct.pack("!I", len(payload))[1:] # 24-bit length (drop MSB of the 32-bit pack) @@ -161,6 +167,11 @@ def encode_frame(ftype, flags, stream_id, payload=b""): return header + payload def decode_frame_header(nine): + """Parse the 9-byte frame header into (length, type, flags, stream_id); the reserved high bit of the stream id is masked off. + + >>> decode_frame_header(encode_frame(DATA, 0, 0x80000001, b'')[:9]) + (0, 0, 0, 1) + """ if len(nine) != 9: raise ValueError("frame header must be exactly 9 bytes") length = struct.unpack("!I", b"\x00" + nine[:3])[0] @@ -169,6 +180,13 @@ def decode_frame_header(nine): # ---------- Huffman ---------- def huffman_encode(data): + """Huffman-encode a byte string per the RFC 7541 static table (s5.2), padding with EOS 1-bits. + + >>> huffman_decode(huffman_encode(b'www.example.com')) == b'www.example.com' + True + >>> huffman_encode(b'') == b'' + True + """ if not data: return b"" acc = 0 @@ -224,6 +242,13 @@ def huffman_decode(data): # ---------- integer / string (RFC 7541 5.1 / 5.2) ---------- def encode_integer(value, prefix_bits, first_byte=0): + """Encode an integer with an N-bit prefix (RFC 7541 s5.1); the C.1.2 example is 1337 / 5-bit prefix. + + >>> list(encode_integer(10, 5)) + [10] + >>> list(encode_integer(1337, 5)) + [31, 154, 10] + """ mask = (1 << prefix_bits) - 1 if value < mask: return bytearray([first_byte | value]) @@ -236,6 +261,11 @@ def encode_integer(value, prefix_bits, first_byte=0): return out def decode_integer(data, pos, prefix_bits): + """Decode an N-bit-prefixed integer, returning (value, new_pos) (RFC 7541 s5.1). + + >>> decode_integer(bytearray([31, 154, 10]), 0, 5) + (1337, 3) + """ mask = (1 << prefix_bits) - 1 value = data[pos] & mask pos += 1 @@ -296,6 +326,11 @@ def _get(self, index): return self.dynamic[index] def decode(self, data): + """Decode an HPACK header block into a list of (name, value) byte pairs (RFC 7541 s6). + + >>> Decoder().decode(bytes(bytearray([0x82, 0x86, 0x84]))) == [(b':method', b'GET'), (b':scheme', b'http'), (b':path', b'/')] + True + """ data = bytearray(data) pos = 0 headers = [] @@ -397,44 +432,86 @@ def _connect_socket(host, port, proxy, timeout): pass raise -def h2_request(host, port=443, method="GET", path="/", authority=None, headers=None, body=None, timeout=30, proxy=None): - authority = authority or host - ctx = ssl._create_unverified_context() - ctx.set_alpn_protocols(["h2"]) - sock = ctx.wrap_socket(_connect_socket(host, port, proxy, timeout), server_hostname=host) - try: - if sock.selected_alpn_protocol() != "h2": - raise IOError("server did not negotiate h2 (ALPN=%r)" % sock.selected_alpn_protocol()) - sock.settimeout(timeout) +class _UnprocessedStream(IOError): + """Raised when the server made it clear our stream was NOT processed (GOAWAY with last-stream-id below + ours), so the request is always safe to retry on a fresh connection.""" + +class _H2Connection(object): + """A single HTTP/2 connection reused for sequential (one-stream-at-a-time) requests within a thread. + + Multiplexing is intentionally NOT used - one stream is fully consumed before the next is opened - which + preserves request<->response isolation (clean time-based latency, no desync), exactly like the + thread-local HTTP/1.1 keep-alive pool. Reuse amortizes the TCP+TLS+preface cost across all of a thread's + requests to a host. Correctness note: only the HPACK Decoder (server->client dynamic table) is stateful, + so it is kept per-connection and fed responses in order; the Encoder is literal-without-indexing + (stateless), hence a fresh one per request is safe on a reused socket.""" + + def __init__(self, host, port, proxy, timeout): + self.host, self.port, self.proxy = host, port, proxy + self.dec = Decoder() # persistent server->client HPACK table + self.next_sid = 1 # odd, strictly increasing per RFC 7540 + self.usable = True + ctx = ssl._create_unverified_context() + ctx.set_alpn_protocols(["h2"]) + self.sock = ctx.wrap_socket(_connect_socket(host, port, proxy, timeout), server_hostname=host) + try: + if self.sock.selected_alpn_protocol() != "h2": + raise IOError("server did not negotiate h2 (ALPN=%r)" % self.sock.selected_alpn_protocol()) + self.sock.settimeout(timeout) + # connection preface + client SETTINGS (advertise a large per-stream window) + bump conn window + self.sock.sendall(CONNECTION_PREFACE) + self.sock.sendall(encode_frame(SETTINGS, 0, 0, struct.pack("!HI", SETTINGS_INITIAL_WINDOW_SIZE, BIG_WINDOW))) + self.sock.sendall(encode_frame(WINDOW_UPDATE, 0, 0, struct.pack("!I", BIG_WINDOW - 65535))) + except Exception: + self.close() + raise - # connection preface + client SETTINGS (advertise a large per-stream window) + bump conn window - sock.sendall(CONNECTION_PREFACE) - sock.sendall(encode_frame(SETTINGS, 0, 0, struct.pack("!HI", SETTINGS_INITIAL_WINDOW_SIZE, BIG_WINDOW))) - sock.sendall(encode_frame(WINDOW_UPDATE, 0, 0, struct.pack("!I", BIG_WINDOW - 65535))) + def close(self): + self.usable = False + try: + self.sock.close() + except Exception: + pass + + def __del__(self): + self.close() + + def exchange(self, method, path, authority, headers, body, timeout): + if not self.usable: + raise IOError("HTTP/2 connection no longer usable") + + sid = self.next_sid + self.next_sid += 2 + if self.next_sid >= BIG_WINDOW: # stream-id space nearly exhausted -> retire after this + self.usable = False + self.sock.settimeout(timeout) req = [(b":method", _tob(method)), (b":scheme", b"https"), (b":path", _tob(path)), (b":authority", _tob(authority))] for k, v in (headers or {}).items(): req.append((_tob(k).lower(), _tob(v))) hblock = Encoder().encode(req) - sock.sendall(encode_frame(HEADERS, FLAG_END_HEADERS | (0 if body else FLAG_END_STREAM), 1, hblock)) + self.sock.sendall(encode_frame(HEADERS, FLAG_END_HEADERS | (0 if body else FLAG_END_STREAM), sid, hblock)) if body: - sock.sendall(encode_frame(DATA, FLAG_END_STREAM, 1, _tob(body))) + self.sock.sendall(encode_frame(DATA, FLAG_END_STREAM, sid, _tob(body))) - dec = Decoder() header_block, resp_headers, resp_body, done = b"", None, bytearray(), False while not done: - ftype, flags, sid, payload = _read_frame(sock) + ftype, flags, fsid, payload = _read_frame(self.sock) if ftype == SETTINGS: if not (flags & FLAG_ACK): - sock.sendall(encode_frame(SETTINGS, FLAG_ACK, 0, b"")) + self.sock.sendall(encode_frame(SETTINGS, FLAG_ACK, 0, b"")) elif ftype == PING: if not (flags & FLAG_ACK): - sock.sendall(encode_frame(PING, FLAG_ACK, 0, payload)) + self.sock.sendall(encode_frame(PING, FLAG_ACK, 0, payload)) elif ftype == GOAWAY: - done = True - elif ftype == RST_STREAM and sid == 1: + self.usable = False # server won't accept new streams -> retire connection + last_sid = (struct.unpack("!I", payload[4:8])[0] & 0x7fffffff) if len(payload) >= 8 else 0 + if sid > last_sid: # our stream was not processed -> safe to retry fresh + raise _UnprocessedStream("GOAWAY (last stream %d) before stream %d was processed" % (last_sid, sid)) + elif ftype == RST_STREAM and fsid == sid: + self.usable = False raise IOError("stream reset by server (error %d)" % struct.unpack("!I", payload[:4])[0]) - elif ftype in (HEADERS, CONTINUATION) and sid == 1: + elif ftype in (HEADERS, CONTINUATION) and fsid == sid: p = payload if ftype == HEADERS: if flags & FLAG_PADDED: @@ -443,17 +520,17 @@ def h2_request(host, port=443, method="GET", path="/", authority=None, headers=N p = p[5:] header_block += p if flags & FLAG_END_HEADERS: - resp_headers = dec.decode(header_block) + resp_headers = self.dec.decode(header_block) if flags & FLAG_END_STREAM: done = True - elif ftype == DATA and sid == 1: + elif ftype == DATA and fsid == sid: p = payload if flags & FLAG_PADDED: p = p[1:len(p) - bytearray(payload)[0]] resp_body += p if payload: # replenish stream + connection windows - sock.sendall(encode_frame(WINDOW_UPDATE, 0, 1, struct.pack("!I", len(payload)))) - sock.sendall(encode_frame(WINDOW_UPDATE, 0, 0, struct.pack("!I", len(payload)))) + self.sock.sendall(encode_frame(WINDOW_UPDATE, 0, sid, struct.pack("!I", len(payload)))) + self.sock.sendall(encode_frame(WINDOW_UPDATE, 0, 0, struct.pack("!I", len(payload)))) if flags & FLAG_END_STREAM: done = True status = None @@ -462,14 +539,62 @@ def h2_request(host, port=443, method="GET", path="/", authority=None, headers=N status = int(v) break return status, resp_headers, bytes(resp_body) + +# Thread-local pool: one live connection per (host, port, proxy) per thread. Mirrors keepalive.py's model +# (one connection per host per thread) so streams never interleave across threads and time-based +# measurements stay clean. +_h2_pool = threading.local() + +def _pooledExchange(host, port, proxy, method, path, authority, headers, body, timeout): + pool = getattr(_h2_pool, "connections", None) + if pool is None: + pool = _h2_pool.connections = {} + key = (host, port, proxy) + + conn = pool.get(key) + reused = conn is not None and conn.usable + if not reused: + if conn is not None: + conn.close() + conn = pool[key] = _H2Connection(host, port, proxy, timeout) + + try: + result = conn.exchange(method, path, authority, headers, body, timeout) + except _UnprocessedStream: # explicitly not processed -> always safe to retry fresh + conn.close(); pool.pop(key, None) + conn = pool[key] = _H2Connection(host, port, proxy, timeout) + result = conn.exchange(method, path, authority, headers, body, timeout) + except (socket.error, ssl.SSLError, IOError): + conn.close(); pool.pop(key, None) + if reused: # stale keep-alive socket (server closed idle conn) -> reopen once + conn = pool[key] = _H2Connection(host, port, proxy, timeout) + result = conn.exchange(method, path, authority, headers, body, timeout) + else: + raise + if not conn.usable: # GOAWAY / id-exhaustion mid-exchange -> don't keep it pooled + conn.close(); pool.pop(key, None) + return result + +def h2_request(host, port=443, method="GET", path="/", authority=None, headers=None, body=None, timeout=30, proxy=None): + """One-shot request on a throwaway connection (kept for direct/back-compat callers; the engine path + goes through open_url -> the reusing pool).""" + conn = _H2Connection(host, port, proxy, timeout) + try: + return conn.exchange(method, path, authority or host, headers, body, timeout) finally: - try: sock.close() - except Exception: pass + conn.close() class H2Response(object): """A urllib-response-compatible wrapper around a native HTTP/2 response, so the rest of sqlmap's - request pipeline can consume it exactly like a urllib response (code/msg/info()/read()/geturl()).""" + request pipeline can consume it exactly like a urllib response (code/msg/info()/read()/geturl()). + + >>> r = H2Response('https://x/', 200, [(b':status', b'200'), (b'content-type', b'text/html')], b'body') + >>> (r.code, r.msg, r.read() == b'body', r.geturl()) + (200, 'OK', True, 'https://x/') + >>> ':status' in r.info() + False + """ def __init__(self, url, status, headers, body): self.url = url @@ -526,8 +651,8 @@ def open_url(url, method="GET", headers=None, body=None, timeout=30, follow_redi path = parts.path or "/" if parts.query: path += "?" + parts.query - status, resp_headers, resp_body = h2_request(parts.hostname, parts.port or 443, method=method, path=path, - authority=parts.netloc.split("@")[-1], headers=req_headers, body=body, timeout=timeout, proxy=proxy) + status, resp_headers, resp_body = _pooledExchange(parts.hostname, parts.port or 443, proxy, method, path, + parts.netloc.split("@")[-1], req_headers, body, timeout) if follow_redirects and status in REDIRECT_CODES: location = None for name, value in (resp_headers or []): diff --git a/lib/request/keepalive.py b/lib/request/keepalive.py index 299a5450f59..e3f19264390 100644 --- a/lib/request/keepalive.py +++ b/lib/request/keepalive.py @@ -60,6 +60,22 @@ def _take(self, key): def _give_back(self, key, conn, count): self._pool.conns[key] = [conn, count, time.time()] + @staticmethod + def _takeTunnelHeaders(req): + """ + Pops the Proxy-Authorization header off L{req} (returning it as a dict) so it rides on the + CONNECT request only and is never forwarded through the tunnel to the origin server, mirroring + the stock C{urllib.request.AbstractHTTPHandler.do_open} tunnel setup + """ + + result = {} + for store in (getattr(req, "unredirected_hdrs", None), getattr(req, "headers", None)): + if store: + for name in list(store): + if name.lower() == "proxy-authorization": + result[name] = store.pop(name) + return result + def do_open(self, req): # Note: 'selector'/'host' attributes on Python 3 (Request.get_host() was deprecated since # 3.3 and removed in 3.12); the get_*() fallbacks are only reachable under Python 2 @@ -68,7 +84,14 @@ def do_open(self, req): if not host: raise _urllib.error.URLError("no host given") - key = "%s://%s" % (self._scheme, host) + # When routed through an HTTP(s) proxy, ProxyHandler has already rewritten the request: for a + # plain-HTTP target 'host' is the proxy and the selector is absolute; for an HTTPS target + # '_tunnel_host' holds the origin reached via a CONNECT tunnel. Pool by the tunnel origin when + # tunneling (each origin needs its own tunnelled socket) and by 'host' otherwise (one HTTP-proxy + # socket serves many origins, and a direct connection is keyed by its own host exactly as before). + tunnelHost = getattr(req, "_tunnel_host", None) + tunnelHeaders = self._takeTunnelHeaders(req) if tunnelHost else None + key = "%s://%s" % (self._scheme, tunnelHost or host) conn, count = self._take(key) reused = conn is not None @@ -93,6 +116,8 @@ def do_open(self, req): if conn is None: conn = self._get_connection(host) + if tunnelHost: + conn.set_tunnel(tunnelHost, headers=tunnelHeaders or {}) count = 0 self._send_request(conn, req) response = conn.getresponse() diff --git a/lib/techniques/union/test.py b/lib/techniques/union/test.py index 0a8facf784c..5c2022c3a7f 100644 --- a/lib/techniques/union/test.py +++ b/lib/techniques/union/test.py @@ -38,6 +38,7 @@ from lib.core.enums import PAYLOAD from lib.core.settings import FUZZ_UNION_ERROR_REGEX from lib.core.settings import FUZZ_UNION_MAX_COLUMNS +from lib.core.settings import FUZZ_UNION_MAX_REQUESTS from lib.core.settings import LIMITED_ROWS_TEST_NUMBER from lib.core.settings import MAX_RATIO from lib.core.settings import MIN_RATIO @@ -190,12 +191,14 @@ def _fuzzUnionCols(place, parameter, prefix, suffix): choices = getPublicTypeMembers(FUZZ_UNION_COLUMN, True) random.shuffle(choices) + attempts = 0 for candidate in itertools.product(choices, repeat=kb.orderByColumns): - if retVal: + if retVal or attempts >= FUZZ_UNION_MAX_REQUESTS: # bound the exponential type-combination search break elif FUZZ_UNION_COLUMN.STRING not in candidate: continue else: + attempts += 1 candidate = [_.replace(FUZZ_UNION_COLUMN.INTEGER, str(randomInt())).replace(FUZZ_UNION_COLUMN.STRING, "'%s'" % randomStr(20)) for _ in candidate] query = agent.prefixQuery("UNION ALL SELECT %s%s" % (','.join(candidate), FROM_DUMMY_TABLE.get(Backend.getIdentifiedDbms(), "")), prefix=prefix) @@ -332,16 +335,21 @@ def _unionTestByCharBruteforce(comment, place, parameter, value, prefix, suffix) if Backend.getIdentifiedDbms() and kb.orderByColumns and kb.orderByColumns < FUZZ_UNION_MAX_COLUMNS: if kb.fuzzUnionTest is None: msg = "do you want to (re)try to find proper " - msg += "UNION column types with fuzzy test? [y/N] " + msg += "UNION column types with a fuzzy test? [Y/n] " - kb.fuzzUnionTest = readInput(msg, default='N', boolean=True) + kb.fuzzUnionTest = readInput(msg, default='Y', boolean=True) if kb.fuzzUnionTest: kb.unionTemplate = _fuzzUnionCols(place, parameter, prefix, suffix) + # apply the discovered per-column type template through a normal confirmation so + # the resulting vector (and later extraction) is built with type-compatible columns + if kb.unionTemplate: + validPayload, vector = _unionConfirm(comment, place, parameter, prefix, suffix, len(kb.unionTemplate)) + warnMsg = "if UNION based SQL injection is not detected, " warnMsg += "please consider " - if not conf.uChar and count > 1 and kb.uChar == NULL and conf.uValues is None: + if not all((validPayload, vector)) and not conf.uChar and count > 1 and kb.uChar == NULL and conf.uValues is None: message = "injection not exploitable with NULL values. Do you want to try with a random integer value for option '--union-char'? [Y/n] " if not readInput(message, default='Y', boolean=True): @@ -380,6 +388,8 @@ def unionTest(comment, place, parameter, value, prefix, suffix): negativeLogic = kb.negativeLogic setTechnique(PAYLOAD.TECHNIQUE.UNION) + kb.unionTemplate = None # reset any per-column type template carried over from a previous parameter + try: if negativeLogic: pushValue(kb.negativeLogic) diff --git a/lib/techniques/union/use.py b/lib/techniques/union/use.py index dc85170962e..e28244c05bc 100644 --- a/lib/techniques/union/use.py +++ b/lib/techniques/union/use.py @@ -62,7 +62,7 @@ from lib.utils.progress import ProgressBar from lib.utils.safe2bin import safecharencode from thirdparty import six -from thirdparty.odict import OrderedDict +from collections import OrderedDict def _oneShotUnionUse(expression, unpack=True, limited=False): retVal = hashDBRetrieve("%s%s" % (conf.hexConvert or False, expression), checkConf=True) # as UNION data is stored raw unconverted diff --git a/lib/utils/api.py b/lib/utils/api.py index 90d0c0b9e3c..7b5f39f4389 100644 --- a/lib/utils/api.py +++ b/lib/utils/api.py @@ -253,6 +253,12 @@ def setupReportCollector(): collector = Database(":memory:") collector.connect("report") collector.init() + + # record error/critical log messages into the collector so that a CLI --report-json report carries + # the same 'error' content the REST API exposes via /scan//data - letting consumers tell a + # failed/unreachable run apart from a clean "nothing found" one (both otherwise have empty 'data') + logger.addHandler(ReportErrorRecorder(collector)) + return collector def writeReportJson(collector, filepath): @@ -449,6 +455,22 @@ def emit(self, record): """ conf.databaseCursor.execute("INSERT INTO logs VALUES(NULL, ?, ?, ?, ?)", (conf.taskid, time.strftime("%X"), record.levelname, str(record.msg % record.args if record.args else record.msg))) +class ReportErrorRecorder(logging.Handler): + def __init__(self, collector): + """ + Records error/critical log messages into a report collector's 'errors' table (the counterpart + of StdDbOut's stderr branch for CLI --report-json runs) + """ + logging.Handler.__init__(self) + self.setLevel(logging.ERROR) + self.collector = collector + + def emit(self, record): + try: + self.collector.execute("INSERT INTO errors VALUES(NULL, ?, ?)", (REPORT_TASKID, str(record.msg % record.args if record.args else record.msg))) + except Exception: + pass + def setRestAPILog(): if conf.api: try: diff --git a/lib/utils/hash.py b/lib/utils/hash.py index 11831534f84..cca7d4fc3e3 100644 --- a/lib/utils/hash.py +++ b/lib/utils/hash.py @@ -19,19 +19,27 @@ from thirdparty.pydes.pyDes import CBC from thirdparty.pydes.pyDes import des +try: + from hashlib import scrypt as _scrypt # not available on Python 2 (added in 3.6) +except ImportError: + _scrypt = None + _multiprocessing = None import base64 import binascii import gc +import hmac import math import os import re +import struct import tempfile import time import zipfile from hashlib import md5 +from hashlib import pbkdf2_hmac from hashlib import sha1 from hashlib import sha224 from hashlib import sha256 @@ -146,6 +154,21 @@ def postgres_passwd(password, username, uppercase=False): return retVal.upper() if uppercase else retVal.lower() +def postgres_scram_passwd(password, salt, iterations, **kwargs): # since version '10' + """ + Reference(s): + https://www.rfc-editor.org/rfc/rfc5803 + + >>> postgres_scram_passwd(password='testpass', salt='c2FsdHNhbHRzYWx0', iterations=4096) + 'SCRAM-SHA-256$4096:c2FsdHNhbHRzYWx0$AzDKnszrCJPfdiFrFLbdoiqdocK4KWksHHcs3Jx7R5w=:lmWF1kOl/PbOyhpnGuBGzKyuP3XYMK6whWukBxHiHLc=' + """ + + salted = pbkdf2_hmac("sha256", getBytes(password), decodeBase64(salt, binary=True), iterations) + stored_key = sha256(hmac.new(salted, b"Client Key", sha256).digest()).digest() + server_key = hmac.new(salted, b"Server Key", sha256).digest() + + return "SCRAM-SHA-256$%d:%s$%s:%s" % (iterations, salt, getText(base64.b64encode(stored_key)), getText(base64.b64encode(server_key))) + def mssql_new_passwd(password, salt, uppercase=False): # since version '2012' """ Reference(s): @@ -439,6 +462,243 @@ def _encode64(value, count): return getText(magic + salt + b'$' + getBytes(hash_)) +# SHA-crypt (Drepper) final-permutation byte orders for the 32/64-byte digests +_SHA256_CRYPT_ORDER = ((0, 10, 20), (21, 1, 11), (12, 22, 2), (3, 13, 23), (24, 4, 14), (15, 25, 5), (6, 16, 26), (27, 7, 17), (18, 28, 8), (9, 19, 29), (31, 30)) +_SHA512_CRYPT_ORDER = ((0, 21, 42), (22, 43, 1), (44, 2, 23), (3, 24, 45), (25, 46, 4), (47, 5, 26), (6, 27, 48), (28, 49, 7), (50, 8, 29), (9, 30, 51), (31, 52, 10), (53, 11, 32), (12, 33, 54), (34, 55, 13), (56, 14, 35), (15, 36, 57), (37, 58, 16), (59, 17, 38), (18, 39, 60), (40, 61, 19), (62, 20, 41), (63,)) + +def _shaCryptDigest(password, salt, rounds, digestmod, order): + dsize = digestmod().digest_size + + B = digestmod(password + salt + password).digest() + + ctx = digestmod(password + salt) + cnt = len(password) + while cnt > dsize: + ctx.update(B) + cnt -= dsize + ctx.update(B[:cnt]) + + i = len(password) + while i: + ctx.update(B if i & 1 else password) + i >>= 1 + A = ctx.digest() + + dp = digestmod() + for _ in xrange(len(password)): + dp.update(password) + DP = dp.digest() + P = DP * (len(password) // dsize) + DP[:len(password) % dsize] + + ds = digestmod() + for _ in xrange(16 + (A[0] if isinstance(A[0], int) else ord(A[0]))): + ds.update(salt) + DS = ds.digest() + S = DS * (len(salt) // dsize) + DS[:len(salt) % dsize] + + C = A + for i in xrange(rounds): + c = digestmod() + c.update(P if i & 1 else C) + if i % 3: + c.update(S) + if i % 7: + c.update(P) + c.update(C if i & 1 else P) + C = c.digest() + + retVal = "" + for group in order: + value = 0 + for idx in group: + value = (value << 8) | (C[idx] if isinstance(C[idx], int) else ord(C[idx])) + for _ in xrange((len(group) * 8 + 5) // 6): + retVal += ITOA64[value & 0x3f] + value >>= 6 + + return retVal + +def sha2_crypt_passwd(password, salt, magic="$5$", **kwargs): + """ + Reference(s): + https://www.akkadia.org/drepper/SHA-crypt.txt + + >>> sha2_crypt_passwd(password='testpass', salt='saltstring', magic='$5$') + '$5$saltstring$rn/td51LeVLXb2RR8WT672g4QhAuobh1gQQFGFiRCT.' + >>> sha2_crypt_passwd(password='testpass', salt='saltstring', magic='$6$') + '$6$saltstring$Oxduy3vBZ8CEBR5mER96ach5GlbbBT1Oz5g1UNdPqomx5bB1.IwS1ZFoW8fpb0xvz/BCS7.LzpkW7GAFOW9yC.' + """ + + rounds, saltstr = 5000, salt + if salt.startswith("rounds="): + prefix, saltstr = salt.split('$', 1) + rounds = int(prefix[len("rounds="):]) + + order, digestmod = (_SHA256_CRYPT_ORDER, sha256) if magic == "$5$" else (_SHA512_CRYPT_ORDER, sha512) + digest = _shaCryptDigest(getBytes(password), getBytes(saltstr)[:16], rounds, digestmod, order) + + return "%s%s$%s" % (magic, salt, digest) + +def mysql_sha2_passwd(password, salt, rounds, prefix, **kwargs): # MySQL 8 'caching_sha2_password' (sha256crypt, 20-byte salt) + """ + Reference(s): + https://hashcat.net/wiki/doku.php?id=example_hashes + + >>> mysql_sha2_passwd(password='hashcat', salt=decodeHex('F9CC98CE08892924F50A213B6BC571A2C11778C5'), rounds=5000, prefix='$mysql$A$005*F9CC98CE08892924F50A213B6BC571A2C11778C5*') + '$mysql$A$005*F9CC98CE08892924F50A213B6BC571A2C11778C5*625479393559393965414D45316477456B484F41316E64484742577A2E3162785353526B7554584647562F' + """ + + digest = _shaCryptDigest(getBytes(password), bytes(salt), rounds, sha256, _SHA256_CRYPT_ORDER) + + return "%s%s" % (prefix, getText(encodeHex(getBytes(digest), binary=False)).upper()) + +# bcrypt (Provos-Mazieres EksBlowfish); the Blowfish P/S init constants are the fractional hex digits of pi +BCRYPT_ITOA64 = "./ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" + +_bcryptState = None + +def _bcryptInitState(): + global _bcryptState + + if _bcryptState is None: + count = 18 + 4 * 256 + ndigits = count * 8 + prec = ndigits + 16 + one = 1 << (4 * prec) + + def _arctan(inv): + total = term = one // inv + square = inv * inv + i = 1 + while term: + term //= square + total += (term // (2 * i + 1)) * (-1 if i % 2 else 1) + i += 1 + return total + + frac = (16 * _arctan(5) - 4 * _arctan(239) - 3 * one) >> (4 * (prec - ndigits)) + hexstr = "%0*x" % (ndigits, frac) + words = [int(hexstr[i * 8:(i + 1) * 8], 16) for i in xrange(count)] + _bcryptState = (words[:18], [words[18 + i * 256:18 + (i + 1) * 256] for i in xrange(4)]) + + return _bcryptState + +def _bcryptEncipher(P, S, L, R): + for i in xrange(16): + L ^= P[i] + R ^= (((S[0][(L >> 24) & 0xff] + S[1][(L >> 16) & 0xff]) & 0xffffffff) ^ S[2][(L >> 8) & 0xff]) + S[3][L & 0xff] & 0xffffffff + L, R = R, L + L, R = R, L + return (L ^ P[17]) & 0xffffffff, (R ^ P[16]) & 0xffffffff + +def _bcryptStream(data, offset): + word = 0 + for _ in xrange(4): + word = ((word << 8) | data[offset[0]]) & 0xffffffff + offset[0] = (offset[0] + 1) % len(data) + return word + +def _bcryptExpand(P, S, data, key): + koffset = [0] + for i in xrange(18): + P[i] ^= _bcryptStream(key, koffset) + + doffset = [0] + L = R = 0 + for i in xrange(0, 18, 2): + if data: + L ^= _bcryptStream(data, doffset) + R ^= _bcryptStream(data, doffset) + L, R = _bcryptEncipher(P, S, L, R) + P[i], P[i + 1] = L, R + + for b in xrange(4): + for k in xrange(0, 256, 2): + if data: + L ^= _bcryptStream(data, doffset) + R ^= _bcryptStream(data, doffset) + L, R = _bcryptEncipher(P, S, L, R) + S[b][k], S[b][k + 1] = L, R + +def _bcryptBase64(data): + retVal = "" + i = 0 + while i < len(data): + c = data[i]; i += 1 + retVal += BCRYPT_ITOA64[(c >> 2) & 0x3f] + c = (c & 3) << 4 + if i >= len(data): + retVal += BCRYPT_ITOA64[c & 0x3f]; break + d = data[i]; i += 1 + retVal += BCRYPT_ITOA64[(c | (d >> 4) & 0x0f) & 0x3f] + c = (d & 0x0f) << 2 + if i >= len(data): + retVal += BCRYPT_ITOA64[c & 0x3f]; break + e = data[i]; i += 1 + retVal += BCRYPT_ITOA64[(c | (e >> 6) & 3) & 0x3f] + retVal += BCRYPT_ITOA64[e & 0x3f] + return retVal + +def _bcryptUnbase64(value, length): + retVal = bytearray() + positions = [BCRYPT_ITOA64.index(_) for _ in value] + i = 0 + while i < len(positions) and len(retVal) < length: + c1 = positions[i] + c2 = positions[i + 1] if i + 1 < len(positions) else 0 + retVal.append(((c1 << 2) | (c2 >> 4)) & 0xff) + if len(retVal) >= length: + break + c3 = positions[i + 2] if i + 2 < len(positions) else 0 + retVal.append((((c2 & 0x0f) << 4) | (c3 >> 2)) & 0xff) + if len(retVal) >= length: + break + c4 = positions[i + 3] if i + 3 < len(positions) else 0 + retVal.append((((c3 & 3) << 6) | c4) & 0xff) + i += 4 + return retVal[:length] + +def bcrypt_passwd(password, salt, magic="$2a$", cost=5, **kwargs): + """ + Reference(s): + https://www.openwall.com/crypt/ + + >>> bcrypt_passwd(password='U*U', salt='CCCCCCCCCCCCCCCCCCCCC.', magic='$2a$', cost=5) + '$2a$05$CCCCCCCCCCCCCCCCCCCCC.E5YPO9kmyuRGyh0XouQYb4YMJKvyOeW' + """ + + P0, S0 = _bcryptInitState() + P, S = list(P0), [list(_) for _ in S0] + + key = bytearray(getBytes(password) + b"\0") + saltbytes = _bcryptUnbase64(salt, 16) + + _bcryptExpand(P, S, saltbytes, key) + for _ in xrange(1 << cost): + _bcryptExpand(P, S, b"", key) + _bcryptExpand(P, S, b"", saltbytes) + + ctext = list(struct.unpack(">6I", b"OrpheanBeholderScryDoubt")) + for _ in xrange(64): + for j in xrange(0, 6, 2): + ctext[j], ctext[j + 1] = _bcryptEncipher(P, S, ctext[j], ctext[j + 1]) + + digest = bytearray(struct.pack(">6I", *ctext))[:23] + + return "%s%02d$%s%s" % (magic, cost, salt, _bcryptBase64(digest)) + +def wordpress_bcrypt_passwd(password, salt, magic="$2y$", cost=10, **kwargs): # WordPress 6.8+ 'bcrypt(base64(hmac-sha384(pass)))' + """ + Reference: https://make.wordpress.org/core/2025/02/17/wordpress-6-8-will-use-bcrypt-for-password-hashing/ + + >>> wordpress_bcrypt_passwd(password='hashcat', salt='lzlQrRRhLSjz486bA9CKHu', magic='$2y$', cost=10) + '$wp$2y$10$lzlQrRRhLSjz486bA9CKHuZRPoKz4uviT251Sq/r5OzKUBbrXwnQW' + """ + + prehashed = getText(base64.b64encode(hmac.new(b"wp-sha384", getBytes(password.strip()), sha384).digest())) + + return "$wp%s" % bcrypt_passwd(prehashed, salt, magic, cost) + def joomla_passwd(password, salt, **kwargs): """ Reference: https://stackoverflow.com/a/10428239 @@ -469,6 +729,56 @@ def django_sha1_passwd(password, salt, **kwargs): return "sha1$%s$%s" % (salt, sha1(getBytes(salt) + getBytes(password)).hexdigest()) +def django_pbkdf2_sha256_passwd(password, salt, iterations, **kwargs): + """ + Reference: https://github.com/django/django/blob/main/django/contrib/auth/hashers.py + + >>> django_pbkdf2_sha256_passwd(password='testpass', salt='salt', iterations=1000) + 'pbkdf2_sha256$1000$salt$N3DLJstEJ6mIjp0fq/KRcHmJ/4FtMzHYmW9fBHci/aI=' + """ + + dk = pbkdf2_hmac("sha256", getBytes(password), getBytes(salt), iterations) + + return "pbkdf2_sha256$%d$%s$%s" % (iterations, salt, getText(base64.b64encode(dk))) + +def werkzeug_pbkdf2_passwd(password, salt, iterations, digestmod="sha256", **kwargs): + """ + Reference: https://github.com/pallets/werkzeug/blob/main/src/werkzeug/security.py + + >>> werkzeug_pbkdf2_passwd(password='testpass', salt='salt', iterations=1000, digestmod='sha256') + 'pbkdf2:sha256:1000$salt$3770cb26cb4427a9888e9d1fabf291707989ff816d3331d8996f5f047722fda2' + """ + + dk = pbkdf2_hmac(digestmod, getBytes(password), getBytes(salt), iterations) + + return "pbkdf2:%s:%d$%s$%s" % (digestmod, iterations, salt, getText(encodeHex(dk, binary=False))) + +def werkzeug_scrypt_passwd(password, salt, N, r, p, **kwargs): + """ + Reference: https://github.com/pallets/werkzeug/blob/main/src/werkzeug/security.py + + >>> werkzeug_scrypt_passwd(password='testpass', salt='saltsalt', N=32768, r=8, p=1) if _scrypt else 'scrypt:32768:8:1$saltsalt$1e0f97c3f6609024022fbe698da29c2fe53ef1087a8e396dc6d5d2a041e886dee09ea922781f2c2a1c85e46c77060147e43487f8fe6226bcb635915af9b0518b' + 'scrypt:32768:8:1$saltsalt$1e0f97c3f6609024022fbe698da29c2fe53ef1087a8e396dc6d5d2a041e886dee09ea922781f2c2a1c85e46c77060147e43487f8fe6226bcb635915af9b0518b' + """ + + dk = _scrypt(getBytes(password), salt=getBytes(salt), n=N, r=r, p=p, dklen=64, maxmem=132 * N * r + 1024) + + return "scrypt:%d:%d:%d$%s$%s" % (N, r, p, salt, getText(encodeHex(dk, binary=False))) + +def aspnet_identity_passwd(password, salt, iterations, prf, dklen, **kwargs): + """ + Reference(s): + https://github.com/dotnet/AspNetCore/blob/main/src/Identity/Extensions.Core/src/PasswordHasher.cs + + >>> aspnet_identity_passwd(password='cutecats', salt=decodeBase64('AQAAAAEAACcQAAAAEFWLthQDW2xiWaS3vLgY4ItJdModbW0kzKtb8IVuXBY3fFaIntkbbdqTj8mTXH4mmA==', binary=True)[13:29], iterations=10000, prf=1, dklen=32) + 'AQAAAAEAACcQAAAAEFWLthQDW2xiWaS3vLgY4ItJdModbW0kzKtb8IVuXBY3fFaIntkbbdqTj8mTXH4mmA==' + """ + + subkey = pbkdf2_hmac({0: "sha1", 1: "sha256", 2: "sha512"}[prf], getBytes(password), bytes(salt), iterations, dklen) + blob = struct.pack(">BIII", 1, prf, iterations, len(salt)) + bytes(salt) + subkey + + return getText(base64.b64encode(blob)) + def vbulletin_passwd(password, salt, **kwargs): """ Reference: https://stackoverflow.com/a/2202810 @@ -560,6 +870,8 @@ def _encode64(input_, count): HASH.MYSQL: mysql_passwd, HASH.MYSQL_OLD: mysql_old_passwd, HASH.POSTGRES: postgres_passwd, + HASH.POSTGRES_SCRAM: postgres_scram_passwd, + HASH.MYSQL_SHA2: mysql_sha2_passwd, HASH.MSSQL: mssql_passwd, HASH.MSSQL_OLD: mssql_old_passwd, HASH.MSSQL_NEW: mssql_new_passwd, @@ -572,9 +884,16 @@ def _encode64(input_, count): HASH.SHA384_GENERIC: sha384_generic_passwd, HASH.SHA512_GENERIC: sha512_generic_passwd, HASH.CRYPT_GENERIC: crypt_generic_passwd, + HASH.SHA256_UNIX_CRYPT: sha2_crypt_passwd, + HASH.SHA512_UNIX_CRYPT: sha2_crypt_passwd, + HASH.BCRYPT: bcrypt_passwd, + HASH.WORDPRESS_BCRYPT: wordpress_bcrypt_passwd, HASH.JOOMLA: joomla_passwd, HASH.DJANGO_MD5: django_md5_passwd, HASH.DJANGO_SHA1: django_sha1_passwd, + HASH.DJANGO_PBKDF2_SHA256: django_pbkdf2_sha256_passwd, + HASH.ASPNET_IDENTITY: aspnet_identity_passwd, + HASH.WERKZEUG_PBKDF2: werkzeug_pbkdf2_passwd, HASH.PHPASS: phpass_passwd, HASH.APACHE_MD5_CRYPT: unix_md5_passwd, HASH.UNIX_MD5_CRYPT: unix_md5_passwd, @@ -591,6 +910,14 @@ def _encode64(input_, count): HASH.SHA512_BASE64: sha512_generic_passwd, } +if _scrypt is not None: + __functions__[HASH.WERKZEUG_SCRYPT] = werkzeug_scrypt_passwd + +# Recognized-only formats with no pure-Python/stdlib crack path; identified and pointed to dedicated tools +HASH_TOOL_HINTS = { + HASH.ARGON2: "an Argon2 hash (e.g. 'hashcat -m 34000' or 'john --format=argon2')", +} + def _finalize(retVal, results, processes, attack_info=None): if _multiprocessing: gc.enable() @@ -898,6 +1225,7 @@ def _bruteProcessVariantA(attack_info, hash_regex, suffix, retVal, proc_id, proc pass finally: + wordlist.closeFP() # release the wordlist file handle (else it leaks; Windows can't rmtree an open file) if hasattr(proc_count, "value"): with proc_count.get_lock(): proc_count.value -= 1 @@ -977,6 +1305,7 @@ def _bruteProcessVariantB(user, hash_, kwargs, hash_regex, suffix, retVal, found pass finally: + wordlist.closeFP() # release the wordlist file handle (else it leaks; Windows can't rmtree an open file) if hasattr(proc_count, "value"): with proc_count.get_lock(): proc_count.value -= 1 @@ -1023,9 +1352,14 @@ def dictionaryAttack(attack_dict): regex = hashRecognition(hash_) if regex and regex not in hash_regexes: - hash_regexes.append(regex) - infoMsg = "using hash method '%s'" % __functions__[regex].__name__ - logger.info(infoMsg) + if regex in __functions__: + hash_regexes.append(regex) + infoMsg = "using hash method '%s'" % __functions__[regex].__name__ + logger.info(infoMsg) + else: + warnMsg = "sqlmap identified %s that cannot be cracked with the " % HASH_TOOL_HINTS.get(regex, "a hash") + warnMsg += "built-in dictionary attack" + singleTimeWarnMessage(warnMsg) for hash_regex in hash_regexes: keys = set() @@ -1043,7 +1377,7 @@ def dictionaryAttack(attack_dict): try: item = None - if hash_regex not in (HASH.CRYPT_GENERIC, HASH.JOOMLA, HASH.PHPASS, HASH.UNIX_MD5_CRYPT, HASH.APACHE_MD5_CRYPT, HASH.APACHE_SHA1, HASH.VBULLETIN, HASH.VBULLETIN_OLD, HASH.SSHA, HASH.SSHA256, HASH.SSHA512, HASH.DJANGO_MD5, HASH.DJANGO_SHA1, HASH.MD5_BASE64, HASH.SHA1_BASE64, HASH.SHA256_BASE64, HASH.SHA512_BASE64): + if hash_regex not in (HASH.CRYPT_GENERIC, HASH.JOOMLA, HASH.PHPASS, HASH.UNIX_MD5_CRYPT, HASH.APACHE_MD5_CRYPT, HASH.APACHE_SHA1, HASH.VBULLETIN, HASH.VBULLETIN_OLD, HASH.SSHA, HASH.SSHA256, HASH.SSHA512, HASH.DJANGO_MD5, HASH.DJANGO_SHA1, HASH.DJANGO_PBKDF2_SHA256, HASH.POSTGRES_SCRAM, HASH.MYSQL_SHA2, HASH.WERKZEUG_PBKDF2, HASH.WERKZEUG_SCRYPT, HASH.SHA256_UNIX_CRYPT, HASH.SHA512_UNIX_CRYPT, HASH.BCRYPT, HASH.WORDPRESS_BCRYPT, HASH.ASPNET_IDENTITY, HASH.MD5_BASE64, HASH.SHA1_BASE64, HASH.SHA256_BASE64, HASH.SHA512_BASE64): hash_ = hash_.lower() if hash_regex in (HASH.MD5_BASE64, HASH.SHA1_BASE64, HASH.SHA256_BASE64, HASH.SHA512_BASE64): @@ -1068,10 +1402,32 @@ def dictionaryAttack(attack_dict): item = [(user, hash_), {"salt": hash_[0:2]}] elif hash_regex in (HASH.UNIX_MD5_CRYPT, HASH.APACHE_MD5_CRYPT): item = [(user, hash_), {"salt": hash_.split('$')[2], "magic": "$%s$" % hash_.split('$')[1]}] + elif hash_regex in (HASH.SHA256_UNIX_CRYPT, HASH.SHA512_UNIX_CRYPT): + item = [(user, hash_), {"salt": '$'.join(hash_.split('$')[2:-1]), "magic": "$%s$" % hash_.split('$')[1]}] + elif hash_regex in (HASH.BCRYPT,): + item = [(user, hash_), {"salt": hash_[7:29], "magic": hash_[:4], "cost": int(hash_[4:6])}] + elif hash_regex in (HASH.WORDPRESS_BCRYPT,): + item = [(user, hash_), {"salt": hash_[10:32], "magic": hash_[3:7], "cost": int(hash_[7:9])}] + elif hash_regex in (HASH.ASPNET_IDENTITY,): + _ = decodeBase64(hash_, binary=True) + prf, iterations, saltlen = struct.unpack(">III", _[1:13]) + item = [(user, hash_), {"salt": _[13:13 + saltlen], "iterations": iterations, "prf": prf, "dklen": len(_) - 13 - saltlen}] + elif hash_regex in (HASH.MYSQL_SHA2,): + _ = hash_.split('*') + item = [(user, hash_), {"salt": decodeHex(_[1]), "rounds": int(_[0].split('$')[-1], 16) * 1000, "prefix": hash_[:hash_.rindex('*') + 1]}] elif hash_regex in (HASH.JOOMLA, HASH.VBULLETIN, HASH.VBULLETIN_OLD, HASH.OSCOMMERCE_OLD): item = [(user, hash_), {"salt": hash_.split(':')[-1]}] elif hash_regex in (HASH.DJANGO_MD5, HASH.DJANGO_SHA1): item = [(user, hash_), {"salt": hash_.split('$')[1]}] + elif hash_regex in (HASH.DJANGO_PBKDF2_SHA256,): + item = [(user, hash_), {"salt": hash_.split('$')[2], "iterations": int(hash_.split('$')[1])}] + elif hash_regex in (HASH.POSTGRES_SCRAM,): + item = [(user, hash_), {"salt": hash_.split('$')[1].split(':')[1], "iterations": int(hash_.split('$')[1].split(':')[0])}] + elif hash_regex in (HASH.WERKZEUG_PBKDF2,): + item = [(user, hash_), {"salt": hash_.split('$')[1], "iterations": int(hash_.split('$')[0].split(':')[2]), "digestmod": hash_.split('$')[0].split(':')[1]}] + elif hash_regex in (HASH.WERKZEUG_SCRYPT,): + _ = hash_.split('$')[0].split(':') + item = [(user, hash_), {"salt": hash_.split('$')[1], "N": int(_[1]), "r": int(_[2]), "p": int(_[3])}] elif hash_regex in (HASH.PHPASS,): if ITOA64.index(hash_[3]) < 32: item = [(user, hash_), {"salt": hash_[4:12], "count": 1 << ITOA64.index(hash_[3]), "prefix": hash_[:3]}] @@ -1102,7 +1458,7 @@ def dictionaryAttack(attack_dict): while not kb.wordlists: # the slowest of all methods hence smaller default dict - if hash_regex in (HASH.ORACLE_OLD, HASH.PHPASS): + if hash_regex in (HASH.ORACLE_OLD, HASH.PHPASS, HASH.SHA256_UNIX_CRYPT, HASH.SHA512_UNIX_CRYPT, HASH.WERKZEUG_SCRYPT, HASH.BCRYPT, HASH.WORDPRESS_BCRYPT, HASH.MYSQL_SHA2): dictPaths = [paths.SMALL_DICT] else: dictPaths = [paths.WORDLIST] diff --git a/lib/utils/library.py b/lib/utils/library.py new file mode 100644 index 00000000000..c30cdeff322 --- /dev/null +++ b/lib/utils/library.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python + +""" +Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org) +See the file 'LICENSE' for copying permission +""" + +# Library facade for programmatic (in-code) usage: 'import sqlmap; sqlmap.scan(...)'. +# +# This is the code-level sibling of the REST API (lib/utils/api.py): both drive the engine as an +# isolated subprocess for programmatic callers. The public names here are re-exported by sqlmap.py so +# that they are reachable as 'sqlmap.scan', 'sqlmap.scanFromRequest' and 'sqlmap.SqlmapError'. + +import json +import os +import sys +import tempfile + +__all__ = ["scan", "scanFromRequest", "SqlmapError"] + +# Absolute path of the engine entry point (this module lives at /lib/utils/library.py) +SQLMAP_FILE = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "sqlmap.py") + +class SqlmapError(Exception): + """ + Raised by the library facade (scan/scanFromRequest) when a scan can not produce a result report + """ + + pass + +def _terminateProcess(process): + """ + Best-effort hard teardown of a scan subprocess together with its whole process group, so a + timed-out scan never leaves orphaned sqlmap workers behind (POSIX kills the group, others fall + back to killing the process itself) + """ + + import signal + + try: + if os.name != "nt" and hasattr(os, "killpg"): + os.killpg(os.getpgid(process.pid), getattr(signal, "SIGKILL", signal.SIGTERM)) + else: + process.kill() + except (OSError, AttributeError): + try: + process.kill() + except (OSError, AttributeError): + pass + +def scan(url=None, requestFile=None, timeout=None, outputDir=None, raw=None, **options): + """ + Runs a sqlmap scan in a dedicated subprocess and returns its structured result (library usage). + + Keyword options are plain sqlmap option names - exactly the names used in a sqlmap configuration + file (data/sqlmap.conf) and by the REST API, i.e. the 'conf' names, NOT command line switches. So + scan(url, technique="BEU", getBanner=True, dumpTable=True, tbl="users", level=3) is equivalent to + the config file lines 'technique = BEU', 'getBanner = True', 'dumpTable = True', 'tbl = users', + 'level = 3'. Unknown names are rejected. The scan is driven through a generated config file passed + with '-c' (the same mechanism the REST API uses), so there is a single option namespace and no + argument escaping. 'raw' takes a list of extra raw command line switches for the rare thing not + expressible as a config option (e.g. raw=["--fresh-queries"]). + + The engine runs fully out-of-process, so a scan can never affect the calling process (no shared + global state, no HTTP-stack patching, no risk of the host being exited). The return value is the + parsed '--report-json' report - the same structure as the REST API '/scan//data' response: a + dict with keys 'success', 'data' (a list of {'type_name', 'value'} entries: TARGET, TECHNIQUES, + BANNER, DUMP_TABLE, ...), 'error' and 'meta'. + + scan() is blocking and thread-safe, so it is both thread- and asyncio-ready: run several at once + in threads, or from an event loop with 'await loop.run_in_executor(None, functools.partial(scan, + url, dumpTable=True))'. For unattended/concurrent use the run is hardened like the REST API + subprocess: batch mode (never prompts) with stdin closed, isolated file descriptors, its own + output directory (so parallel scans of the same target can not collide on session/dump files and + nothing accumulates on disk), engine output streamed to a temporary file rather than buffered in + memory, and - when 'timeout' is set - the whole subprocess group is torn down on expiry. Pass + 'outputDir' to keep the run's files. + + Example: + import sqlmap + result = sqlmap.scan("http://target/vuln.php?id=1", dumpTable=True, tbl="users") + """ + + import shutil + import subprocess + import time + + from lib.core.common import saveConfig + from lib.core.optiondict import optDict + + if not (url or requestFile): + raise SqlmapError("scan() requires either 'url' or 'requestFile'") + + if not os.path.isfile(SQLMAP_FILE): + raise SqlmapError("could not locate the sqlmap engine ('%s')" % SQLMAP_FILE) + + knownOptions = set() + for family in optDict.values(): + knownOptions.update(family) + + config = {} + if url: + config["url"] = url + if requestFile: + config["requestFile"] = requestFile + config.update(options) + + unknown = [_ for _ in config if _ not in knownOptions] + if unknown: + raise SqlmapError("unknown option(s) %s - scan() expects sqlmap option names as used in a configuration file (e.g. getBanner, dumpTable, tbl, technique, level), not command line switches" % ", ".join(repr(_) for _ in sorted(unknown))) + + handle, report = tempfile.mkstemp(prefix="sqlmap-", suffix=".json") + os.close(handle) + + # Each run gets its own output directory so concurrent scans can not collide on session/dump files + # and no scan state piles up on disk. A caller-provided 'outputDir' is respected and left in place. + ownOutput = not outputDir + if ownOutput: + outputDir = tempfile.mkdtemp(prefix="sqlmap-output-") + + # engine plumbing goes through the very same option namespace + config["batch"] = True + config["disableColoring"] = True + config["outputDir"] = outputDir + config["reportJson"] = report + + handle, configFile = tempfile.mkstemp(prefix="sqlmap-", suffix=".conf") + os.close(handle) + saveConfig(config, configFile) + + argv = [sys.executable or "python", SQLMAP_FILE, "-c", configFile, "--ignore-stdin"] + if raw: + argv += list(raw) + + logHandle, logFile = tempfile.mkstemp(prefix="sqlmap-", suffix=".log") + devnull = open(os.devnull, "rb") + + kwargs = {"shell": False, "close_fds": os.name != "nt", "cwd": os.path.dirname(SQLMAP_FILE) or '.', "stdin": devnull, "stdout": logHandle, "stderr": subprocess.STDOUT} + if os.name == "nt": + kwargs["creationflags"] = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) + elif sys.version_info >= (3, 2): + kwargs["start_new_session"] = True # own process group -> clean group teardown + else: + kwargs["preexec_fn"] = os.setsid + + process = None + try: + process = subprocess.Popen(argv, **kwargs) + + if timeout is None: + process.wait() + else: + end = time.time() + timeout + while process.poll() is None: + if time.time() > end: + _terminateProcess(process) + process.wait() + raise SqlmapError("scan timed out after %s second(s)" % timeout) + time.sleep(0.5) + + try: + with open(report, "rb") as f: + return json.loads(f.read().decode("utf-8", "replace")) + except (IOError, OSError, ValueError): + try: + with open(logFile, "rb") as f: + tail = f.read().decode("utf-8", "replace").strip() + except (IOError, OSError): + tail = "" + raise SqlmapError("scan did not produce a valid report (exit code %s)\n%s" % (getattr(process, "returncode", None), tail[-1000:])) + finally: + try: + os.close(logHandle) + except OSError: + pass + devnull.close() + for path in (report, logFile, configFile): + try: + os.remove(path) + except OSError: + pass + if ownOutput: + shutil.rmtree(outputDir, ignore_errors=True) + +def scanFromRequest(requestFile, **options): + """ + Convenience wrapper for scan(requestFile=...) - runs a scan from a saved HTTP request file ('-r') + """ + + return scan(requestFile=requestFile, **options) diff --git a/lib/utils/pivotdumptable.py b/lib/utils/pivotdumptable.py index b1a10adf2f1..96a30d58c89 100644 --- a/lib/utils/pivotdumptable.py +++ b/lib/utils/pivotdumptable.py @@ -45,6 +45,7 @@ def pivotDumpTable(table, colList, count=None, blind=True, alias=None): validColumnList = False validPivotValue = False + compositePivot = None if count is None: query = dumpNode.count % table @@ -118,6 +119,26 @@ def pivotDumpTable(table, colList, count=None, blind=True, alias=None): errMsg = "all provided column name(s) are non-existent" raise SqlmapNoneDataException(errMsg) + if not validPivotValue: + # No single column holds all-distinct values. Fall back to a COMPOSITE pivot (a + # concatenation of every column) whose combined value is unique per row, so rows sharing + # a value in every individual column are no longer silently dropped (ref: #1545). + _composite = agent.concatQuery(','.join(colList)) + query = dumpNode.count2 % (_composite, table) + query = agent.whereQuery(query) + value = inject.getValue(query, blind=blind, union=not blind, error=not blind, expected=EXPECTED.INT, charsetType=CHARSET_TYPE.DIGITS) + + if isNumPosStrValue(value) and int(value) == count: + infoMsg = "using a concatenation of all columns as a " + infoMsg += "composite pivot for retrieving row data" + logger.info(infoMsg) + + compositePivot = _composite + lengths[compositePivot] = 0 + entries[compositePivot] = BigArray() + colList.insert(0, compositePivot) + validPivotValue = True + if not validPivotValue: warnMsg = "no proper pivot column provided (with unique values)." warnMsg += " It won't be possible to retrieve all rows" @@ -186,4 +207,9 @@ def _(column, pivotValue): logger.critical(errMsg) + # The composite pivot is a synthetic paging key, not a real column - drop it from the output + if compositePivot is not None: + entries.pop(compositePivot, None) + lengths.pop(compositePivot, None) + return entries, lengths diff --git a/sqlmap.py b/sqlmap.py index 59c7e8510f3..77a67d017f9 100755 --- a/sqlmap.py +++ b/sqlmap.py @@ -664,3 +664,9 @@ def main(): else: # cancelling postponed imports (because of CI/CD checks) __import__("lib.controller.controller") + + # exposing the programmatic library facade as 'sqlmap.scan()' / 'sqlmap.scanFromRequest()' + from lib.utils.library import scan, scanFromRequest, SqlmapError + +# public library API (also marks the re-exported names above as intentional for pyflakes) +__all__ = ["scan", "scanFromRequest", "SqlmapError"] diff --git a/tests/_testutils.py b/tests/_testutils.py index 781f54749ad..a856b1ebc2d 100644 --- a/tests/_testutils.py +++ b/tests/_testutils.py @@ -98,6 +98,22 @@ def set_dbms(name): Backend.forceDbms(name) +def reset_dbms(): + """Clear any DBMS forced via set_dbms()/Backend, restoring the clean post-bootstrap state. + + A forced DBMS lives on the global `kb` singleton and is read by every dialect/agent path, so a + module that forces one without clearing it would leak that back-end into later test modules + (order-dependent flakiness). Modules that call set_dbms() should expose this as their + `tearDownModule` so the leak can never cross a module boundary. + """ + from lib.core.common import Backend + from lib.core.data import kb + from lib.core.settings import UNKNOWN_DBMS_VERSION + Backend.flushForcedDbms(force=True) # kb.forcedDbms = None; kb.stickyDBMS = False + kb.resolutionDbms = None + kb.dbmsVersion = [UNKNOWN_DBMS_VERSION] + + # --- property/fuzz testing harness (shared so individual test files don't each reinvent it) --- _PROPERTY_BASE = 0x51A1 diff --git a/tests/test_agent.py b/tests/test_agent.py index d49a7d76fee..203e2789691 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -34,7 +34,7 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from _testutils import bootstrap, set_dbms +from _testutils import bootstrap, set_dbms, reset_dbms bootstrap() from lib.core.agent import agent @@ -766,3 +766,7 @@ def test_splices_before_order_by(self): if __name__ == "__main__": unittest.main(verbosity=2) + + +def tearDownModule(): + reset_dbms() # clear any DBMS forced via set_dbms() so it can't leak into later test modules diff --git a/tests/test_api.py b/tests/test_api.py index a76d814d64c..4360c3e7058 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -86,6 +86,7 @@ class _ApiServerCase(unittest.TestCase): """ def setUp(self): + self._saved_batch = conf.batch conf.batch = True # snapshot mutated globals @@ -122,6 +123,7 @@ def tearDown(self): api.DataStore.username = self._saved["username"] api.DataStore.password = self._saved["password"] api.Database.filepath = self._saved["filepath"] + conf.batch = self._saved_batch def _new_task(self): code, parsed, _ = _wsgi_call("GET", "/task/new") diff --git a/tests/test_bigarray.py b/tests/test_bigarray.py index 8d033f77c5c..9d65d8e97fe 100644 --- a/tests/test_bigarray.py +++ b/tests/test_bigarray.py @@ -28,14 +28,26 @@ N = 5000 +_SPILLED = [] + def _make_spilled(): # tiny chunk_size guarantees many on-disk chunks for N items ba = BigArray(chunk_size=1024) for i in range(N): ba.append("item-%d" % i) + _SPILLED.append(ba) # tracked so tearDownModule closes it (release the on-disk chunk files) return ba +def tearDownModule(): + for ba in _SPILLED: + try: + ba.close() + except Exception: + pass + del _SPILLED[:] + + class TestSpill(unittest.TestCase): def test_actually_spilled_to_disk(self): ba = _make_spilled() diff --git a/tests/test_brute.py b/tests/test_brute.py index 3d8143b915a..c13395978d5 100644 --- a/tests/test_brute.py +++ b/tests/test_brute.py @@ -22,7 +22,7 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from _testutils import bootstrap, set_dbms +from _testutils import bootstrap, set_dbms, reset_dbms bootstrap() from lib.core.data import conf, kb @@ -196,3 +196,7 @@ def test_add_page_text_words_filters(self): if __name__ == "__main__": unittest.main(verbosity=2) + + +def tearDownModule(): + reset_dbms() # clear any DBMS forced via set_dbms() so it can't leak into later test modules diff --git a/tests/test_checks.py b/tests/test_checks.py index 7300c39bb7f..54988ac5817 100644 --- a/tests/test_checks.py +++ b/tests/test_checks.py @@ -53,9 +53,10 @@ "notString", "regexp", "regex", "dummy", "offline", "skipWaf", "data", "hashDB", "cj", "cookie", "dropSetCookie", "httpHeaders", "proxy", "tor", "tamper", "timeout", "retries", "textOnly", "ignoreCode", "disablePrecon", - "ipv6", "multipleTargets", "level", "base64Parameter", "batch", + "ipv6", "multipleTargets", "level", "base64Parameter", "batch", "code", "titles", ) _KB_KEYS = ( + "pageTemplate", "negativeLogic", "heavilyDynamic", "dynamicParameter", "originalPage", "originalPageTime", "originalCode", "ignoreCasted", "heuristicMode", "disableHtmlDecoding", "heuristicTest", "heuristicPage", "heuristicCode", "pageStable", diff --git a/tests/test_common.py b/tests/test_common.py index 87369fe42fb..e8d217627fc 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -19,14 +19,16 @@ stdlib unittest only (no pytest / no pip); works on Python 2.7 and 3.x. """ +import atexit import base64 import os +import shutil import sys import tempfile import unittest sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from _testutils import bootstrap, set_dbms +from _testutils import bootstrap, set_dbms, reset_dbms bootstrap() from lib.core.data import conf, kb, paths @@ -119,7 +121,8 @@ zeroDepthSearch, ) -SCRATCH = "/tmp/claude-1000/-tmp-tmp-oUnlQJzlQN/fcd55d25-6313-49ed-817e-dcbe7fc2bf22/scratchpad" +SCRATCH = tempfile.mkdtemp(prefix="sqlmap-tests-") # per-run temp dir (portable; replaces a stale hardcoded path) +atexit.register(lambda: shutil.rmtree(SCRATCH, ignore_errors=True)) def _write_temp(content, suffix): @@ -1317,10 +1320,9 @@ def test_chunk_split_post_data(self): random.choice, random.randint, random.sample, random.seed = _saved def test_chunk_split_terminator(self): - import random from lib.core.common import chunkSplitPostData - random.seed(123) # regardless of content, the chunked stream must end with the zero-length terminator + # (assertion is seed-independent, so don't touch the global RNG) self.assertTrue(chunkSplitPostData("abc").endswith("0\r\n\r\n")) @@ -1714,3 +1716,7 @@ def test_no_old_options_is_noop(self): if __name__ == "__main__": unittest.main(verbosity=2) + + +def tearDownModule(): + reset_dbms() # clear any DBMS forced via set_dbms() so it can't leak into later test modules diff --git a/tests/test_databases_enum.py b/tests/test_databases_enum.py index 3bba88dde33..13cc6a54a7a 100644 --- a/tests/test_databases_enum.py +++ b/tests/test_databases_enum.py @@ -21,7 +21,7 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from _testutils import bootstrap, set_dbms +from _testutils import bootstrap, set_dbms, reset_dbms bootstrap() @@ -765,3 +765,7 @@ def test_get_columns_bruteforce_dumpmode_collist_on_decline(self): if __name__ == "__main__": unittest.main() + + +def tearDownModule(): + reset_dbms() # clear any DBMS forced via set_dbms() so it can't leak into later test modules diff --git a/tests/test_dbms_enum.py b/tests/test_dbms_enum.py index dff6a04656b..97325506378 100644 --- a/tests/test_dbms_enum.py +++ b/tests/test_dbms_enum.py @@ -24,7 +24,7 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from _testutils import bootstrap, set_dbms +from _testutils import bootstrap, set_dbms, reset_dbms bootstrap() from lib.core.common import Backend @@ -720,3 +720,7 @@ def test_get_current_db_default_schema(self): if __name__ == "__main__": unittest.main() + + +def tearDownModule(): + reset_dbms() # clear any DBMS forced via set_dbms() so it can't leak into later test modules diff --git a/tests/test_dialect.py b/tests/test_dialect.py index 4cce55abffc..5e82127633c 100644 --- a/tests/test_dialect.py +++ b/tests/test_dialect.py @@ -20,7 +20,7 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from _testutils import bootstrap, set_dbms +from _testutils import bootstrap, set_dbms, reset_dbms bootstrap() from lib.core.agent import agent @@ -105,3 +105,7 @@ def test_position_and_count(self): if __name__ == "__main__": unittest.main(verbosity=2) + + +def tearDownModule(): + reset_dbms() # clear any DBMS forced via set_dbms() so it can't leak into later test modules diff --git a/tests/test_dns_engine.py b/tests/test_dns_engine.py index 767a5019c8f..e1194142d75 100644 --- a/tests/test_dns_engine.py +++ b/tests/test_dns_engine.py @@ -38,7 +38,7 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from _testutils import bootstrap, set_dbms +from _testutils import bootstrap, set_dbms, reset_dbms bootstrap() from lib.core.agent import agent @@ -188,7 +188,7 @@ def oracle(payload=None, *args, **kwargs): finally: c.close() served[0] += len(chunk) - for _ in range(100): + for _ in range(500): # ~5s deadline (was ~1s) - loopback packet can lag on a loaded CI runner with self.server._lock: if any(host.encode() in r for r in self.server._requests): break @@ -313,7 +313,7 @@ def oracle(payload=None, *args, **kwargs): finally: c.close() served[0] += len(chunk) - for _ in range(100): + for _ in range(500): # ~5s deadline (was ~1s) - loopback packet can lag on a loaded CI runner with self.server._lock: matched = [r for r in self.server._requests if host.encode() in r] if matched: @@ -394,3 +394,7 @@ def test_detection_failure_with_force_dns_raises(self): if __name__ == "__main__": unittest.main(verbosity=2) + + +def tearDownModule(): + reset_dbms() # clear any DBMS forced via set_dbms() so it can't leak into later test modules diff --git a/tests/test_dump_format.py b/tests/test_dump_format.py index ce9076c6ba1..d3484c28fe0 100644 --- a/tests/test_dump_format.py +++ b/tests/test_dump_format.py @@ -27,7 +27,7 @@ from collections import OrderedDict as _PlainOrderedDict sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from _testutils import bootstrap +from _testutils import bootstrap, reset_dbms bootstrap() from lib.core.common import Backend @@ -408,3 +408,7 @@ def test_datatype_str(self): if __name__ == "__main__": unittest.main(verbosity=2) + + +def tearDownModule(): + reset_dbms() # clear any DBMS forced via set_dbms() so it can't leak into later test modules diff --git a/tests/test_dump_jsonl.py b/tests/test_dump_jsonl.py index 9dc5cac8a2b..515b68bf3ef 100644 --- a/tests/test_dump_jsonl.py +++ b/tests/test_dump_jsonl.py @@ -26,7 +26,7 @@ from collections import OrderedDict sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from _testutils import bootstrap +from _testutils import bootstrap, reset_dbms bootstrap() from lib.core.common import Backend @@ -165,3 +165,7 @@ def test_unicode_value_not_escaped(self): if __name__ == "__main__": unittest.main() + + +def tearDownModule(): + reset_dbms() # clear any DBMS forced via set_dbms() so it can't leak into later test modules diff --git a/tests/test_entries.py b/tests/test_entries.py index d54a92bbcd2..b4cb78dfbc4 100644 --- a/tests/test_entries.py +++ b/tests/test_entries.py @@ -22,7 +22,7 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from _testutils import bootstrap, set_dbms +from _testutils import bootstrap, set_dbms, reset_dbms bootstrap() @@ -800,3 +800,7 @@ def gv(query, *a, **k): if __name__ == "__main__": unittest.main() + + +def tearDownModule(): + reset_dbms() # clear any DBMS forced via set_dbms() so it can't leak into later test modules diff --git a/tests/test_error_engine.py b/tests/test_error_engine.py index 2c9b54c5a45..d1323172978 100644 --- a/tests/test_error_engine.py +++ b/tests/test_error_engine.py @@ -22,7 +22,7 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from _testutils import bootstrap, set_dbms +from _testutils import bootstrap, set_dbms, reset_dbms bootstrap() from lib.core.data import conf, kb @@ -111,3 +111,7 @@ def oracle(payload=None, content=False, raise404=True, **kwargs): if __name__ == "__main__": unittest.main(verbosity=2) + + +def tearDownModule(): + reset_dbms() # clear any DBMS forced via set_dbms() so it can't leak into later test modules diff --git a/tests/test_filesystem.py b/tests/test_filesystem.py index 70b6192e10b..6eb4e6bcfe4 100644 --- a/tests/test_filesystem.py +++ b/tests/test_filesystem.py @@ -25,10 +25,11 @@ import os import sys +import tempfile import unittest sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from _testutils import bootstrap, set_dbms +from _testutils import bootstrap, set_dbms, reset_dbms bootstrap() from lib.core.data import conf, kb @@ -108,7 +109,7 @@ def test_fileContentEncode_chunk_below_threshold_is_single(self): def test_fileEncode_reads_then_encodes(self): # fileEncode must read the file bytes and delegate to fileContentEncode path = os.path.join( - os.environ.get("TMPDIR", "/tmp"), "sqlmap_fe_%d.bin" % os.getpid()) + tempfile.gettempdir(), "sqlmap_fe_%d.bin" % os.getpid()) with open(path, "wb") as f: f.write(b"hello") try: @@ -138,7 +139,7 @@ def test_checkFileLength_mysql_query_and_samefile(self): # MySQL builds LENGTH(LOAD_FILE('')) and compares to local size. set_dbms("MySQL") path = os.path.join( - os.environ.get("TMPDIR", "/tmp"), "sqlmap_cl_%d.bin" % os.getpid()) + tempfile.gettempdir(), "sqlmap_cl_%d.bin" % os.getpid()) with open(path, "wb") as f: f.write(b"12345") # 5 bytes captured = {} @@ -159,7 +160,7 @@ def getValue(query, *a, **k): def test_checkFileLength_size_differs(self): set_dbms("MySQL") path = os.path.join( - os.environ.get("TMPDIR", "/tmp"), "sqlmap_cl2_%d.bin" % os.getpid()) + tempfile.gettempdir(), "sqlmap_cl2_%d.bin" % os.getpid()) with open(path, "wb") as f: f.write(b"12345") # local 5 self.patch(self.module.inject, "getValue", lambda q, *a, **k: "9") @@ -176,7 +177,7 @@ def test_checkFileLength_mssql_openrowset_stacked(self): # OPENROWSET-building branch runs in isolation. set_dbms("Microsoft SQL Server") path = os.path.join( - os.environ.get("TMPDIR", "/tmp"), "sqlmap_cl3_%d.bin" % os.getpid()) + tempfile.gettempdir(), "sqlmap_cl3_%d.bin" % os.getpid()) with open(path, "wb") as f: f.write(b"ABCD") # 4 bytes stacked = [] @@ -205,7 +206,7 @@ def test_checkFileLength_not_written_warns_false(self): # non-positive remote size -> treated as "not written" -> sameFile False set_dbms("MySQL") path = os.path.join( - os.environ.get("TMPDIR", "/tmp"), "sqlmap_cl4_%d.bin" % os.getpid()) + tempfile.gettempdir(), "sqlmap_cl4_%d.bin" % os.getpid()) with open(path, "wb") as f: f.write(b"x") self.patch(self.module.inject, "getValue", lambda q, *a, **k: None) @@ -282,7 +283,7 @@ def test_writeFile_dispatches_to_stacked(self): # stackedWriteFile and return its result. set_dbms("MySQL") path = os.path.join( - os.environ.get("TMPDIR", "/tmp"), "sqlmap_wf_%d.bin" % os.getpid()) + tempfile.gettempdir(), "sqlmap_wf_%d.bin" % os.getpid()) with open(path, "wb") as f: f.write(b"data") calls = {} @@ -733,3 +734,7 @@ def test_pgsql_udfSetRemotePath_linux_and_windows(self): if __name__ == "__main__": unittest.main() + + +def tearDownModule(): + reset_dbms() # clear any DBMS forced via set_dbms() so it can't leak into later test modules diff --git a/tests/test_fingerprint.py b/tests/test_fingerprint.py index 0aefbd3dae9..b583ea061fa 100644 --- a/tests/test_fingerprint.py +++ b/tests/test_fingerprint.py @@ -18,7 +18,7 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from _testutils import bootstrap, set_dbms +from _testutils import bootstrap, set_dbms, reset_dbms bootstrap() from lib.core.data import conf, kb @@ -93,12 +93,18 @@ def setUp(self): conf.batch = True conf.extensiveFp = False conf.api = False + # _drive() stubs the SHARED lib.request.inject module (plugins do `from lib.request import inject`), + # so snapshot the originals and restore them, else stubbed getValue/checkBooleanExpression leak process-wide + import lib.request.inject as _inject + self._inject = _inject + self._inject_saved = (_inject.getValue, _inject.checkBooleanExpression) def tearDown(self): for k, v in self._saved.items(): conf[k] = v for k, v in self._kb.items(): kb[k] = v + self._inject.getValue, self._inject.checkBooleanExpression = self._inject_saved def _drive(self, name, modpath, pkg, oracle): set_dbms(name) @@ -201,3 +207,7 @@ def _t(self): if __name__ == "__main__": unittest.main() + + +def tearDownModule(): + reset_dbms() # clear any DBMS forced via set_dbms() so it can't leak into later test modules diff --git a/tests/test_generic_takeover.py b/tests/test_generic_takeover.py index 89449adf40e..40f0f0c9d20 100644 --- a/tests/test_generic_takeover.py +++ b/tests/test_generic_takeover.py @@ -26,7 +26,7 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from _testutils import bootstrap, set_dbms +from _testutils import bootstrap, set_dbms, reset_dbms bootstrap() @@ -599,3 +599,7 @@ def test_ossmb_windows_invokes_smb(self): if __name__ == "__main__": unittest.main() + + +def tearDownModule(): + reset_dbms() # clear any DBMS forced via set_dbms() so it can't leak into later test modules diff --git a/tests/test_graphql.py b/tests/test_graphql.py index 5be9d901b8a..506e8f1027f 100644 --- a/tests/test_graphql.py +++ b/tests/test_graphql.py @@ -246,6 +246,7 @@ class TestGraphqlBooleanDetection(unittest.TestCase): def setUp(self): self._gql = gi._gqlSend + self._conf = gi.conf gi.conf = type("C", (), {"url": "http://test/graphql"})() pages = {"true": MATCH, "false": NOMATCH} @@ -259,6 +260,7 @@ def fakeSend(endpoint, query, variables=None): def tearDown(self): gi._gqlSend = self._gql + gi.conf = self._conf def test_boolean_detected(self): slot = _slot("query", "Query", "user", "username", "string") @@ -277,6 +279,7 @@ class TestGraphqlErrorDetection(unittest.TestCase): def setUp(self): self._gql = gi._gqlSend + self._conf = gi.conf gi.conf = type("C", (), {"url": "http://test/graphql"})() def fakeSend(endpoint, query, variables=None): @@ -287,6 +290,7 @@ def fakeSend(endpoint, query, variables=None): def tearDown(self): gi._gqlSend = self._gql + gi.conf = self._conf def test_error_detected(self): slot = _slot("query", "Query", "user", "username", "string") @@ -372,10 +376,12 @@ class TestGraphqlIntrospectionFallback(unittest.TestCase): def setUp(self): self._gql = gi._gqlSend + self._conf = gi.conf gi.conf = type("C", (), {"url": "http://test/graphql"})() def tearDown(self): gi._gqlSend = self._gql + gi.conf = self._conf def test_fallback_without_specifiedByURL(self): calls = [] diff --git a/tests/test_http2.py b/tests/test_http2.py new file mode 100644 index 00000000000..7c762648176 --- /dev/null +++ b/tests/test_http2.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python + +""" +Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org) +See the file 'LICENSE' for copying permission + +Unit coverage for the PURE (network-free) parts of the native HTTP/2 client in +lib/request/http2.py: the RFC 7540 frame codec, the RFC 7541 HPACK integer / +Huffman / string primitives, the HPACK Decoder/Encoder (static + dynamic table), +and the urllib-compatible H2Response wrapper. + +Nothing here opens a socket or negotiates TLS - only the deterministic codecs and +the response adapter are exercised. Known vectors are the canonical RFC 7541 +examples; everything else is a round-trip / invariant check. + +stdlib unittest only (no pytest / no pip); works on Python 2.7 and 3.x. +""" + +import binascii +import os +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from _testutils import bootstrap +bootstrap() + +from lib.request.http2 import ( + Decoder, + Encoder, + H2Response, + REDIRECT_CODES, + STATIC_LEN, + STATIC_TABLE, + DATA, + HEADERS, + FLAG_END_HEADERS, + FLAG_END_STREAM, + decode_frame_header, + decode_integer, + decode_string, + encode_frame, + encode_integer, + encode_string, + huffman_decode, + huffman_encode, +) + + +def _b(*ints): + # build a bytes object from ints (identical on Python 2 and 3) + return bytes(bytearray(ints)) + + +class TestFrameCodec(unittest.TestCase): + def test_roundtrip(self): + header = encode_frame(HEADERS, FLAG_END_HEADERS, 1, b"abc")[:9] + self.assertEqual(decode_frame_header(header), (3, HEADERS, FLAG_END_HEADERS, 1)) + + def test_payload_is_appended_verbatim(self): + frame = encode_frame(DATA, 0, 1, b"hello") + self.assertEqual(frame[9:], b"hello") + + def test_reserved_stream_bit_is_masked(self): + # the high (reserved) bit of the 31-bit stream id must be dropped on both ends + header = encode_frame(DATA, 0, 0x80000001, b"")[:9] + self.assertEqual(decode_frame_header(header), (0, DATA, 0, 1)) + + def test_zero_length_payload(self): + header = encode_frame(DATA, FLAG_END_STREAM, 1, b"")[:9] + length, _, flags, _ = decode_frame_header(header) + self.assertEqual(length, 0) + self.assertEqual(flags, FLAG_END_STREAM) + + def test_oversized_payload_rejected(self): + with self.assertRaises(ValueError): + encode_frame(DATA, 0, 1, b"x" * (0xFFFFFF + 1)) + + def test_bad_header_length_rejected(self): + with self.assertRaises(ValueError): + decode_frame_header(b"123") + + +class TestIntegerCoding(unittest.TestCase): + def test_rfc_c11_small(self): + # RFC 7541 C.1.1: 10 with a 5-bit prefix fits in the prefix + self.assertEqual(list(encode_integer(10, 5)), [10]) + + def test_rfc_c12_multibyte(self): + # RFC 7541 C.1.2: 1337 with a 5-bit prefix + self.assertEqual(list(encode_integer(1337, 5)), [31, 154, 10]) + self.assertEqual(decode_integer(bytearray([31, 154, 10]), 0, 5), (1337, 3)) + + def test_rfc_c13_full_byte_prefix(self): + # RFC 7541 C.1.3: 42 starting from a full (8-bit prefix at an octet boundary) + self.assertEqual(list(encode_integer(42, 8)), [42]) + + def test_roundtrip_across_prefixes(self): + for prefix in (4, 5, 6, 7, 8): + for value in (0, 1, 2, 30, 31, 32, 127, 128, 255, 256, 16384, 1000000): + encoded = bytearray(encode_integer(value, prefix)) + decoded, pos = decode_integer(encoded, 0, prefix) + self.assertEqual(decoded, value) + self.assertEqual(pos, len(encoded)) + + def test_first_byte_bits_preserved(self): + # a caller-supplied opcode in the high bits must survive a small value + self.assertEqual(bytearray(encode_integer(5, 7, 0x80))[0], 0x80 | 5) + + +class TestHuffman(unittest.TestCase): + def test_known_vector_www_example_com(self): + # RFC 7541 C.4.1 + self.assertEqual(binascii.hexlify(huffman_encode(b"www.example.com")), b"f1e3c2e5f23a6ba0ab90f4ff") + + def test_empty(self): + self.assertEqual(huffman_encode(b""), b"") + self.assertEqual(huffman_decode(b""), b"") + + def test_roundtrip(self): + for sample in (b"a", b"hello world", b"/index.html?a=1&b=2", + b"GET", b"application/json", b"ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789", + bytes(bytearray(range(256)))): + self.assertEqual(huffman_decode(huffman_encode(sample)), sample) + + def test_shrinks_typical_text(self): + sample = b"www.example.com" + self.assertLess(len(huffman_encode(sample)), len(sample)) + + def test_padding_too_long_rejected(self): + # 0xfe walks eight 1-bits into a long (unterminated) code -> more than a byte of padding + with self.assertRaises(ValueError): + huffman_decode(_b(0xFE)) + + +class TestStringCoding(unittest.TestCase): + def test_huffman_branch_roundtrip(self): + encoded = encode_string(b"custom-value") + self.assertTrue(bytearray(encoded)[0] & 0x80) # huffman flag set for compressible text + self.assertEqual(decode_string(bytearray(encoded), 0), (b"custom-value", len(encoded))) + + def test_literal_branch_when_huffman_would_not_shrink(self): + encoded = encode_string(_b(0xFF)) + self.assertFalse(bytearray(encoded)[0] & 0x80) # falls back to a literal string + self.assertEqual(decode_string(bytearray(encoded), 0), (_b(0xFF), len(encoded))) + + def test_disable_huffman(self): + encoded = encode_string(b"abc", huffman=False) + self.assertFalse(bytearray(encoded)[0] & 0x80) + self.assertEqual(decode_string(bytearray(encoded), 0), (b"abc", len(encoded))) + + +class TestHpackDecoder(unittest.TestCase): + def test_indexed_static_entries(self): + # 0x82/0x86/0x84 -> static indices 2, 6, 4 + self.assertEqual( + Decoder().decode(_b(0x82, 0x86, 0x84)), + [(b":method", b"GET"), (b":scheme", b"http"), (b":path", b"/")], + ) + + def test_static_lookup_bounds(self): + d = Decoder() + self.assertEqual(d._get(1), (b":authority", b"")) + self.assertEqual(d._get(2), (b":method", b"GET")) + self.assertEqual(d._get(STATIC_LEN), STATIC_TABLE[-1]) + + def test_index_zero_rejected(self): + with self.assertRaises(ValueError): + Decoder()._get(0) + + def test_index_out_of_range_rejected(self): + with self.assertRaises(ValueError): + Decoder()._get(STATIC_LEN + 1) # no dynamic entries yet + + def test_literal_incremental_indexing_populates_dynamic_table(self): + # 0x40 = literal with incremental indexing, new name + block = bytearray([0x40]) + encode_string(b"custom-key") + encode_string(b"custom-value") + d = Decoder() + self.assertEqual(d.decode(bytes(block)), [(b"custom-key", b"custom-value")]) + # entry is now addressable at the first dynamic index (STATIC_LEN + 1) + self.assertEqual(d._get(STATIC_LEN + 1), (b"custom-key", b"custom-value")) + self.assertEqual(d._size, 32 + len(b"custom-key") + len(b"custom-value")) + + def test_literal_without_indexing_does_not_touch_dynamic_table(self): + block = bytearray([0x00]) + encode_string(b"k") + encode_string(b"v") + d = Decoder() + self.assertEqual(d.decode(bytes(block)), [(b"k", b"v")]) + self.assertEqual(d.dynamic, []) + + def test_dynamic_table_eviction(self): + d = Decoder(max_size=40) # each 2+2 byte entry costs 32+2+2 = 36 + d._add(b"aa", b"bb") + self.assertEqual(len(d.dynamic), 1) + d._add(b"cc", b"dd") # 72 > 40 -> oldest evicted + self.assertEqual(d.dynamic, [(b"cc", b"dd")]) + self.assertEqual(d._size, 36) + + def test_dynamic_size_update_clears(self): + d = Decoder() + d._add(b"x", b"y") + d.decode(_b(0x20)) # 0x20 = dynamic table size update to 0 + self.assertEqual(d.max_size, 0) + self.assertEqual(d.dynamic, []) + + +class TestHpackEncoderRoundTrip(unittest.TestCase): + def test_roundtrip_through_decoder(self): + headers = [ + (b":method", b"GET"), + (b":scheme", b"https"), + (b":path", b"/a/b?c=d"), + (b":authority", b"example.com"), + (b"user-agent", b"sqlmap"), + (b"accept", b""), # empty value + (b"x-custom", b"\x00\x01\xff"), # non-ASCII value + ] + self.assertEqual(Decoder().decode(Encoder().encode(headers)), headers) + + def test_encoder_output_is_bytes(self): + self.assertIsInstance(Encoder().encode([(b"a", b"b")]), bytes) + + +class TestH2Response(unittest.TestCase): + def _make(self, status=200, headers=None, body=b"body"): + headers = headers if headers is not None else [(b":status", b"200"), (b"content-type", b"text/html")] + return H2Response("https://target/x", status, headers, body) + + def test_basic_fields(self): + r = self._make() + self.assertEqual(r.code, 200) + self.assertEqual(r.status, 200) + self.assertEqual(r.msg, "OK") + self.assertEqual(r.http_version, "HTTP/2.0") + self.assertEqual(r.geturl(), "https://target/x") + + def test_unknown_status_message(self): + self.assertEqual(self._make(status=799).msg, "") + + def test_pseudo_headers_stripped(self): + r = self._make() + self.assertNotIn(":status", r.info()) + self.assertEqual(r.info().get("content-type"), "text/html") + + def test_read_full_then_empty(self): + r = self._make(body=b"hello") + self.assertEqual(r.read(), b"hello") + self.assertEqual(r.read(), b"") # offset exhausted + + def test_read_in_chunks(self): + r = self._make(body=b"abcdef") + self.assertEqual(r.read(2), b"ab") + self.assertEqual(r.read(3), b"cde") + self.assertEqual(r.read(10), b"f") # asking past the end returns the remainder + self.assertEqual(r.read(10), b"") + + def test_str_header_names_accepted(self): + # headers may arrive already decoded to str (not only bytes) + r = H2Response("https://t/", 200, [("content-type", "application/json")], b"{}") + self.assertEqual(r.info().get("content-type"), "application/json") + + def test_mimetools_style_headers_list(self): + # patchHeaders() relies on a '.headers' list of "Name: value\r\n" lines being present + r = self._make() + self.assertTrue(hasattr(r.info(), "headers")) + self.assertIn("content-type: text/html\r\n", r.info().headers) + + def test_close_is_noop(self): + self.assertIsNone(self._make().close()) + + +class TestConstants(unittest.TestCase): + def test_redirect_codes(self): + for code in (301, 302, 303, 307, 308): + self.assertIn(code, REDIRECT_CODES) + self.assertNotIn(200, REDIRECT_CODES) + + def test_static_table_length(self): + self.assertEqual(STATIC_LEN, len(STATIC_TABLE)) + self.assertEqual(STATIC_LEN, 61) # RFC 7541 Appendix A + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tests/test_identifiers_output.py b/tests/test_identifiers_output.py index dfa27ab27ae..39a97f06625 100644 --- a/tests/test_identifiers_output.py +++ b/tests/test_identifiers_output.py @@ -13,7 +13,7 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from _testutils import bootstrap, set_dbms +from _testutils import bootstrap, set_dbms, reset_dbms bootstrap() from lib.core.common import safeSQLIdentificatorNaming, unsafeSQLIdentificatorNaming, safeCSValue @@ -83,3 +83,7 @@ def test_csv_roundtrip(self): if __name__ == "__main__": unittest.main(verbosity=2) + + +def tearDownModule(): + reset_dbms() # clear any DBMS forced via set_dbms() so it can't leak into later test modules diff --git a/tests/test_inference_engine.py b/tests/test_inference_engine.py index bbc0b5a1f15..066c70406c7 100644 --- a/tests/test_inference_engine.py +++ b/tests/test_inference_engine.py @@ -24,7 +24,7 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from _testutils import bootstrap, set_dbms +from _testutils import bootstrap, set_dbms, reset_dbms bootstrap() from lib.core.data import conf, kb @@ -151,3 +151,7 @@ def test_query_count_is_sublinear_in_charset(self): if __name__ == "__main__": unittest.main(verbosity=2) + + +def tearDownModule(): + reset_dbms() # clear any DBMS forced via set_dbms() so it can't leak into later test modules diff --git a/tests/test_ldap.py b/tests/test_ldap.py index f590dcfb846..469f4fed223 100644 --- a/tests/test_ldap.py +++ b/tests/test_ldap.py @@ -16,6 +16,21 @@ import lib.techniques.ldap.inject as ldap +# several setUps here write these conf keys without restoring them; snapshot/restore at the module +# boundary so they can't leak into later test modules (order-dependent flakiness) +_LDAP_CONF_KEYS = ("parameters", "paramDict", "skipUrlEncode", "cookieDel") +_saved_conf = {} + +def setUpModule(): + from lib.core.data import conf + for k in _LDAP_CONF_KEYS: + _saved_conf[k] = conf.get(k) + +def tearDownModule(): + from lib.core.data import conf + for k, v in _saved_conf.items(): + conf[k] = v + # --- Helpers ---------------------------------------------------------------- SENTINEL = ldap.SENTINEL diff --git a/tests/test_library.py b/tests/test_library.py new file mode 100644 index 00000000000..8437238e517 --- /dev/null +++ b/tests/test_library.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python + +""" +Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org) +See the file 'LICENSE' for copying permission + +Unit coverage for the library facade (import sqlmap; sqlmap.scan(...)). + +The facade drives the engine out-of-process through a generated configuration file (the same '-c' +mechanism the REST API uses) and reads back a '--report-json' report. These tests stub +subprocess.Popen to (a) capture the argv/config sqlmap.scan() builds from its keyword options and +(b) feed back a canned report - keeping the test fast, offline and network-free (no real scan runs). + +stdlib unittest only (no pytest / no pip); works on Python 2.7 and 3.x. +""" + +import json +import os +import re +import subprocess +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import sqlmap + + +class _FakePopen(object): + """Stub that records argv/config and writes a canned report to the config's 'reportJson' path.""" + + captured = {} + returncode = 0 + + def __init__(self, argv, **kwargs): + _FakePopen.captured["argv"] = argv + _FakePopen.captured["kwargs"] = kwargs + with open(argv[argv.index("-c") + 1]) as f: + config = f.read() + _FakePopen.captured["config"] = config + report = re.search(r"(?im)^reportjson\s*=\s*(.+)$", config).group(1).strip() + with open(report, "w") as f: + json.dump({"success": True, "data": [{"type_name": "BANNER", "value": "3.45.1"}], "error": []}, f) + + def wait(self, timeout=None): + return 0 + + def poll(self): + return 0 + + def kill(self): + pass + + +class TestLibraryFacade(unittest.TestCase): + def setUp(self): + self._realPopen = subprocess.Popen + subprocess.Popen = _FakePopen + _FakePopen.captured = {} + + def tearDown(self): + subprocess.Popen = self._realPopen + + def test_requires_a_target(self): + subprocess.Popen = self._realPopen # never reached; guard fires first + self.assertRaises(sqlmap.SqlmapError, sqlmap.scan) + + def test_rejects_unknown_option(self): + # a command line switch spelling (rather than a conf option name) must be rejected loudly + self.assertRaises(sqlmap.SqlmapError, sqlmap.scan, "http://target/?id=1", current_user=True) + + def test_options_go_through_config(self): + result = sqlmap.scan("http://target/vuln.php?id=1", technique="BEU", dumpTable=True, + tbl="users", level=3, getBanner=True, raw=["--fresh-queries"]) + argv = _FakePopen.captured["argv"] + config = _FakePopen.captured["config"] + # driven via a generated config file, stdin ignored, engine plumbing set - no arg escaping + self.assertIn("-c", argv) + self.assertIn("--ignore-stdin", argv) + self.assertIn("--fresh-queries", argv) # raw escape hatch stays on the CLI + # options land in the config using sqlmap's own (conf) names (ConfigParser lowercases keys) + self.assertTrue(re.search(r"(?im)^url\s*=\s*http://target/vuln.php\?id=1$", config)) + self.assertTrue(re.search(r"(?im)^technique\s*=\s*BEU$", config)) + self.assertTrue(re.search(r"(?im)^tbl\s*=\s*users$", config)) + self.assertTrue(re.search(r"(?im)^level\s*=\s*3$", config)) + self.assertTrue(re.search(r"(?im)^dumptable\s*=\s*True$", config)) + self.assertTrue(re.search(r"(?im)^getbanner\s*=\s*True$", config)) + self.assertTrue(re.search(r"(?im)^batch\s*=\s*True$", config)) + self.assertTrue(re.search(r"(?im)^outputdir\s*=", config)) # each run isolated on disk + # file descriptors are not leaked to the engine (matches the REST API subprocess) + self.assertFalse(_FakePopen.captured["kwargs"].get("close_fds") and os.name == "nt") + # canned report is returned verbatim + self.assertTrue(result["success"]) + self.assertEqual(result["data"][0]["value"], "3.45.1") + + def test_scan_from_request_uses_request_file(self): + sqlmap.scanFromRequest("/tmp/req.txt", technique="U") + config = _FakePopen.captured["config"] + self.assertTrue(re.search(r"(?im)^requestfile\s*=\s*/tmp/req.txt$", config)) + self.assertTrue(re.search(r"(?im)^technique\s*=\s*U$", config)) + + def test_missing_report_raises(self): + class _NoReportPopen(_FakePopen): + def __init__(self, argv, **kwargs): + _FakePopen.captured["argv"] = argv # write nothing -> no report file + subprocess.Popen = _NoReportPopen + self.assertRaises(sqlmap.SqlmapError, sqlmap.scan, "http://target/?id=1") + + +class TestReportErrorCapture(unittest.TestCase): + """ + The library tells failure modes apart (unreachable vs nothing-found) because a CLI --report-json + run now records error/critical log messages into the report 'error' array, like the REST API. + """ + + def test_errors_reach_the_report(self): + import logging + from lib.core.data import logger + from lib.utils.api import setupReportCollector, _assembleData, ReportErrorRecorder, REPORT_TASKID + + # represent a normal run: the shared test bootstrap silences the logger (CRITICAL+1), which would + # otherwise gate the ERROR record before it reaches the recorder (order-dependent flakiness) + saved_level = logger.level + logger.setLevel(logging.ERROR) + collector = setupReportCollector() + try: + logger.error("boom %s", "here") + result = _assembleData(collector, REPORT_TASKID) + self.assertTrue(any("boom here" in _ for _ in result["error"])) + finally: + logger.setLevel(saved_level) + for handler in list(logger.handlers): + if isinstance(handler, ReportErrorRecorder): + logger.removeHandler(handler) + collector.disconnect() + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tests/test_misc.py b/tests/test_misc.py index d92b72b17af..f3bf3faef25 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -13,7 +13,7 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from _testutils import bootstrap, set_dbms +from _testutils import bootstrap, set_dbms, reset_dbms bootstrap() from lib.core import common as C @@ -123,3 +123,7 @@ def test_roundtrip_scalar(self): if __name__ == "__main__": unittest.main(verbosity=2) + + +def tearDownModule(): + reset_dbms() # clear any DBMS forced via set_dbms() so it can't leak into later test modules diff --git a/tests/test_nosql.py b/tests/test_nosql.py index 3703471f8ce..d0987272669 100644 --- a/tests/test_nosql.py +++ b/tests/test_nosql.py @@ -18,6 +18,21 @@ import lib.techniques.nosql.inject as ni +# several setUps here write these conf keys without restoring them; snapshot/restore at the module +# boundary so they can't leak into later test modules (order-dependent flakiness) +_NOSQL_CONF_KEYS = ("parameters", "paramDict", "timeSec", "cookieDel") +_saved_conf = {} + +def setUpModule(): + from lib.core.data import conf + for k in _NOSQL_CONF_KEYS: + _saved_conf[k] = conf.get(k) + +def tearDownModule(): + from lib.core.data import conf + for k, v in _saved_conf.items(): + conf[k] = v + SECRET = "S3cr3t_9" MATCH = "Welcome user; rows: alpha, bravo, charlie" NOMATCH = "Invalid credentials; no rows" diff --git a/tests/test_openapi.py b/tests/test_openapi.py new file mode 100644 index 00000000000..40c8cd9305a --- /dev/null +++ b/tests/test_openapi.py @@ -0,0 +1,456 @@ +#!/usr/bin/env python + +""" +Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org) +See the file 'LICENSE' for copying permission + +Unit coverage for the OpenAPI/Swagger target extractor (lib/parse/openapi.py): schema example +synthesis, $ref resolution (incl. cycles), base-URL resolution (v2 + v3, relative/templated servers), +request-body handling (JSON / form), parameter->PLACE mapping, and (importantly) graceful handling of +malformed / poorly-defined specifications (a broken spec must never crash or hang the parser). + +stdlib unittest only (no pytest / no pip); works on Python 2.7 and 3.x. +""" + +import json +import os +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from _testutils import bootstrap +bootstrap() + +from lib.parse.openapi import openApiTargets, yaml as _yaml + +HAS_YAML = _yaml is not None + + +def _targets(spec, origin="http://h"): + return openApiTargets(json.dumps(spec) if isinstance(spec, dict) else spec, origin) + +def _byMethodPath(targets): + return dict(("%s %s" % (method, url), (method, url, data, headers)) for url, method, data, headers in targets) + + +class TestOpenApi(unittest.TestCase): + def test_v3_query_path_and_base(self): + spec = {"openapi": "3.0.0", "servers": [{"url": "/api"}], + "paths": {"/pet/{id}": {"get": {"parameters": [ + {"name": "id", "in": "path", "schema": {"type": "integer"}}, + {"name": "q", "in": "query", "schema": {"type": "string", "example": "x"}}]}}}} + targets = _targets(spec, "http://host:8080") + self.assertEqual(len(targets), 1) + url, method, data, headers = targets[0] + self.assertEqual(method, "GET") + from lib.core.settings import CUSTOM_INJECTION_MARK_CHAR as MARK + self.assertEqual(url, "http://host:8080/api/pet/1%s?q=x" % MARK) # relative server + filled+marked path + query + self.assertIsNone(data) + + def test_v3_json_body_sets_data_and_content_type(self): + spec = {"openapi": "3.0.0", "paths": {"/o": {"post": {"requestBody": {"content": {"application/json": + {"schema": {"type": "object", "properties": {"name": {"type": "string"}, "qty": {"type": "integer"}}}}}}}}}} + url, method, data, headers = _targets(spec)[0] + self.assertEqual(method, "POST") + self.assertEqual(json.loads(data), {"name": "1", "qty": 1}) + self.assertIn(("Content-Type", "application/json"), headers) + + def test_form_urlencoded_body(self): + spec = {"openapi": "3.0.0", "paths": {"/login": {"post": {"requestBody": {"content": + {"application/x-www-form-urlencoded": {"schema": {"type": "object", + "properties": {"u": {"type": "string"}, "p": {"type": "string"}}}}}}}}}} + url, method, data, headers = _targets(spec)[0] + self.assertEqual(sorted(data.split("&")), ["p=1", "u=1"]) + + def test_value_synthesis(self): + spec = {"openapi": "3.0.0", "paths": {"/x": {"get": {"parameters": [ + {"name": "a", "in": "query", "schema": {"type": "integer"}}, + {"name": "b", "in": "query", "schema": {"type": "boolean"}}, + {"name": "c", "in": "query", "schema": {"type": "string", "enum": ["first", "second"]}}, + {"name": "d", "in": "query", "schema": {"type": "string", "default": "dd"}}, + {"name": "e", "in": "query", "schema": {"type": "string", "format": "uuid"}}]}}}} + url = _targets(spec)[0][0] + self.assertIn("a=1", url) + self.assertIn("b=true", url) + self.assertIn("c=first", url) # enum[0] + self.assertIn("d=dd", url) # default + self.assertIn("e=11111111-1111-1111-1111-111111111111", url) # format uuid + + def test_ref_resolution_and_allof_oneof(self): + spec = {"openapi": "3.0.0", + "components": {"schemas": {"Tag": {"type": "object", "properties": {"n": {"type": "string"}}}}}, + "paths": { + "/ref": {"post": {"requestBody": {"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Tag"}}}}}}, + "/all": {"post": {"requestBody": {"content": {"application/json": {"schema": {"allOf": [ + {"type": "object", "properties": {"x": {"type": "string"}}}, + {"type": "object", "properties": {"y": {"type": "integer"}}}]}}}}}}, + "/one": {"post": {"requestBody": {"content": {"application/json": {"schema": {"oneOf": [ + {"type": "object", "properties": {"only": {"type": "string"}}}, + {"type": "object", "properties": {"other": {"type": "string"}}}]}}}}}}}} + m = _byMethodPath(_targets(spec)) + self.assertEqual(json.loads(m["POST http://h/ref"][2]), {"n": "1"}) + self.assertEqual(json.loads(m["POST http://h/all"][2]), {"x": "1", "y": 1}) # allOf merged + self.assertEqual(json.loads(m["POST http://h/one"][2]), {"only": "1"}) # oneOf -> first + + def test_ref_cycle_terminates(self): + spec = {"openapi": "3.0.0", + "components": {"schemas": {"Node": {"type": "object", "properties": { + "name": {"type": "string"}, "parent": {"$ref": "#/components/schemas/Node"}}}}}, + "paths": {"/n": {"post": {"requestBody": {"content": {"application/json": + {"schema": {"$ref": "#/components/schemas/Node"}}}}}}}} + targets = _targets(spec) # must not hang / recurse forever + self.assertEqual(len(targets), 1) + self.assertTrue(json.loads(targets[0][2]).get("name") == "1") + + def test_swagger_v2_base_and_body(self): + spec = {"swagger": "2.0", "host": "api.example.com", "basePath": "/v2", "schemes": ["https"], + "paths": {"/pet": {"post": {"parameters": [{"name": "b", "in": "body", + "schema": {"type": "object", "properties": {"id": {"type": "integer"}}}}]}}}} + url, method, data, headers = _targets(spec, None)[0] + self.assertEqual(url, "https://api.example.com/v2/pet") + self.assertEqual(json.loads(data), {"id": 1}) + + def test_server_template_variables(self): + spec = {"openapi": "3.0.0", "servers": [{"url": "https://{env}.x.io/{ver}", + "variables": {"env": {"default": "prod"}, "ver": {"default": "v3"}}}], + "paths": {"/p": {"get": {}}}} + self.assertEqual(_targets(spec, None)[0][0], "https://prod.x.io/v3/p") + + def test_headers_are_hashable_tuples(self): + # kb.targets is an OrderedSet, so the emitted headers must be hashable (tuple, not list) + spec = {"openapi": "3.0.0", "paths": {"/x": {"get": {"parameters": [ + {"name": "h", "in": "header", "schema": {"type": "string"}}]}}}} + headers = _targets(spec)[0][3] + self.assertTrue(headers is None or isinstance(tuple(headers), tuple)) + + def test_header_and_cookie_params_are_injection_marked(self): + # header/cookie params get the custom injection mark ('*') appended so they become testable + # (custom) injection points (query/body params are still auto-tested alongside them) + from lib.core.settings import CUSTOM_INJECTION_MARK_CHAR as MARK + spec = {"openapi": "3.0.0", "paths": {"/x": {"get": {"parameters": [ + {"name": "X-Api", "in": "header", "schema": {"type": "string", "example": "k"}}, + {"name": "sess", "in": "cookie", "schema": {"type": "string", "example": "v"}}]}}}} + headers = dict(_targets(spec)[0][3]) + self.assertEqual(headers["X-Api"], "k" + MARK) + self.assertEqual(headers["Cookie"], "sess=v" + MARK) + + # --- graceful degradation: a broken/poorly-defined spec must never crash the parser --- + + def test_malformed_raises_valueerror(self): + for bad in ("{not json,,,", "[1,2,3]", "{}", '{"openapi":"3.0.0"}', '{"openapi":"3.0.0","paths":[1,2]}'): + self.assertRaises(ValueError, openApiTargets, bad, "http://h") + + def test_malformed_servers_do_not_crash(self): + for servers in ('{"url":"/a"}', '"http://h"', "[]"): + spec = '{"openapi":"3.0.0","servers":%s,"paths":{"/x":{"get":{}}}}' % servers + self.assertEqual(len(openApiTargets(spec, "http://h")), 1) # no crash, still one target + + def test_url_and_body_values_are_encoded(self): + # special characters in synthesized values must be percent-encoded so they can not break the + # URL structure (param smuggling) or the form body + spec = {"openapi": "3.0.0", "paths": { + "/x/{p}": {"get": {"parameters": [ + {"name": "p", "in": "path", "schema": {"type": "string", "example": "a/b"}}, + {"name": "q", "in": "query", "schema": {"type": "string", "example": "a b&c=d"}}]}}, + "/f": {"post": {"requestBody": {"content": {"application/x-www-form-urlencoded": + {"schema": {"type": "object", "properties": {"u": {"type": "string", "example": "a b&x"}}}}}}}}}} + byMethod = dict((method, (url, data)) for url, method, data, headers in _targets(spec)) + getUrl = byMethod["GET"][0] + self.assertIn("/x/a%2Fb", getUrl) # path value '/' encoded (no extra segment) + self.assertIn("q=a%20b%26c%3Dd", getUrl) # query value space/&/= encoded (no smuggling) + self.assertNotIn(" ", getUrl) + self.assertEqual(byMethod["POST"][1], "u=a%20b%26x") + + @unittest.skipUnless(HAS_YAML, "pyyaml not available") + def test_yaml_spec(self): + y = ("openapi: 3.0.0\n" + "paths:\n" + " /y:\n" + " get:\n" + " parameters:\n" + " - name: q\n" + " in: query\n" + " schema: {type: string, example: hi}\n") + targets = openApiTargets(y, "http://h") + self.assertEqual(len(targets), 1) + self.assertEqual(targets[0][0], "http://h/y?q=hi") + + def test_shared_recursive_refs_scale(self): + # a self-referential schema reused across many operations must terminate promptly (depth cap + + # per-$ref memoization); without them this would blow up exponentially and hang the test + schemas = {"Node": {"type": "object", "properties": { + "name": {"type": "string"}, + "child": {"$ref": "#/components/schemas/Node"}, + "list": {"type": "array", "items": {"$ref": "#/components/schemas/Node"}}}}} + paths = dict(("/n%d" % i, {"post": {"requestBody": {"content": {"application/json": + {"schema": {"$ref": "#/components/schemas/Node"}}}}}}) for i in range(60)) + targets = _targets({"openapi": "3.0.0", "components": {"schemas": schemas}, "paths": paths}) + self.assertEqual(len(targets), 60) + self.assertEqual(json.loads(targets[0][2]).get("name"), "1") + + def test_swagger_v2_formdata_body(self): + # in:"formData" params must become a urlencoded body (previously dropped -> empty POST) + spec = {"swagger": "2.0", "host": "h", "paths": {"/l": {"post": {"parameters": [ + {"name": "u", "in": "formData", "type": "string"}, + {"name": "p", "in": "formData", "type": "string"}]}}}} + url, method, data, headers = _targets(spec, None)[0] + self.assertEqual(method, "POST") + self.assertEqual(sorted(data.split("&")), ["p=1", "u=1"]) + + def test_relative_base_is_skipped(self): + # a spec that yields no scheme/host (relative server + no origin) must be skipped, not emitted + spec = {"openapi": "3.0.0", "servers": [{"url": "/api"}], "paths": {"/x": {"get": {}}}} + self.assertEqual(openApiTargets(json.dumps(spec), None), []) # relative -> skipped + self.assertEqual(len(openApiTargets(json.dumps(spec), "http://h")), 1) # absolute with origin -> kept + + def test_unsupported_body_media_type_no_crash(self): + # a structured body under a non-JSON/form media type must not crash and must not fabricate a body, + # but the endpoint URL is still produced + spec = {"openapi": "3.0.0", "paths": {"/x": {"post": {"requestBody": {"content": {"application/xml": + {"schema": {"type": "object", "properties": {"a": {"type": "string"}}}}}}}}}} + url, method, data, headers = _targets(spec)[0] + self.assertEqual((url, method, data), ("http://h/x", "POST", None)) + + def test_injection_mark_char_in_value_is_not_doubled(self): + # an example value already containing the custom injection mark must not create a stray point + from lib.core.settings import CUSTOM_INJECTION_MARK_CHAR as MARK + spec = {"openapi": "3.0.0", "paths": {"/x": {"post": { + "parameters": [{"name": "H", "in": "header", "schema": {"type": "string", "example": "a%sb" % MARK}}], + "requestBody": {"content": {"application/json": {"schema": {"type": "object", + "properties": {"n": {"type": "string", "example": "x%sy" % MARK}}}}}}}}}} + url, method, data, headers = _targets(spec)[0] + self.assertEqual(dict(headers)["H"], "ab" + MARK) # single trailing mark only + self.assertEqual(json.loads(data), {"n": "xy"}) # mark stripped from body value + + @unittest.skipUnless(HAS_YAML, "pyyaml not available") + def test_non_string_method_keys_do_not_crash(self): + # YAML path-item keys are not guaranteed to be strings (404 -> int, on -> bool); must not crash + y = ("openapi: 3.0.0\n" + "servers: [{url: 'http://h'}]\n" + "paths:\n" + " /x:\n" + " get: {}\n" + " 404: {}\n" + " on: {}\n") + targets = openApiTargets(y, "http://h") + self.assertEqual(len(targets), 1) # only the real GET operation + self.assertEqual(targets[0][1], "GET") + + def test_hostile_base_url_metadata_does_not_crash(self): + # _baseUrl runs once, OUTSIDE the per-operation try, so malformed server/scheme/basePath metadata + # must not raise (it would abort the entire extraction) + hostile = [ + {"openapi": "3.0.0", "servers": [{"url": "https://{e}.x/", "variables": [1, 2]}], "paths": {"/x": {"get": {}}}}, + {"openapi": "3.0.0", "servers": [{"url": "https://{e}.x/", "variables": {"e": "prod"}}], "paths": {"/x": {"get": {}}}}, + {"openapi": "3.0.0", "servers": [{"url": 123}], "paths": {"/x": {"get": {}}}}, + {"swagger": "2.0", "host": "h", "schemes": {"a": 1}, "paths": {"/x": {"get": {}}}}, + {"swagger": "2.0", "host": "h", "basePath": 123, "paths": {"/x": {"get": {}}}}] + for spec in hostile: + self.assertEqual(len(_targets(spec)), 1) # no crash, still one target + + def test_param_entry_not_a_dict_is_skipped(self): + spec = {"openapi": "3.0.0", "paths": {"/x": {"get": {"parameters": ["oops", {"name": "q", "in": "query"}]}}}} + self.assertIn("q=1", _targets(spec)[0][0]) # bad entry skipped, good one still used + + @unittest.skipUnless(HAS_YAML, "pyyaml not available") + def test_yaml_date_examples_serialize(self): + # unquoted YAML dates parse to datetime.date, which is not JSON-serializable -> must be stringified, + # not silently dropped (dates are pervasive in real specs) + y = ("openapi: 3.0.0\n" + "servers: [{url: 'http://h'}]\n" + "paths:\n" + " /x:\n" + " post:\n" + " requestBody:\n" + " content:\n" + " application/json:\n" + " schema: {type: object, properties: {created: {type: string, example: 2020-01-01}}}\n") + url, method, data, headers = openApiTargets(y, "http://h")[0] + self.assertEqual(json.loads(data), {"created": "2020-01-01"}) + + def test_crlf_in_header_and_cookie_is_stripped(self): + # a spec-supplied header/cookie name or value must not carry CR/LF (header injection / request + # corruption); query/path values are separately percent-encoded + spec = {"openapi": "3.0.0", "paths": {"/x": {"get": {"parameters": [ + {"name": "X-A", "in": "header", "schema": {"type": "string", "example": "a\r\nX-Evil: 1"}}, + {"name": "X\r\nB", "in": "header", "schema": {"type": "string", "example": "v"}}, + {"name": "sid", "in": "cookie", "schema": {"type": "string", "example": "a\r\nSet: x"}}]}}}} + headers = dict(_targets(spec)[0][3]) + for name, value in headers.items(): + self.assertNotIn("\r", name + value) + self.assertNotIn("\n", name + value) + self.assertIn("X-A", headers) + self.assertIn("XB", headers) # control chars removed from the name + + def test_explicit_examples_preferred_over_schema(self): + # a concrete example/examples on the media-type or parameter object must win over schema synthesis + # (real specs carry the canonical, validation-passing value there) + body = {"openapi": "3.0.0", "paths": {"/x": {"post": {"requestBody": {"content": {"application/json": { + "schema": {"type": "object", "properties": {"name": {"type": "string"}}}, "example": {"name": "real"}}}}}}}} + self.assertEqual(json.loads(_targets(body)[0][2]), {"name": "real"}) + examples = {"openapi": "3.0.0", "paths": {"/x": {"post": {"requestBody": {"content": {"application/json": { + "schema": {"type": "object"}, "examples": {"first": {"value": {"k": "v1"}}}}}}}}}} + self.assertEqual(json.loads(_targets(examples)[0][2]), {"k": "v1"}) + param = {"openapi": "3.0.0", "paths": {"/x": {"get": {"parameters": [ + {"name": "q", "in": "query", "example": "E", "schema": {"type": "string"}}]}}}} + self.assertIn("q=E", _targets(param)[0][0]) + + def test_openapi_31_const_and_type_array(self): + spec = {"openapi": "3.1.0", "paths": {"/x": {"get": {"parameters": [ + {"name": "c", "in": "query", "schema": {"const": "CV"}}, + {"name": "n", "in": "query", "schema": {"type": ["integer", "null"]}}]}}}} + url = _targets(spec)[0][0] + self.assertIn("c=CV", url) # const used + self.assertIn("n=1", url) # ["integer","null"] resolved to integer, not the generic fallback + + def test_parameter_names_are_encoded(self): + # a param NAME with structural chars must be encoded so it can not split/smuggle params or truncate + # at a fragment; deep-object brackets ([]) are preserved + spec = {"openapi": "3.0.0", "paths": { + "/q": {"get": {"parameters": [ + {"name": "a&b=c", "in": "query", "schema": {"type": "string"}}, + {"name": "a#b", "in": "query", "schema": {"type": "string"}}, + {"name": "filter[status]", "in": "query", "schema": {"type": "string"}}]}}, + "/f": {"post": {"requestBody": {"content": {"application/x-www-form-urlencoded": + {"schema": {"type": "object", "properties": {"x&y": {"type": "string"}}}}}}}}}} + byMethod = dict((method, (url, data)) for url, method, data, headers in _targets(spec)) + getUrl = byMethod["GET"][0] + self.assertIn("a%26b%3Dc=1", getUrl) + self.assertIn("a%23b=1", getUrl) + self.assertIn("filter[status]=1", getUrl) # brackets kept (deep-object param names) + self.assertNotIn("#", getUrl) + self.assertEqual(byMethod["POST"][1], "x%26y=1") + + def test_undefined_template_var_does_not_leak(self): + # a server/path template variable with no definition must not leave a literal '{...}' in the URL + spec = {"openapi": "3.0.0", "servers": [{"url": "https://api.x.com/{basePath}/v3"}], + "paths": {"/pets": {"get": {}}}} + url = _targets(spec, "http://h")[0][0] + self.assertNotIn("{", url) + self.assertEqual(url, "https://api.x.com/1/v3/pets") # absolute server used as-is (host not rewritten) + + def test_absolute_server_url_is_not_rewritten_to_origin(self): + # a spec served from one host but declaring an absolute API server on another host must scan the + # DECLARED API host, not the spec's origin + spec = {"openapi": "3.0.0", "servers": [{"url": "https://api.example.com/v1"}], + "paths": {"/pets": {"get": {}}}} + self.assertEqual(_targets(spec, "https://docs.example.com")[0][0], "https://api.example.com/v1/pets") + + def test_path_parameter_is_injection_marked(self): + from lib.core.settings import CUSTOM_INJECTION_MARK_CHAR as MARK + spec = {"openapi": "3.0.0", "paths": {"/users/{id}": {"get": {"parameters": [ + {"name": "id", "in": "path", "schema": {"type": "integer"}}]}}}} + self.assertEqual(_targets(spec)[0][0], "http://h/users/1" + MARK) + + def test_form_urlencoded_sets_content_type_and_multipart_skipped(self): + form = {"openapi": "3.0.0", "paths": {"/f": {"post": {"requestBody": {"content": + {"application/x-www-form-urlencoded": {"schema": {"type": "object", "properties": {"u": {"type": "string"}}}}}}}}}} + url, method, data, headers = _targets(form)[0] + self.assertEqual(data, "u=1") + self.assertIn(("Content-Type", "application/x-www-form-urlencoded"), headers) + multipart = {"openapi": "3.0.0", "paths": {"/m": {"post": {"requestBody": {"content": + {"multipart/form-data": {"schema": {"type": "object", "properties": {"u": {"type": "string"}}}}}}}}}} + url, method, data, headers = _targets(multipart)[0] + self.assertIsNone(data) # multipart is skipped, not mis-serialized as urlencoded + + def test_path_item_ref_is_resolved(self): + spec = {"openapi": "3.1.0", + "components": {"pathItems": {"Ping": {"get": {"parameters": [ + {"name": "q", "in": "query", "schema": {"type": "string", "example": "z"}}]}}}}, + "paths": {"/ping": {"$ref": "#/components/pathItems/Ping"}}} + targets = _targets(spec) + self.assertEqual(len(targets), 1) + self.assertIn("q=z", targets[0][0]) + + def test_operation_parameter_overrides_path_level(self): + spec = {"openapi": "3.0.0", "paths": {"/x": { + "parameters": [{"name": "q", "in": "query", "schema": {"type": "string", "example": "shared"}}], + "get": {"parameters": [{"name": "q", "in": "query", "schema": {"type": "string", "example": "op"}}]}}}} + url = _targets(spec)[0][0] + self.assertIn("q=op", url) # operation value wins + self.assertEqual(url.count("q="), 1) # not duplicated + + def test_multiple_cookies_aggregate_into_one_header(self): + from lib.core.settings import CUSTOM_INJECTION_MARK_CHAR as MARK + spec = {"openapi": "3.0.0", "paths": {"/x": {"get": {"parameters": [ + {"name": "a", "in": "cookie", "schema": {"type": "string"}}, + {"name": "b", "in": "cookie", "schema": {"type": "string"}}]}}}} + headers = _targets(spec)[0][3] + cookieHeaders = [v for (k, v) in headers if k == "Cookie"] + self.assertEqual(cookieHeaders, ["a=1%s; b=1%s" % (MARK, MARK)]) # one aggregated Cookie header + + def test_cookie_name_value_cannot_smuggle_pairs(self): + # a cookie name that is not a token is dropped; structural chars in the value ('; ,' / whitespace) + # are stripped so a spec can not inject additional cookie pairs + spec = {"openapi": "3.0.0", "paths": {"/x": {"get": {"parameters": [ + {"name": "a; injected", "in": "cookie", "schema": {"type": "string"}}, + {"name": "sid", "in": "cookie", "schema": {"type": "string", "example": "v; z=1"}}]}}}} + cookieHeaders = [v for (k, v) in (_targets(spec)[0][3] or []) if k == "Cookie"] + self.assertEqual(len(cookieHeaders), 1) + cookie = cookieHeaders[0] + self.assertNotIn(";", cookie.rstrip("*")) # no interior ';' -> no smuggled pair + self.assertNotIn("injected", cookie) # invalid cookie name dropped + self.assertNotIn(" ", cookie) + + def test_loose_path_without_leading_slash(self): + # a malformed path key missing its leading '/' must not glue onto the base (".../v1pets") + spec = {"openapi": "3.0.0", "servers": [{"url": "https://api.x/v1"}], "paths": {"pets": {"get": {}}}} + self.assertEqual(_targets(spec, None)[0][0], "https://api.x/v1/pets") + + def test_array_query_param_is_best_effort_scalar(self): + # documents current best-effort behavior: an array query param is scalarized+encoded, NOT expanded + # per style/explode. If richer serialization is added later, update this expectation deliberately. + spec = {"openapi": "3.0.0", "paths": {"/x": {"get": {"parameters": [ + {"name": "ids", "in": "query", "schema": {"type": "array", "items": {"type": "integer"}}}]}}}} + url = _targets(spec)[0][0] + self.assertIn("ids=", url) + self.assertNotIn(" ", url) # whatever the encoding, it must not break the URL + self.assertTrue(url.startswith("http://h/x?ids=")) + + def test_invalid_header_name_is_skipped(self): + spec = {"openapi": "3.0.0", "paths": {"/x": {"get": {"parameters": [ + {"name": "Bad Name", "in": "header", "schema": {"type": "string"}}, + {"name": "Also:Bad", "in": "header", "schema": {"type": "string"}}, + {"name": "X-Good", "in": "header", "schema": {"type": "string"}}]}}}} + headers = dict(_targets(spec)[0][3] or []) + self.assertIn("X-Good", headers) + self.assertNotIn("Bad Name", headers) + self.assertNotIn("Also:Bad", headers) + + def test_explicit_null_example_falls_back_to_schema(self): + # 'example: null' must not serialize as null/"null" - fall back to schema synthesis + q = {"openapi": "3.0.0", "paths": {"/x": {"get": {"parameters": [ + {"name": "q", "in": "query", "example": None, "schema": {"type": "string", "example": "good"}}]}}}} + self.assertIn("q=good", _targets(q)[0][0]) + b = {"openapi": "3.0.0", "paths": {"/x": {"post": {"requestBody": {"content": {"application/json": + {"example": None, "schema": {"type": "object", "properties": {"a": {"type": "integer"}}}}}}}}}} + self.assertEqual(json.loads(_targets(b)[0][2]), {"a": 1}) + + def test_degrade_not_skip_on_odd_shapes(self): + # enum-as-dict, non-string param name, and content[type]-as-list must degrade (op preserved) + for spec in ( + {"openapi": "3.0.0", "paths": {"/x": {"get": {"parameters": [{"name": "q", "in": "query", "schema": {"enum": {"a": 1}}}]}}}}, + {"openapi": "3.0.0", "paths": {"/x": {"get": {"parameters": [{"name": 5, "in": "header", "schema": {"type": "string"}}]}}}}, + {"openapi": "3.0.0", "paths": {"/x": {"post": {"requestBody": {"content": {"application/json": [1, 2]}}}}}}): + self.assertEqual(len(_targets(spec)), 1) + + def test_malformed_ref_and_properties_degrade_not_skip(self): + # a non-string/unhashable $ref or a non-dict 'properties' must degrade the value (not lose the op) + for schema in ({"$ref": 123}, {"$ref": [1, 2]}, {"type": "object", "properties": [1, 2]}): + spec = {"openapi": "3.0.0", "paths": {"/x": {"post": {"requestBody": + {"content": {"application/json": {"schema": schema}}}}}}} + self.assertEqual(len(_targets(spec)), 1) # operation preserved, not skipped + + def test_undefined_bits_are_skipped_not_fatal(self): + spec = {"openapi": "3.0.0", "paths": { + "/a": {"get": {"parameters": [{}]}}, # param with no name + "/b": {"post": {"requestBody": {"content": {"application/json": + {"schema": {"$ref": "#/components/schemas/DoesNotExist"}}}}}}, # dangling $ref + "/c": {"get": {"parameters": [{"name": "p", "in": "query", + "schema": {"$ref": "https://other/x.json#/Y"}}]}}}} # external $ref + targets = _targets(spec) + self.assertEqual(len(targets), 3) # all three still produced + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_pagecontent.py b/tests/test_pagecontent.py index 3f6edcf500d..6d777ef21d8 100644 --- a/tests/test_pagecontent.py +++ b/tests/test_pagecontent.py @@ -60,6 +60,7 @@ def test_multiple_tags(self): class TestParseSqliteTableSchema(unittest.TestCase): def setUp(self): + self.addCleanup(setattr, kb.data, "cachedColumns", kb.data.get("cachedColumns")) kb.data.cachedColumns = {} def _cols(self): diff --git a/tests/test_parse_modules.py b/tests/test_parse_modules.py index 37e90cc2eaf..f94a4d27b10 100644 --- a/tests/test_parse_modules.py +++ b/tests/test_parse_modules.py @@ -18,7 +18,7 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from _testutils import bootstrap, set_dbms +from _testutils import bootstrap, set_dbms, reset_dbms bootstrap() from lib.core.data import kb, conf @@ -173,3 +173,7 @@ def test_missing_target_section_raises(self): if __name__ == "__main__": unittest.main() + + +def tearDownModule(): + reset_dbms() # clear any DBMS forced via set_dbms() so it can't leak into later test modules diff --git a/tests/test_payload_marking.py b/tests/test_payload_marking.py index 04f97941ab4..f0271bf9c53 100644 --- a/tests/test_payload_marking.py +++ b/tests/test_payload_marking.py @@ -28,6 +28,24 @@ # change there is reflected here too. MARK = CUSTOM_INJECTION_MARK_CHAR +# the _drive_* helpers set sticky conf/kb flags (notably conf.hpp, which changes queryPage +# behaviour) without restoring them; snapshot/restore at the module boundary so they can't leak +_PM_CONF_KEYS = ("hpp", "skipUrlEncode", "method", "paramDel", "url", "data", "parameters", "paramDict") +_PM_KB_KEYS = ("tamperFunctions", "postHint", "customInjectionMark", "postUrlEncode", "postSpaceToPlus", "processUserMarks") +_pm_saved = {} + +def setUpModule(): + from lib.core.data import conf, kb + for k in _PM_CONF_KEYS: + _pm_saved[("conf", k)] = conf.get(k) + for k in _PM_KB_KEYS: + _pm_saved[("kb", k)] = kb.get(k) + +def tearDownModule(): + from lib.core.data import conf, kb + for (scope, k), v in _pm_saved.items(): + (conf if scope == "conf" else kb)[k] = v + def classify(d): if re.search(JSON_RECOGNITION_REGEX, d): diff --git a/tests/test_property.py b/tests/test_property.py index 04cf72180b1..789eea47633 100644 --- a/tests/test_property.py +++ b/tests/test_property.py @@ -28,7 +28,7 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from _testutils import bootstrap, for_all, set_dbms +from _testutils import bootstrap, for_all, set_dbms, reset_dbms bootstrap() from extra.cloak.cloak import cloak, decloak @@ -272,3 +272,7 @@ def test_stdoutencode(self): if __name__ == "__main__": unittest.main() + + +def tearDownModule(): + reset_dbms() # clear any DBMS forced via set_dbms() so it can't leak into later test modules diff --git a/tests/test_purge.py b/tests/test_purge.py index b4520f40444..c532d7b73cf 100644 --- a/tests/test_purge.py +++ b/tests/test_purge.py @@ -83,7 +83,10 @@ def test_overwrites_and_truncates_file_contents(self): nonempty = [p for p in survivors if os.path.getsize(p) > 0] self.assertEqual(nonempty, [], msg="files were not truncated to zero: %r" % nonempty) - blob = b"".join(open(p, "rb").read() for p in survivors) + blob = b"" + for p in survivors: + with open(p, "rb") as fh: + blob += fh.read() for secret in plaintexts.values(): self.assertNotIn(secret.encode("utf-8"), blob, msg="original plaintext %r survived the purge" % secret) diff --git a/tests/test_report.py b/tests/test_report.py index 63c4fd7e06a..d5dade14161 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -38,6 +38,12 @@ def setUp(self): def tearDown(self): kb.partRun = self._saved_partRun + # setupReportCollector() attaches a ReportErrorRecorder to the GLOBAL logger; drop it so it does + # not leak a handler bound to a now-closed collector into later tests + from lib.core.data import logger + for handler in list(logger.handlers): + if isinstance(handler, api.ReportErrorRecorder): + logger.removeHandler(handler) try: self.c.disconnect() except Exception: diff --git a/tests/test_search_enum.py b/tests/test_search_enum.py index ae9437ec7e0..66b3b850a5c 100644 --- a/tests/test_search_enum.py +++ b/tests/test_search_enum.py @@ -20,7 +20,7 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from _testutils import bootstrap, set_dbms +from _testutils import bootstrap, set_dbms, reset_dbms bootstrap() @@ -548,3 +548,7 @@ def test_search_column_mysql_lt5_bruteforce_decline(self): if __name__ == "__main__": unittest.main() + + +def tearDownModule(): + reset_dbms() # clear any DBMS forced via set_dbms() so it can't leak into later test modules diff --git a/tests/test_sgmllib.py b/tests/test_sgmllib.py index 5343ef95260..4195ed8b1f2 100644 --- a/tests/test_sgmllib.py +++ b/tests/test_sgmllib.py @@ -191,6 +191,7 @@ def test_convert_codepoint(self): class TestCustomEntitydefs(unittest.TestCase): def test_custom_entity(self): p = RecordingParser() + p.entitydefs = dict(p.entitydefs) # shadow the shared SGMLParser class dict so 'copy' doesn't leak process-wide p.entitydefs["copy"] = "\xa9" p.feed("©") p.close() diff --git a/tests/test_ssti.py b/tests/test_ssti.py index 96b714bc0cb..8a5e15e9a4d 100644 --- a/tests/test_ssti.py +++ b/tests/test_ssti.py @@ -393,8 +393,7 @@ def setUp(self): def tearDown(self): ssti._send = self.original_send - if self.original_dumper is not None: - ssti.conf.dumper = self.original_dumper + ssti.conf.dumper = self.original_dumper # restore unconditionally (was None -> don't leak the mock dumper) def test_error_page_skipped(self): """RCE payload that triggers a template error is skipped; next payload tried.""" diff --git a/tests/test_target_parsing.py b/tests/test_target_parsing.py index 0dcd8312c87..c5a981f4a5a 100644 --- a/tests/test_target_parsing.py +++ b/tests/test_target_parsing.py @@ -22,6 +22,7 @@ All expected values below were probed from actual output, not assumed. """ +import atexit import os import shutil import sys @@ -29,7 +30,7 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from _testutils import bootstrap +from _testutils import bootstrap, reset_dbms bootstrap() from lib.core.data import conf @@ -62,7 +63,8 @@ from lib.core.target import _setResultsFile from lib.core.target import initTargetEnv -SCRATCH = "/tmp/claude-1000/-tmp-tmp-oUnlQJzlQN/fcd55d25-6313-49ed-817e-dcbe7fc2bf22/scratchpad" +SCRATCH = tempfile.mkdtemp(prefix="sqlmap-tests-") # per-run temp dir (portable; replaces a stale hardcoded path) +atexit.register(lambda: shutil.rmtree(SCRATCH, ignore_errors=True)) # conf/kb keys that the tests below mutate; saved in setUp, restored in tearDown so # one test can never leak global state into another (or into the rest of the suite). @@ -519,3 +521,7 @@ def test_creates_csv_with_header_in_multiple_target_mode(self): if __name__ == "__main__": unittest.main(verbosity=2) + + +def tearDownModule(): + reset_dbms() # clear any DBMS forced via set_dbms() so it can't leak into later test modules diff --git a/tests/test_targeturl.py b/tests/test_targeturl.py index a0e05ac851d..74c14c07163 100644 --- a/tests/test_targeturl.py +++ b/tests/test_targeturl.py @@ -26,6 +26,20 @@ from lib.core.common import parseTargetUrl from lib.core.data import conf +_TARGETURL_KEYS = ("url", "hostname", "port", "scheme", "path") +_saved = {} + + +def setUpModule(): + for k in _TARGETURL_KEYS: + _saved[k] = conf.get(k) + + +def tearDownModule(): + # parseTargetUrl() writes these onto the global conf singleton; restore so it can't leak to later modules + for k, v in _saved.items(): + conf[k] = v + def _parse(url): conf.url = url diff --git a/tests/test_techniques.py b/tests/test_techniques.py index c1f1b6313f3..6ab50de7eb5 100644 --- a/tests/test_techniques.py +++ b/tests/test_techniques.py @@ -30,7 +30,7 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from _testutils import bootstrap, set_dbms +from _testutils import bootstrap, set_dbms, reset_dbms bootstrap() from lib.core.data import conf, kb @@ -1518,3 +1518,7 @@ def test_non_string_char_ignored(self): if __name__ == "__main__": unittest.main(verbosity=2) + + +def tearDownModule(): + reset_dbms() # clear any DBMS forced via set_dbms() so it can't leak into later test modules diff --git a/tests/test_texthelpers.py b/tests/test_texthelpers.py index 2726e6747fe..0df01ee7a98 100644 --- a/tests/test_texthelpers.py +++ b/tests/test_texthelpers.py @@ -46,6 +46,7 @@ def test_all_match(self): class TestParseFilePaths(unittest.TestCase): def setUp(self): + self.addCleanup(setattr, kb, "absFilePaths", kb.get("absFilePaths")) kb.absFilePaths = set() def test_unix_paths_from_php_error(self): diff --git a/tests/test_threads.py b/tests/test_threads.py index 28a852850a5..602d2c5acb8 100644 --- a/tests/test_threads.py +++ b/tests/test_threads.py @@ -38,6 +38,7 @@ def test_get_current_thread_data_is_threadlocal(self): # ATTRIBUTE STATE is per-thread. Verify both: same object, independent state. main = T.getCurrentThreadData() self.assertIs(main, T.getCurrentThreadData()) # stable within a thread + self.addCleanup(main.reset) # don't leak the main thread's mutated state to later tests main.retriesCount = 111 diff --git a/tests/test_union_engine.py b/tests/test_union_engine.py index 97ac88081d4..f0592fe4e10 100644 --- a/tests/test_union_engine.py +++ b/tests/test_union_engine.py @@ -23,7 +23,7 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from _testutils import bootstrap, set_dbms +from _testutils import bootstrap, set_dbms, reset_dbms bootstrap() from lib.core.data import conf, kb @@ -105,3 +105,7 @@ def test_detect_beyond_first_step(self): if __name__ == "__main__": unittest.main(verbosity=2) + + +def tearDownModule(): + reset_dbms() # clear any DBMS forced via set_dbms() so it can't leak into later test modules diff --git a/tests/test_users_enum.py b/tests/test_users_enum.py index d23e2db17be..f20c143280a 100644 --- a/tests/test_users_enum.py +++ b/tests/test_users_enum.py @@ -20,7 +20,7 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from _testutils import bootstrap, set_dbms +from _testutils import bootstrap, set_dbms, reset_dbms bootstrap() @@ -476,3 +476,7 @@ def test_is_dba_mssql(self): if __name__ == "__main__": unittest.main() + + +def tearDownModule(): + reset_dbms() # clear any DBMS forced via set_dbms() so it can't leak into later test modules diff --git a/tests/test_xpath.py b/tests/test_xpath.py index 2c3dcfac1a5..99903382ada 100644 --- a/tests/test_xpath.py +++ b/tests/test_xpath.py @@ -9,8 +9,11 @@ formatting can be exercised without a live target. """ +import os +import sys import unittest +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from _testutils import bootstrap bootstrap() diff --git a/thirdparty/odict/__init__.py b/thirdparty/odict/__init__.py deleted file mode 100644 index 8571776ae42..00000000000 --- a/thirdparty/odict/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env python - -import sys - -if sys.version_info[:2] >= (2, 7): - from collections import OrderedDict -else: - from ordereddict import OrderedDict diff --git a/thirdparty/odict/ordereddict.py b/thirdparty/odict/ordereddict.py deleted file mode 100644 index 1cdd6f46edc..00000000000 --- a/thirdparty/odict/ordereddict.py +++ /dev/null @@ -1,133 +0,0 @@ -# Copyright (c) 2009 Raymond Hettinger -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation files -# (the "Software"), to deal in the Software without restriction, -# including without limitation the rights to use, copy, modify, merge, -# publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, -# subject to the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -try: - from UserDict import DictMixin -except ImportError: - try: - from collections.abc import MutableMapping as DictMixin - except ImportError: - from collections import MutableMapping as DictMixin - -class OrderedDict(dict, DictMixin): - - def __init__(self, *args, **kwds): - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - try: - self.__end - except AttributeError: - self.clear() - self.update(*args, **kwds) - - def clear(self): - self.__end = end = [] - end += [None, end, end] # sentinel node for doubly linked list - self.__map = {} # key --> [key, prev, next] - dict.clear(self) - - def __setitem__(self, key, value): - if key not in self: - end = self.__end - curr = end[1] - curr[2] = end[1] = self.__map[key] = [key, curr, end] - dict.__setitem__(self, key, value) - - def __delitem__(self, key): - dict.__delitem__(self, key) - key, prev, next = self.__map.pop(key) - prev[2] = next - next[1] = prev - - def __iter__(self): - end = self.__end - curr = end[2] - while curr is not end: - yield curr[0] - curr = curr[2] - - def __reversed__(self): - end = self.__end - curr = end[1] - while curr is not end: - yield curr[0] - curr = curr[1] - - def popitem(self, last=True): - if not self: - raise KeyError('dictionary is empty') - if last: - key = next(reversed(self)) - else: - key = next(iter(self)) - value = self.pop(key) - return key, value - - def __reduce__(self): - items = [[k, self[k]] for k in self] - tmp = self.__map, self.__end - del self.__map, self.__end - inst_dict = vars(self).copy() - self.__map, self.__end = tmp - if inst_dict: - return (self.__class__, (items,), inst_dict) - return self.__class__, (items,) - - def keys(self): - return list(self) - - setdefault = DictMixin.setdefault - update = DictMixin.update - pop = DictMixin.pop - values = DictMixin.values - items = DictMixin.items - iterkeys = DictMixin.iterkeys - itervalues = DictMixin.itervalues - iteritems = DictMixin.iteritems - - def __repr__(self): - if not self: - return '%s()' % (self.__class__.__name__,) - return '%s(%r)' % (self.__class__.__name__, list(self.items())) - - def copy(self): - return self.__class__(self) - - @classmethod - def fromkeys(cls, iterable, value=None): - d = cls() - for key in iterable: - d[key] = value - return d - - def __eq__(self, other): - if isinstance(other, OrderedDict): - if len(self) != len(other): - return False - for p, q in zip(self.items(), other.items()): - if p != q: - return False - return True - return dict.__eq__(self, other) - - def __ne__(self, other): - return not self == other