Fix Python 3.13 cert tests, add --insecure, and bound DNS threads

.github/workflows/ci.yml

@@ -12,7 +12,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest]
-        python-version: ["3.11", "3.12"]
+        python-version: ["3.11", "3.12", "3.13"]
 
     steps:
       - name: Check out repository

README.md

@@ -52,13 +52,16 @@ python -m gambit example.com
 
 GAMBIT first runs Gather for the target, then runs the per-host checks (takeover, Analyze, Baseline, Inspect, and Map) on the apex host and the subdomains it discovers, and finally ranks everything with Triage. Discovered hosts are de-duplicated and capped (25 by default; see `--max-hosts`) so a scan stays small. Each host first gets a cheap connection pre-flight, so a dead subdomain costs a second instead of stacking every check's timeout; hosts are then checked one at a time with a short pause between them. Web checks try HTTPS first and fall back to plain HTTP, so HTTP-only sites are still checked.
 
+Certificate verification stays on unless you pass `--insecure`. A certificate problem (self-signed, untrusted CA, hostname mismatch) is reported and is **not** retried over plain HTTP, so a scan cannot silently downgrade. `--insecure` accepts those certificates and keeps speaking HTTPS. GAMBIT prints a warning and records the choice in the report. Use it for lab targets and appliances that ship a private certificate, and leave it off when you want the client's trust store to apply.
+
 | Flag | Description |
 |---|---|
 | `--format text\|md\|json` | Report format: human-readable text (default), Markdown (`md`), or JSON (`json`). |
 | `--json` | Print the report as JSON. Shortcut for `--format json`. |
 | `-o FILE`, `--output FILE` | Write the report to `FILE` instead of standard output. |
 | `--max-hosts N` | Maximum number of hosts to check, including the target itself (default: 25). |
 | `--yes` | Skip the interactive authorization confirmation (see [Responsible use](#responsible-use)). Use only where you already have permission. |
+| `--insecure` | Disable TLS certificate verification for HTTPS requests and the TLS handshake. Self-signed certificates and hostname mismatches are accepted. Off by default. Does not fall back to plain HTTP. The report notes that verification was disabled. |
 
 For example, to save a Markdown report for at most 10 hosts without the confirmation prompt:
 
@@ -146,7 +149,7 @@ Adding a check means adding one file. The Triage stage sorts every finding from
 - [x] **Map:** common-port check
 - [x] **Triage:** severity ranking and Markdown/JSON export
 - [x] Tests for every module
-- [x] CI (pytest on Ubuntu and Windows, Python 3.11 and 3.12)
+- [x] CI (pytest on Ubuntu and Windows, Python 3.11, 3.12, and 3.13)
 - [ ] First tagged release
 
 ## Prior art

gambit/__main__.py

@@ -11,6 +11,8 @@
 import sys
 from pathlib import Path
 
+from gambit.http import set_tls_verification
+from gambit.models import Finding, Severity
 from gambit.modules import (
     baseline_headers,
     cors,
@@ -22,6 +24,7 @@
     takeover,
     tls,
 )
+from gambit.net import clear_resolution_cache
 from gambit.pipeline import (
     DEFAULT_MAX_HOSTS,
     discover_hosts,
@@ -31,6 +34,12 @@
 )
 from gambit.triage import to_json, to_markdown, to_text
 
+INSECURE_NOTE = (
+    "NOTE: TLS certificate verification was disabled (--insecure). "
+    "HTTPS requests accepted untrusted certificates and were not retried "
+    "over plain HTTP."
+)
+
 # Run once on the apex target, in this order.
 APEX_MODULES = [gather]
 
@@ -77,16 +86,47 @@ def _positive_int(value: str) -> int:
     return number
 
 
-def format_report(findings, report_format: str) -> str:
-    """Render ``findings`` as ``text``, ``md``, or ``json`` (ranked)."""
+def verification_disabled_finding(host: str) -> Finding:
+    """INFO finding recorded when the operator passed ``--insecure``."""
+    return Finding(
+        severity=Severity.INFO,
+        host=host,
+        title="TLS certificate verification disabled",
+        evidence=(
+            "--insecure: HTTPS fetches and the TLS handshake did not verify "
+            "certificates. A certificate failure was not retried over plain HTTP."
+        ),
+        why=(
+            "Self-signed and untrusted certificates are common on pentest "
+            "targets. With verification off, those sites can still be checked, "
+            "but a network attacker could present a different certificate."
+        ),
+        next_step="Re-run without --insecure when the target uses a certificate you trust.",
+    )
+
+
+def format_report(findings, report_format: str, *, insecure: bool = False) -> str:
+    """Render ``findings`` as ``text``, ``md``, or ``json`` (ranked).
+
+    ``insecure=True`` puts a one-line note at the top of text and Markdown
+    reports. JSON keeps a pure list; the matching INFO finding carries
+    the same fact there.
+    """
     try:
         formatter = _FORMATTERS[report_format]
     except KeyError:
         raise ValueError(f"unknown report format: {report_format!r}") from None
-    return formatter(findings)
+    body = formatter(findings)
+    if not insecure or report_format == "json":
+        return body
+    if report_format == "md":
+        return f"> {INSECURE_NOTE}\n\n{body}"
+    return f"{INSECURE_NOTE}\n\n{body}"
 
 
-def collect_findings(target: str, max_hosts: int = DEFAULT_MAX_HOSTS) -> list:
+def collect_findings(
+    target: str, max_hosts: int = DEFAULT_MAX_HOSTS, *, insecure: bool = False
+) -> list:
     """Run the full pipeline for ``target`` and return all findings.
 
     Gather runs once on the apex; host discovery uses Gather's crt.sh
@@ -95,8 +135,16 @@ def collect_findings(target: str, max_hosts: int = DEFAULT_MAX_HOSTS) -> list:
     then run against the discovered (deduped, capped) hosts that pass a
     cheap liveness pre-flight. Per-module failures are isolated by the
     pipeline helpers, so one failing check never aborts the scan.
+
+    ``insecure=True`` disables TLS certificate verification for the scan
+    (HTTPS is still used) and records that choice as an INFO finding.
+    DNS answers cached by an earlier scan are dropped first.
     """
+    set_tls_verification(not insecure)
+    clear_resolution_cache()
     findings = []
+    if insecure:
+        findings.append(verification_disabled_finding(target))
     findings.extend(run_for_host(target, APEX_MODULES))
 
     hosts = discover_hosts(
@@ -159,6 +207,16 @@ def build_parser() -> argparse.ArgumentParser:
         action="store_true",
         help="confirm you own the target or have written permission to test it",
     )
+    parser.add_argument(
+        "--insecure",
+        action="store_true",
+        help=(
+            "disable TLS certificate verification for HTTPS requests and the "
+            "TLS handshake (self-signed certificates and hostname mismatches "
+            "are accepted). Off by default. Does not fall back to plain HTTP. "
+            "The report notes that verification was disabled."
+        ),
+    )
     return parser
 
 
@@ -174,10 +232,15 @@ def main(argv=None) -> int:
             print("Aborting: authorization not confirmed.", file=sys.stderr)
             return 1
 
-    findings = collect_findings(args.target, max_hosts=args.max_hosts)
+    if args.insecure:
+        print(INSECURE_NOTE, file=sys.stderr)
+
+    findings = collect_findings(
+        args.target, max_hosts=args.max_hosts, insecure=args.insecure
+    )
 
     report_format = "json" if args.json else args.format
-    report = format_report(findings, report_format)
+    report = format_report(findings, report_format, insecure=args.insecure)
 
     if args.output:
         try:

gambit/http.py

@@ -37,9 +37,14 @@
 proxies (that would undo the pin) but still honours
 ``REQUESTS_CA_BUNDLE`` / ``CURL_CA_BUNDLE`` when the caller did not pass
 ``verify=False`` or an explicit bundle path. A TLS *verification*
-failure is not retried over plain HTTP.
-A lookup that outruns its deadline leaves a daemon thread blocked in
-``getaddrinfo`` until the libc call returns; the scan does not wait.
+failure is not retried over plain HTTP. :func:`set_tls_verification`
+can turn verification off for a scan (the ``--insecure`` flag). That
+still speaks HTTPS; it does not downgrade a certificate failure to
+plain HTTP.
+
+DNS lookups are bounded, cached for the scan, and capped at a few
+in-flight resolver threads. One that outruns its deadline does not
+block the scan; see :mod:`gambit.net`.
 
 Response bodies are streamed and capped at :data:`MAX_BODY` bytes unless
 the caller is reading the stream itself.
@@ -49,15 +54,17 @@
 import os
 import socket
 import ssl
+import warnings
 from typing import NamedTuple
 from urllib.parse import urljoin
 
 import requests
 import urllib3.connection
 import urllib3.exceptions
+from urllib3.exceptions import InsecureRequestWarning
 from urllib3.util.url import parse_url
 
-from gambit.net import is_ip_address, is_public_ip, resolve_ips
+from gambit.net import is_ip_address, is_public_ip, resolution_saturated, resolve_ips
 
 # Identity of the real requests entry point. Tests replace
 # ``gambit.http.requests.request``; when they have, pinning is skipped
@@ -78,6 +85,43 @@
 _SCHEME_BY_HOST: dict[str, str] = {}
 # host -> non-default TCP port discovered by the liveness pre-flight
 _PORT_BY_HOST: dict[str, int] = {}
+# Safer default: verify certificates unless the operator opts out.
+_tls_verification_enabled = True
+# The warnings filter installed while verification is off, if any.
+_active_warning_filter: tuple | None = None
+
+
+def tls_verification_enabled() -> bool:
+    """True when HTTPS fetches still check certificates."""
+    return _tls_verification_enabled
+
+
+def set_tls_verification(enabled: bool) -> None:
+    """Turn TLS certificate verification on or off for later fetches.
+
+    Off is the ``--insecure`` opt-in. Hostname and chain checks are
+    skipped (``verify=False``) when a caller did not pass ``verify``
+    itself. A certificate failure is still not retried over plain HTTP.
+    """
+    global _tls_verification_enabled, _active_warning_filter
+    enabled = bool(enabled)
+    _tls_verification_enabled = enabled
+    # requests warns once per unverified call. The CLI surfaces one note
+    # instead. Installing the filter must be idempotent: simplefilter
+    # inserts a new entry on every call.
+    if enabled:
+        if _active_warning_filter is not None:
+            try:
+                warnings.filters.remove(_active_warning_filter)
+            except ValueError:
+                pass
+            _active_warning_filter = None
+        return
+    if _active_warning_filter is not None:
+        return
+    item = ("ignore", None, InsecureRequestWarning, None, 0)
+    warnings.filters.insert(0, item)
+    _active_warning_filter = item
 
 
 def request(
@@ -157,6 +201,11 @@ def request(
                 if pending_ips is None:
                     ips = resolve_ips(_dns_name(target.host), _timeout_seconds(timeout))
                     if not ips:
+                        if resolution_saturated(target.host):
+                            raise requests.ConnectionError(
+                                "DNS resolver saturated; could not resolve "
+                                f"{target.host}"
+                            )
                         raise requests.ConnectionError(
                             f"could not resolve {target.host}"
                         )
@@ -179,7 +228,7 @@ def request(
                     current,
                     headers=merged,
                     timeout=timeout,
-                    **kwargs,
+                    **_apply_tls_verification(kwargs),
                 )
                 send_url = current
         except requests.RequestException:
@@ -318,7 +367,10 @@ def _request_for_host(method: str, target: str, path: str = "", **kwargs):
     as-is. A TLS certificate verification failure is not a dead port:
     it is re-raised, and plain HTTP is not tried. That keeps an internal
     CA problem (or a hostname mismatch) from being silently downgraded.
-    Raises the last exception if every scheme fails.
+    With verification disabled (:func:`set_tls_verification`), those
+    certificates are accepted on HTTPS instead, and plain HTTP is still
+    not used as a substitute. Raises the last exception if every scheme
+    fails.
     """
     if path and not path.startswith("/"):
         path = "/" + path
@@ -503,15 +555,30 @@ def _is_tls_verify_failure(exc: BaseException) -> bool:
     return False
 
 
+def _apply_tls_verification(kwargs: dict) -> dict:
+    """Copy ``kwargs`` and apply the process verification policy.
+
+    An explicit ``verify`` argument always wins, including ``verify=False``
+    and a bundle path. When the caller left it unset and verification is
+    disabled, certificate checks are turned off.
+    """
+    copied = dict(kwargs)
+    if "verify" not in copied and not _tls_verification_enabled:
+        copied["verify"] = False
+    return copied
+
+
 def _with_env_ca_bundle(kwargs: dict) -> dict:
     """Copy ``kwargs`` and apply ``REQUESTS_CA_BUNDLE`` / ``CURL_CA_BUNDLE``.
 
     The pinned session keeps ``trust_env`` false so an environment proxy
     cannot replace the dialed address. requests reads the CA bundle only
     when ``trust_env`` is true, so copy it into ``verify`` here.
-    ``verify=False`` and an explicit bundle path are left alone.
+    ``verify=False`` and an explicit bundle path are left alone. With
+    verification disabled and no explicit ``verify``, the result is
+    ``verify=False`` and the environment bundle is not consulted.
     """
-    copied = dict(kwargs)
+    copied = _apply_tls_verification(kwargs)
     copied.pop("proxies", None)
     if "verify" not in copied:
         verify = True
@@ -867,5 +934,7 @@ def _buffer_limited(response, limit: int) -> None:
     "request",
     "set_preferred_port",
     "set_preferred_scheme",
+    "set_tls_verification",
+    "tls_verification_enabled",
     "url_authority",
 ]

gambit/modules/gather.py

@@ -239,6 +239,9 @@ def _fetch_crtsh_subdomains(host: str) -> list[str]:
             _CRT_SH_URL.format(host=host),
             timeout=TIMEOUT,
             allow_redirects=False,
+            # crt.sh is a third party. --insecure must not turn off
+            # verification for this call.
+            verify=True,
         )
         response.raise_for_status()
         names = parse_crtsh(response.json())

gambit/modules/ports.py

@@ -114,8 +114,8 @@ def run(target: str) -> list[Finding]:
     outside 1–65535 is not probed: the platform resolver would otherwise
     truncate it into a different port.
     """
-    from gambit.net import connect, resolve_ips
-    from gambit.pipeline import split_host_port
+    from gambit.net import connect, resolution_saturated, resolve_ips
+    from gambit.pipeline import resolver_saturated_finding, split_host_port
 
     hostname, explicit_port = split_host_port(target)
     host = hostname or target
@@ -124,6 +124,8 @@ def run(target: str) -> list[Finding]:
         ports.append(explicit_port)
     ips = resolve_ips(host, TIMEOUT)
     if not ips:
+        if resolution_saturated(host):
+            return [resolver_saturated_finding(host)]
         return []
     findings = []
     for port in ports:

gambit/modules/tls.py

@@ -3,10 +3,12 @@
 Standard library only (``ssl`` + ``socket``). The pure :func:`evaluate`
 function does the date/version maths on a certificate dict like the one
 ``ssl.SSLSocket.getpeercert()`` returns, so tests never need a network.
-``run`` performs a single verified handshake; a certificate that fails
-verification (hostname mismatch, self-signed, ...) is reported as INFO
-evidence rather than a vulnerability claim, and its details are still
-collected over an unverified handshake for the expiry/version checks.
+``run`` performs a verified handshake first so a failure can be
+classified (hostname mismatch, self-signed, or an untrusted chain) and
+reported as INFO rather than a vulnerability claim. Certificate details
+are then read over an unverified handshake for the expiry and version
+checks. ``--insecure`` skips verification on content fetches only; this
+module still classifies the certificate the same way.
 """
 
 import os
@@ -378,7 +380,15 @@ def _unverified_context() -> ssl.SSLContext:
 
 
 def run(target: str) -> list[Finding]:
-    """Check the target's TLS certificate and protocol over one handshake."""
+    """Check the target's TLS certificate and protocol.
+
+    The first handshake verifies, including when ``--insecure`` disabled
+    verification for content fetches. That failure is what classifies a
+    hostname mismatch, a self-signed certificate, or an untrusted chain.
+    A second, unverified handshake then supplies expiry and protocol
+    details. The scan-level note still records that content fetches did
+    not verify.
+    """
     hostname, port = _split_host_port(target)
     try:
         cert, protocol = _handshake(hostname, port, ssl.create_default_context())