apply cookies from redirect hops to the hops that follow

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## Unreleased
+
+- Cookies set on a redirect hop are applied to the hops that follow it, the way a browser does. The jar is request-scoped, so nothing carries between requests
+- `redirect_cookies=False` (or `--no-redirect-cookies`) reverts to the previous behavior
+
 ## 0.9.0
 
 - `no_proxy` support — bypass the proxy for specific hosts, domains, IPs, or CIDRs

Cargo.toml

@@ -4,6 +4,8 @@ version = "0.9.0"
 edition = "2024"
 description = "Offensive-first HTTP library with Python bindings"
 license = "GPL-3.0"
+# CI-only pin. Shipping it would override the toolchain of anyone building from the sdist.
+exclude = ["rust-toolchain.toml"]
 
 [lib]
 # When building as a Python module, we need a cdylib (shared library).

README.md

@@ -109,6 +109,7 @@ Output is JSON (one object per response), including status, headers, redirect ch
 | `--rate-limit` | Requests per second (batch mode) | unlimited |
 | `-L, --follow-redirects` | Follow redirects | off |
 | `--max-redirects` | Max redirect hops | `10` |
+| `--no-redirect-cookies` | Don't apply cookies a redirect sets to later hops | off |
 | `-t, --timeout` | Request timeout (seconds) | `10` |
 | `--max-body-size` | Max response body (bytes) | 10 MB |
 | `--verify` | Enable TLS cert validation | off |
@@ -229,6 +230,7 @@ All parameters except `url` are optional:
 | `timeout` | `int` | Request timeout in seconds |
 | `follow_redirects` | `bool` | Follow redirects |
 | `max_redirects` | `int` | Max redirect hops |
+| `redirect_cookies` | `bool` | Apply cookies a redirect sets to later hops (default `True`) |
 | `verify_certs` | `bool` | Enable TLS cert validation (default `False`) |
 | `proxy` | `str` | HTTP/SOCKS proxy URL |
 | `no_proxy` | `list[str]` | Hosts that bypass the proxy |
@@ -313,6 +315,28 @@ response = await client.request(
 )
 ```
 
+### Cookies across redirects
+
+When `follow_redirects` is on, a cookie set by one hop is sent on the hops that follow it, the same way a browser does. That's what makes a login or bot-check page work: it hands you a cookie along with the redirect, and the cookie has to be on the next request to count for anything. Without this you'd loop or land back on the same page.
+
+The jar is **request-scoped**. It's created when the request starts and dropped when it returns, so nothing carries into the next request and no two requests can see each other's cookies. A batch of 500 URLs runs 500 independent jars, which keeps every result reproducible on its own.
+
+Which cookie goes to which hop follows the usual rules (RFC 6265): a cookie with no `Domain` goes back only to the exact host that set it, a `Domain` that doesn't cover the host that sent it is thrown out, `Path` has to match, and `Secure` cookies never go over plain HTTP. So a redirect can't be used to walk a session cookie onto some other host.
+
+Pass `redirect_cookies=False` (or `--no-redirect-cookies` on the CLI) to turn it off and send only your own headers on every hop.
+
+```python
+# On by default.
+r = await client.request("https://example.com/login", method="POST",
+                         body="user=x&pass=y", follow_redirects=True)
+
+# Off: every hop gets only the headers you supplied.
+r = await client.request("https://example.com/login", follow_redirects=True,
+                         redirect_cookies=False)
+```
+
+Cookies you supply yourself via a `Cookie` header are merged with the chain's into a single header, yours first.
+
 ### Proxy exclusions (`no_proxy`)
 
 `proxy` routes a request through an HTTP or SOCKS5 proxy; `no_proxy` is a per-request list of hosts that bypass it and connect directly — the `NO_PROXY` equivalent. It's accepted by `request()`, `download()`, `raw_connect()`, and `BatchConfig`, and as the repeatable `--no-proxy` CLI flag.

rust-toolchain.toml

@@ -0,0 +1,4 @@
+# Pinned so new clippy lints don't break dependabot PRs. Bump deliberately, fixing new lints in the same commit.
+[toolchain]
+channel = "1.97.1"
+components = ["clippy", "rustfmt"]

src/client/hyper.rs

@@ -757,11 +757,12 @@ async fn dispatch_request(
     uri: &http::Uri,
     config: &RequestConfig,
     log: &DebugLog,
+    redirect_cookies: Option<&str>,
 ) -> Result<SingleResponse, ClientError> {
     // The pooled high-level client populates Host / :authority from the URI
     // itself, so we don't add a Host header here. Adding it would cause
     // duplicate :authority + host in the HTTP/2 HPACK block.
-    let request = build_request(uri, config, false, false)?;
+    let request = build_request(uri, config, false, false, redirect_cookies)?;
     let v = config.verbosity;
 
     debug_record(log, v, 1, "   Request headers:");
@@ -1076,7 +1077,7 @@ async fn dispatch_direct(
     // the way the pooled high-level client does, so we ask build_request to do
     // it manually.
     let use_origin_form = config.request_target.is_none();
-    let request = build_request(&request_uri, config, use_origin_form, true)?;
+    let request = build_request(&request_uri, config, use_origin_form, true, None)?;
 
     debug_record(log, v, 1, "   Request headers:");
     for (name, value) in request.headers() {
@@ -1120,6 +1121,7 @@ async fn dispatch_forward_proxy(
     target_uri: &http::Uri,
     config: &RequestConfig,
     log: &DebugLog,
+    redirect_cookies: Option<&str>,
 ) -> Result<SingleResponse, ClientError> {
     let proxy_uri: http::Uri = proxy_url.parse().map_err(|e: http::uri::InvalidUri| {
         ClientError::invalid_url(format!("invalid proxy URL: {}", e))
@@ -1157,7 +1159,7 @@ async fn dispatch_forward_proxy(
 
     // Build request with absolute-form URI (SendRequest does NOT normalize it).
     // The low-level http1 sender doesn't auto-populate Host, so add it manually.
-    let request = build_request(target_uri, config, false, true)?;
+    let request = build_request(target_uri, config, false, true, redirect_cookies)?;
     let v = config.verbosity;
 
     debug_record(log, v, 1, "   Request headers:");
@@ -1184,6 +1186,10 @@ fn build_request(
     config: &RequestConfig,
     origin_form: bool,
     manual_host_header: bool,
+    // Cookies picked up from earlier hops of this redirect chain, already
+    // filtered down to the ones that apply to `uri`. Merged into the
+    // caller's own `Cookie` header when there is one, so we never send two.
+    redirect_cookies: Option<&str>,
 ) -> Result<hyper::Request<FullBody>, ClientError> {
     // For direct connections (dispatch_direct), use origin-form (path + query only)
     // in the request-line per RFC 7230 §5.3.1. For pooled/client connections,
@@ -1241,11 +1247,28 @@ fn build_request(
         builder = builder.header("Accept-Encoding", "gzip, deflate, br");
     }
 
+    // Emit the caller's headers, folding any redirect-chain cookies into the
+    // first `Cookie` header they supplied. If they supplied none, the
+    // chain's cookies go out as their own header after the caller's.
+    let mut merged_cookies = false;
     if let Some(ref custom_headers) = config.headers {
         for (name, value) in custom_headers {
+            if let Some(extra) = redirect_cookies
+                && !merged_cookies
+                && name.eq_ignore_ascii_case("cookie")
+            {
+                merged_cookies = true;
+                builder = builder.header(name.as_str(), format!("{}; {}", value, extra));
+                continue;
+            }
             builder = builder.header(name.as_str(), value.as_str());
         }
     }
+    if let Some(extra) = redirect_cookies
+        && !merged_cookies
+    {
+        builder = builder.header("Cookie", extra);
+    }
 
     let body_bytes = config.body.clone().unwrap_or_default();
     builder
@@ -1512,6 +1535,16 @@ impl HyperClient {
         let mut redirect_chain: Vec<RedirectHop> = Vec::new();
         let mut hops = 0u32;
 
+        // Cookies picked up as we walk this chain. Created here and dropped
+        // when the request returns, so two concurrent requests can never see
+        // each other's cookies and a request's result depends only on its own
+        // inputs. `hop_cookies` is the `Cookie` header for the hop we're about
+        // to make, recomputed per hop because each one may be a different host.
+        let mut jar = config
+            .should_forward_redirect_cookies()
+            .then(crate::cookies::CookieJar::new);
+        let mut hop_cookies: Option<String> = None;
+
         loop {
             // Decide the connection mode for the *current* target host on every
             // hop, not just the first. A redirect can send the request to a
@@ -1541,9 +1574,16 @@ impl HyperClient {
             };
 
             let resp = if let Some(ref proxy_url) = proxy_url_for_fwd {
-                dispatch_forward_proxy(proxy_url, &uri, config, log).await?
+                dispatch_forward_proxy(proxy_url, &uri, config, log, hop_cookies.as_deref()).await?
             } else {
-                dispatch_request(&cached.as_ref().unwrap().inner, &uri, config, log).await?
+                dispatch_request(
+                    &cached.as_ref().unwrap().inner,
+                    &uri,
+                    config,
+                    log,
+                    hop_cookies.as_deref(),
+                )
+                .await?
             };
             let hop_ms = start.elapsed().as_millis();
             debug_record(log, v, 1, &format!("<- {} ({}ms)", resp.status, hop_ms));
@@ -1587,6 +1627,18 @@ impl HyperClient {
                     peer_ip: hop_peer_ip,
                 });
 
+                // Take this hop's `Set-Cookie` headers, then work out which of
+                // everything collected so far applies to where we're going.
+                // The domain / path / Secure rules are what stop a cookie from
+                // following a redirect onto a host it doesn't belong to.
+                if let Some(jar) = jar.as_mut() {
+                    jar.store(&resp.headers, &uri);
+                    hop_cookies = jar.header_for(&next_uri);
+                    if let Some(ref c) = hop_cookies {
+                        debug_record(log, v, 1, &format!("   Sending cookies: {}", c));
+                    }
+                }
+
                 uri = next_uri;
                 continue;
             }
@@ -1740,7 +1792,7 @@ mod tests {
     fn test_build_request_auto_host_from_uri() {
         let uri: http::Uri = "http://example.com:8080/path".parse().unwrap();
         let config = RequestConfig::new("http://example.com:8080/path".to_string());
-        let req = build_request(&uri, &config, true, true).unwrap();
+        let req = build_request(&uri, &config, true, true, None).unwrap();
         assert_eq!(req.headers().get("host").unwrap(), "example.com:8080");
     }
 
@@ -1752,7 +1804,7 @@ mod tests {
         // HTTP/2 HPACK block, which some origin servers reject.
         let uri: http::Uri = "http://example.com:8080/path".parse().unwrap();
         let config = RequestConfig::new("http://example.com:8080/path".to_string());
-        let req = build_request(&uri, &config, false, false).unwrap();
+        let req = build_request(&uri, &config, false, false, None).unwrap();
         assert!(req.headers().get("host").is_none());
     }
 
@@ -1761,7 +1813,7 @@ mod tests {
         let uri: http::Uri = "http://example.com:8080/path".parse().unwrap();
         let mut config = RequestConfig::new("http://example.com:8080/path".to_string());
         config.headers = Some(vec![("Host".to_string(), "custom.host".to_string())]);
-        let req = build_request(&uri, &config, true, true).unwrap();
+        let req = build_request(&uri, &config, true, true, None).unwrap();
         // Should only have the custom Host, not auto-derived
         let hosts: Vec<_> = req.headers().get_all("host").iter().collect();
         assert_eq!(hosts.len(), 1);
@@ -1776,7 +1828,7 @@ mod tests {
         let uri: http::Uri = "http://example.com:8080/path".parse().unwrap();
         let mut config = RequestConfig::new("http://example.com:8080/path".to_string());
         config.headers = Some(vec![("Host".to_string(), "custom.host".to_string())]);
-        let req = build_request(&uri, &config, false, false).unwrap();
+        let req = build_request(&uri, &config, false, false, None).unwrap();
         let hosts: Vec<_> = req.headers().get_all("host").iter().collect();
         assert_eq!(hosts.len(), 1);
         assert_eq!(hosts[0], "custom.host");
@@ -1790,7 +1842,7 @@ mod tests {
             ("Host".to_string(), "first.host".to_string()),
             ("Host".to_string(), "second.host".to_string()),
         ]);
-        let req = build_request(&uri, &config, true, true).unwrap();
+        let req = build_request(&uri, &config, true, true, None).unwrap();
         let hosts: Vec<_> = req.headers().get_all("host").iter().collect();
         assert_eq!(hosts.len(), 2);
         assert_eq!(hosts[0], "first.host");
@@ -1801,15 +1853,15 @@ mod tests {
     fn test_build_request_origin_form_strips_authority() {
         let uri: http::Uri = "http://example.com:8080/path?q=1".parse().unwrap();
         let config = RequestConfig::new("http://example.com:8080/path?q=1".to_string());
-        let req = build_request(&uri, &config, true, true).unwrap();
+        let req = build_request(&uri, &config, true, true, None).unwrap();
         assert_eq!(req.uri(), "/path?q=1");
     }
 
     #[test]
     fn test_build_request_absolute_form_preserves_uri() {
         let uri: http::Uri = "http://example.com:8080/path?q=1".parse().unwrap();
         let config = RequestConfig::new("http://example.com:8080/path?q=1".to_string());
-        let req = build_request(&uri, &config, false, false).unwrap();
+        let req = build_request(&uri, &config, false, false, None).unwrap();
         assert_eq!(req.uri().to_string(), "http://example.com:8080/path?q=1");
     }
 
@@ -1819,7 +1871,7 @@ mod tests {
         // Simulate: origin_form=false (as dispatch_direct does when request_target is Some)
         let uri: http::Uri = "http://evil.com/admin".parse().unwrap();
         let config = RequestConfig::new("http://example.com/".to_string());
-        let req = build_request(&uri, &config, false, true).unwrap();
+        let req = build_request(&uri, &config, false, true, None).unwrap();
         assert_eq!(req.uri().to_string(), "http://evil.com/admin");
     }
 }

src/config.rs

@@ -10,6 +10,11 @@ pub struct RequestConfig {
     pub max_body_size: Option<usize>,
     pub follow_redirects: Option<bool>,
     pub max_redirects: Option<u32>,
+    /// Apply cookies a redirect hop sets to the hops that follow it, within
+    /// this one request (default: true). The jar is request-scoped, so
+    /// nothing carries into the next request and results stay reproducible.
+    /// Set to `false` to send only the caller's own headers on every hop.
+    pub redirect_cookies: Option<bool>,
     pub verify_certs: Option<bool>,
     pub proxy: Option<String>,
     /// Hosts that bypass `proxy` and connect directly (NO_PROXY equivalent).
@@ -55,6 +60,7 @@ impl RequestConfig {
             max_body_size: None,
             follow_redirects: None,
             max_redirects: None,
+            redirect_cookies: None,
             verify_certs: None,
             proxy: None,
             no_proxy: Vec::new(),
@@ -92,6 +98,14 @@ impl RequestConfig {
         self.max_redirects.unwrap_or(10)
     }
 
+    /// Whether cookies set mid-chain are replayed on later hops. On by
+    /// default: it's what a browser does, and it's what lets a login or
+    /// bot-check page (the kind that sets a cookie and redirects you back
+    /// to where you started) actually resolve instead of looping.
+    pub fn should_forward_redirect_cookies(&self) -> bool {
+        self.redirect_cookies.unwrap_or(true)
+    }
+
     pub fn should_verify_certs(&self) -> bool {
         self.verify_certs.unwrap_or(false)
     }