diff --git a/CHANGELOG.md b/CHANGELOG.md index 9038823..508605f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## Unreleased + +- Cookies set on a redirect hop are applied to the hops that follow it, the way a browser does. The jar is request-scoped, so nothing carries between requests +- `redirect_cookies=False` (or `--no-redirect-cookies`) reverts to the previous behavior + ## 0.9.0 - `no_proxy` support — bypass the proxy for specific hosts, domains, IPs, or CIDRs diff --git a/Cargo.toml b/Cargo.toml index d8a7259..355c69b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,8 @@ version = "0.9.0" edition = "2024" description = "Offensive-first HTTP library with Python bindings" license = "GPL-3.0" +# CI-only pin. Shipping it would override the toolchain of anyone building from the sdist. +exclude = ["rust-toolchain.toml"] [lib] # When building as a Python module, we need a cdylib (shared library). diff --git a/README.md b/README.md index 4104f9c..3d36be0 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,7 @@ Output is JSON (one object per response), including status, headers, redirect ch | `--rate-limit` | Requests per second (batch mode) | unlimited | | `-L, --follow-redirects` | Follow redirects | off | | `--max-redirects` | Max redirect hops | `10` | +| `--no-redirect-cookies` | Don't apply cookies a redirect sets to later hops | off | | `-t, --timeout` | Request timeout (seconds) | `10` | | `--max-body-size` | Max response body (bytes) | 10 MB | | `--verify` | Enable TLS cert validation | off | @@ -229,6 +230,7 @@ All parameters except `url` are optional: | `timeout` | `int` | Request timeout in seconds | | `follow_redirects` | `bool` | Follow redirects | | `max_redirects` | `int` | Max redirect hops | +| `redirect_cookies` | `bool` | Apply cookies a redirect sets to later hops (default `True`) | | `verify_certs` | `bool` | Enable TLS cert validation (default `False`) | | `proxy` | `str` | HTTP/SOCKS proxy URL | | `no_proxy` | `list[str]` | Hosts that bypass the proxy | @@ -313,6 +315,28 @@ response = await client.request( ) ``` +### Cookies across redirects + +When `follow_redirects` is on, a cookie set by one hop is sent on the hops that follow it, the same way a browser does. That's what makes a login or bot-check page work: it hands you a cookie along with the redirect, and the cookie has to be on the next request to count for anything. Without this you'd loop or land back on the same page. + +The jar is **request-scoped**. It's created when the request starts and dropped when it returns, so nothing carries into the next request and no two requests can see each other's cookies. A batch of 500 URLs runs 500 independent jars, which keeps every result reproducible on its own. + +Which cookie goes to which hop follows the usual rules (RFC 6265): a cookie with no `Domain` goes back only to the exact host that set it, a `Domain` that doesn't cover the host that sent it is thrown out, `Path` has to match, and `Secure` cookies never go over plain HTTP. So a redirect can't be used to walk a session cookie onto some other host. + +Pass `redirect_cookies=False` (or `--no-redirect-cookies` on the CLI) to turn it off and send only your own headers on every hop. + +```python +# On by default. +r = await client.request("https://example.com/login", method="POST", + body="user=x&pass=y", follow_redirects=True) + +# Off: every hop gets only the headers you supplied. +r = await client.request("https://example.com/login", follow_redirects=True, + redirect_cookies=False) +``` + +Cookies you supply yourself via a `Cookie` header are merged with the chain's into a single header, yours first. + ### Proxy exclusions (`no_proxy`) `proxy` routes a request through an HTTP or SOCKS5 proxy; `no_proxy` is a per-request list of hosts that bypass it and connect directly — the `NO_PROXY` equivalent. It's accepted by `request()`, `download()`, `raw_connect()`, and `BatchConfig`, and as the repeatable `--no-proxy` CLI flag. diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..1199015 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,4 @@ +# Pinned so new clippy lints don't break dependabot PRs. Bump deliberately, fixing new lints in the same commit. +[toolchain] +channel = "1.97.1" +components = ["clippy", "rustfmt"] diff --git a/src/client/hyper.rs b/src/client/hyper.rs index 1845a30..b65bad5 100644 --- a/src/client/hyper.rs +++ b/src/client/hyper.rs @@ -757,11 +757,12 @@ async fn dispatch_request( uri: &http::Uri, config: &RequestConfig, log: &DebugLog, + redirect_cookies: Option<&str>, ) -> Result { // The pooled high-level client populates Host / :authority from the URI // itself, so we don't add a Host header here. Adding it would cause // duplicate :authority + host in the HTTP/2 HPACK block. - let request = build_request(uri, config, false, false)?; + let request = build_request(uri, config, false, false, redirect_cookies)?; let v = config.verbosity; debug_record(log, v, 1, " Request headers:"); @@ -1076,7 +1077,7 @@ async fn dispatch_direct( // the way the pooled high-level client does, so we ask build_request to do // it manually. let use_origin_form = config.request_target.is_none(); - let request = build_request(&request_uri, config, use_origin_form, true)?; + let request = build_request(&request_uri, config, use_origin_form, true, None)?; debug_record(log, v, 1, " Request headers:"); for (name, value) in request.headers() { @@ -1120,6 +1121,7 @@ async fn dispatch_forward_proxy( target_uri: &http::Uri, config: &RequestConfig, log: &DebugLog, + redirect_cookies: Option<&str>, ) -> Result { let proxy_uri: http::Uri = proxy_url.parse().map_err(|e: http::uri::InvalidUri| { ClientError::invalid_url(format!("invalid proxy URL: {}", e)) @@ -1157,7 +1159,7 @@ async fn dispatch_forward_proxy( // Build request with absolute-form URI (SendRequest does NOT normalize it). // The low-level http1 sender doesn't auto-populate Host, so add it manually. - let request = build_request(target_uri, config, false, true)?; + let request = build_request(target_uri, config, false, true, redirect_cookies)?; let v = config.verbosity; debug_record(log, v, 1, " Request headers:"); @@ -1184,6 +1186,10 @@ fn build_request( config: &RequestConfig, origin_form: bool, manual_host_header: bool, + // Cookies picked up from earlier hops of this redirect chain, already + // filtered down to the ones that apply to `uri`. Merged into the + // caller's own `Cookie` header when there is one, so we never send two. + redirect_cookies: Option<&str>, ) -> Result, ClientError> { // For direct connections (dispatch_direct), use origin-form (path + query only) // in the request-line per RFC 7230 §5.3.1. For pooled/client connections, @@ -1241,11 +1247,28 @@ fn build_request( builder = builder.header("Accept-Encoding", "gzip, deflate, br"); } + // Emit the caller's headers, folding any redirect-chain cookies into the + // first `Cookie` header they supplied. If they supplied none, the + // chain's cookies go out as their own header after the caller's. + let mut merged_cookies = false; if let Some(ref custom_headers) = config.headers { for (name, value) in custom_headers { + if let Some(extra) = redirect_cookies + && !merged_cookies + && name.eq_ignore_ascii_case("cookie") + { + merged_cookies = true; + builder = builder.header(name.as_str(), format!("{}; {}", value, extra)); + continue; + } builder = builder.header(name.as_str(), value.as_str()); } } + if let Some(extra) = redirect_cookies + && !merged_cookies + { + builder = builder.header("Cookie", extra); + } let body_bytes = config.body.clone().unwrap_or_default(); builder @@ -1512,6 +1535,16 @@ impl HyperClient { let mut redirect_chain: Vec = Vec::new(); let mut hops = 0u32; + // Cookies picked up as we walk this chain. Created here and dropped + // when the request returns, so two concurrent requests can never see + // each other's cookies and a request's result depends only on its own + // inputs. `hop_cookies` is the `Cookie` header for the hop we're about + // to make, recomputed per hop because each one may be a different host. + let mut jar = config + .should_forward_redirect_cookies() + .then(crate::cookies::CookieJar::new); + let mut hop_cookies: Option = None; + loop { // Decide the connection mode for the *current* target host on every // hop, not just the first. A redirect can send the request to a @@ -1541,9 +1574,16 @@ impl HyperClient { }; let resp = if let Some(ref proxy_url) = proxy_url_for_fwd { - dispatch_forward_proxy(proxy_url, &uri, config, log).await? + dispatch_forward_proxy(proxy_url, &uri, config, log, hop_cookies.as_deref()).await? } else { - dispatch_request(&cached.as_ref().unwrap().inner, &uri, config, log).await? + dispatch_request( + &cached.as_ref().unwrap().inner, + &uri, + config, + log, + hop_cookies.as_deref(), + ) + .await? }; let hop_ms = start.elapsed().as_millis(); debug_record(log, v, 1, &format!("<- {} ({}ms)", resp.status, hop_ms)); @@ -1587,6 +1627,18 @@ impl HyperClient { peer_ip: hop_peer_ip, }); + // Take this hop's `Set-Cookie` headers, then work out which of + // everything collected so far applies to where we're going. + // The domain / path / Secure rules are what stop a cookie from + // following a redirect onto a host it doesn't belong to. + if let Some(jar) = jar.as_mut() { + jar.store(&resp.headers, &uri); + hop_cookies = jar.header_for(&next_uri); + if let Some(ref c) = hop_cookies { + debug_record(log, v, 1, &format!(" Sending cookies: {}", c)); + } + } + uri = next_uri; continue; } @@ -1740,7 +1792,7 @@ mod tests { fn test_build_request_auto_host_from_uri() { let uri: http::Uri = "http://example.com:8080/path".parse().unwrap(); let config = RequestConfig::new("http://example.com:8080/path".to_string()); - let req = build_request(&uri, &config, true, true).unwrap(); + let req = build_request(&uri, &config, true, true, None).unwrap(); assert_eq!(req.headers().get("host").unwrap(), "example.com:8080"); } @@ -1752,7 +1804,7 @@ mod tests { // HTTP/2 HPACK block, which some origin servers reject. let uri: http::Uri = "http://example.com:8080/path".parse().unwrap(); let config = RequestConfig::new("http://example.com:8080/path".to_string()); - let req = build_request(&uri, &config, false, false).unwrap(); + let req = build_request(&uri, &config, false, false, None).unwrap(); assert!(req.headers().get("host").is_none()); } @@ -1761,7 +1813,7 @@ mod tests { let uri: http::Uri = "http://example.com:8080/path".parse().unwrap(); let mut config = RequestConfig::new("http://example.com:8080/path".to_string()); config.headers = Some(vec![("Host".to_string(), "custom.host".to_string())]); - let req = build_request(&uri, &config, true, true).unwrap(); + let req = build_request(&uri, &config, true, true, None).unwrap(); // Should only have the custom Host, not auto-derived let hosts: Vec<_> = req.headers().get_all("host").iter().collect(); assert_eq!(hosts.len(), 1); @@ -1776,7 +1828,7 @@ mod tests { let uri: http::Uri = "http://example.com:8080/path".parse().unwrap(); let mut config = RequestConfig::new("http://example.com:8080/path".to_string()); config.headers = Some(vec![("Host".to_string(), "custom.host".to_string())]); - let req = build_request(&uri, &config, false, false).unwrap(); + let req = build_request(&uri, &config, false, false, None).unwrap(); let hosts: Vec<_> = req.headers().get_all("host").iter().collect(); assert_eq!(hosts.len(), 1); assert_eq!(hosts[0], "custom.host"); @@ -1790,7 +1842,7 @@ mod tests { ("Host".to_string(), "first.host".to_string()), ("Host".to_string(), "second.host".to_string()), ]); - let req = build_request(&uri, &config, true, true).unwrap(); + let req = build_request(&uri, &config, true, true, None).unwrap(); let hosts: Vec<_> = req.headers().get_all("host").iter().collect(); assert_eq!(hosts.len(), 2); assert_eq!(hosts[0], "first.host"); @@ -1801,7 +1853,7 @@ mod tests { fn test_build_request_origin_form_strips_authority() { let uri: http::Uri = "http://example.com:8080/path?q=1".parse().unwrap(); let config = RequestConfig::new("http://example.com:8080/path?q=1".to_string()); - let req = build_request(&uri, &config, true, true).unwrap(); + let req = build_request(&uri, &config, true, true, None).unwrap(); assert_eq!(req.uri(), "/path?q=1"); } @@ -1809,7 +1861,7 @@ mod tests { fn test_build_request_absolute_form_preserves_uri() { let uri: http::Uri = "http://example.com:8080/path?q=1".parse().unwrap(); let config = RequestConfig::new("http://example.com:8080/path?q=1".to_string()); - let req = build_request(&uri, &config, false, false).unwrap(); + let req = build_request(&uri, &config, false, false, None).unwrap(); assert_eq!(req.uri().to_string(), "http://example.com:8080/path?q=1"); } @@ -1819,7 +1871,7 @@ mod tests { // Simulate: origin_form=false (as dispatch_direct does when request_target is Some) let uri: http::Uri = "http://evil.com/admin".parse().unwrap(); let config = RequestConfig::new("http://example.com/".to_string()); - let req = build_request(&uri, &config, false, true).unwrap(); + let req = build_request(&uri, &config, false, true, None).unwrap(); assert_eq!(req.uri().to_string(), "http://evil.com/admin"); } } diff --git a/src/config.rs b/src/config.rs index 1c381d5..25c05ba 100644 --- a/src/config.rs +++ b/src/config.rs @@ -10,6 +10,11 @@ pub struct RequestConfig { pub max_body_size: Option, pub follow_redirects: Option, pub max_redirects: Option, + /// Apply cookies a redirect hop sets to the hops that follow it, within + /// this one request (default: true). The jar is request-scoped, so + /// nothing carries into the next request and results stay reproducible. + /// Set to `false` to send only the caller's own headers on every hop. + pub redirect_cookies: Option, pub verify_certs: Option, pub proxy: Option, /// Hosts that bypass `proxy` and connect directly (NO_PROXY equivalent). @@ -55,6 +60,7 @@ impl RequestConfig { max_body_size: None, follow_redirects: None, max_redirects: None, + redirect_cookies: None, verify_certs: None, proxy: None, no_proxy: Vec::new(), @@ -92,6 +98,14 @@ impl RequestConfig { self.max_redirects.unwrap_or(10) } + /// Whether cookies set mid-chain are replayed on later hops. On by + /// default: it's what a browser does, and it's what lets a login or + /// bot-check page (the kind that sets a cookie and redirects you back + /// to where you started) actually resolve instead of looping. + pub fn should_forward_redirect_cookies(&self) -> bool { + self.redirect_cookies.unwrap_or(true) + } + pub fn should_verify_certs(&self) -> bool { self.verify_certs.unwrap_or(false) } diff --git a/src/cookies.rs b/src/cookies.rs new file mode 100644 index 0000000..e496bb3 --- /dev/null +++ b/src/cookies.rs @@ -0,0 +1,534 @@ +//! Request-scoped cookie handling for redirect chains. +//! +//! A [`CookieJar`] lives for exactly one request: cookies set by one hop +//! are offered to later hops in the same redirect chain, then dropped when +//! the request returns. Nothing survives into the next request, so a +//! result depends only on that request's own inputs. A client-wide jar +//! would destroy that, since concurrent requests sharing one would race +//! to write it. +//! +//! Within a chain the behavior matches a browser: `Set-Cookie` on a `302` +//! is applied to the hop that follows it. That's how nearly every login +//! flow works: post credentials, get back a session cookie plus a +//! redirect, and the cookie has to be on the next request for it to mean +//! anything. Bot-check pages work the same way. Which cookie goes to which +//! hop follows the RFC 6265 domain, path, and `Secure` rules, so a cookie +//! is never sent to a host it doesn't belong to. + +use std::time::{SystemTime, UNIX_EPOCH}; + +/// One stored cookie, normalized per RFC 6265 §5.3. +#[derive(Debug, Clone, PartialEq, Eq)] +struct Cookie { + name: String, + value: String, + /// Canonicalized (lowercase, no leading dot) domain this cookie is + /// scoped to. + domain: String, + /// True when the response carried no `Domain` attribute, meaning the + /// cookie goes back only to the exact host that set it, never to a + /// subdomain. + host_only: bool, + path: String, + secure: bool, +} + +/// Cookies accumulated over one request's redirect chain. +#[derive(Debug, Default, Clone)] +pub struct CookieJar { + cookies: Vec, +} + +impl CookieJar { + pub fn new() -> Self { + CookieJar { + cookies: Vec::new(), + } + } + + pub fn is_empty(&self) -> bool { + self.cookies.is_empty() + } + + /// Take every `Set-Cookie` out of a response that came back from + /// `uri`. Cookies whose `Domain` doesn't cover `uri`'s host are + /// dropped, and an expired cookie (`Max-Age=0`, or `Expires` in the + /// past, the usual "log out" / "clear this" signal) deletes any + /// matching cookie already held instead of being stored. + pub fn store(&mut self, headers: &[(String, String)], uri: &http::Uri) { + let Some(host) = uri.host() else { return }; + let host = canonical_host(host); + let request_path = uri.path(); + + for (name, value) in headers { + if !name.eq_ignore_ascii_case("set-cookie") { + continue; + } + let Some((cookie, expired)) = parse_set_cookie(value, &host, request_path) else { + continue; + }; + // §5.3 step 11: a new cookie replaces one with the same + // name/domain/path rather than adding a duplicate. + self.cookies.retain(|c| { + !(c.name == cookie.name && c.domain == cookie.domain && c.path == cookie.path) + }); + if !expired { + self.cookies.push(cookie); + } + } + } + + /// The `Cookie` header value to send to `uri`, or `None` when nothing + /// in the jar applies to it. + pub fn header_for(&self, uri: &http::Uri) -> Option { + let host = canonical_host(uri.host()?); + let path = uri.path(); + let secure_transport = uri.scheme_str() == Some("https"); + + let mut matched: Vec<&Cookie> = self + .cookies + .iter() + .filter(|c| { + if c.secure && !secure_transport { + return false; + } + if c.host_only { + if host != c.domain { + return false; + } + } else if !domain_matches(&host, &c.domain) { + return false; + } + path_matches(path, &c.path) + }) + .collect(); + + if matched.is_empty() { + return None; + } + + // §5.4: longer paths first. `sort_by_key` is stable, so cookies + // with equal path length keep the order they were set in, which is the + // spec's creation-time tiebreak. + matched.sort_by_key(|c| std::cmp::Reverse(c.path.len())); + + Some( + matched + .iter() + .map(|c| format!("{}={}", c.name, c.value)) + .collect::>() + .join("; "), + ) + } +} + +/// Lowercase a host and strip a single trailing dot so `Example.COM.` and +/// `example.com` compare equal. +fn canonical_host(host: &str) -> String { + host.trim_end_matches('.').to_ascii_lowercase() +} + +/// RFC 6265 §5.1.3. True when `host` is `domain` or a subdomain of it. An +/// IP literal only ever matches itself, so a `Domain` attribute can never +/// widen a cookie set by an IP address. +fn domain_matches(host: &str, domain: &str) -> bool { + if host == domain { + return true; + } + if host.parse::().is_ok() { + return false; + } + host.len() > domain.len() + && host.ends_with(domain) + && host.as_bytes()[host.len() - domain.len() - 1] == b'.' +} + +/// RFC 6265 §5.1.4. `cookie_path` covers `request_path` when it is equal, +/// or is a prefix ending at a `/` boundary. +fn path_matches(request_path: &str, cookie_path: &str) -> bool { + if request_path == cookie_path { + return true; + } + if !request_path.starts_with(cookie_path) { + return false; + } + cookie_path.ends_with('/') || request_path.as_bytes()[cookie_path.len()] == b'/' +} + +/// RFC 6265 §5.1.4 default-path: everything up to the last `/` of the +/// request path, or `/` when there isn't one to cut at. +fn default_path(request_path: &str) -> String { + if !request_path.starts_with('/') { + return "/".to_string(); + } + match request_path.rfind('/') { + Some(0) | None => "/".to_string(), + Some(i) => request_path[..i].to_string(), + } +} + +/// Parse one `Set-Cookie` value in the context of the request it answered. +/// +/// Returns the normalized cookie plus whether it is already expired (so the +/// caller deletes rather than stores it), or `None` when the cookie is +/// malformed or its `Domain` doesn't cover `request_host` (§5.3 step 6), +/// which is what stops a redirect target from setting cookies for +/// unrelated hosts. +fn parse_set_cookie(value: &str, request_host: &str, request_path: &str) -> Option<(Cookie, bool)> { + let mut parts = value.split(';'); + let pair = parts.next()?.trim(); + let (name, val) = pair.split_once('=')?; + let name = name.trim(); + if name.is_empty() { + return None; + } + + let mut domain: Option = None; + let mut path: Option = None; + let mut secure = false; + let mut max_age: Option = None; + let mut expires: Option = None; + + for attr in parts { + let attr = attr.trim(); + let (key, aval) = match attr.split_once('=') { + Some((k, v)) => (k.trim(), v.trim()), + None => (attr, ""), + }; + if key.eq_ignore_ascii_case("domain") { + let d = canonical_host(aval.trim_start_matches('.')); + if !d.is_empty() { + domain = Some(d); + } + } else if key.eq_ignore_ascii_case("path") { + if aval.starts_with('/') { + path = Some(aval.to_string()); + } + } else if key.eq_ignore_ascii_case("secure") { + secure = true; + } else if key.eq_ignore_ascii_case("max-age") { + max_age = aval.parse::().ok(); + } else if key.eq_ignore_ascii_case("expires") { + expires = parse_http_date(aval); + } + } + + // §5.3 step 6: reject outright if the Domain attribute doesn't cover + // the host that sent it. + let (domain, host_only) = match domain { + Some(d) => { + if !domain_matches(request_host, &d) { + return None; + } + (d, false) + } + None => (request_host.to_string(), true), + }; + + // Max-Age wins over Expires (§5.3 step 3). A non-positive Max-Age, or + // an Expires in the past, means "delete this". + let expired = match max_age { + Some(secs) => secs <= 0, + None => match expires { + Some(when) => when <= now_unix(), + None => false, + }, + }; + + Some(( + Cookie { + name: name.to_string(), + value: val.trim().to_string(), + domain, + host_only, + path: path.unwrap_or_else(|| default_path(request_path)), + secure, + }, + expired, + )) +} + +fn now_unix() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or(0) +} + +/// Tolerant HTTP-date parser covering the formats cookies actually use: +/// IMF-fixdate (`Sun, 06 Nov 1994 08:49:37 GMT`), RFC 850 with a 2-digit +/// year (`Sunday, 06-Nov-94 08:49:37 GMT`), and asctime. Rather than match +/// formats it tokenizes and picks out day / month / year / time, which is +/// how real clients cope with the variety servers emit. Returns seconds +/// since the Unix epoch. +fn parse_http_date(s: &str) -> Option { + let normalized: String = s + .chars() + .map(|c| if c == '-' || c == ',' { ' ' } else { c }) + .collect(); + + const MONTHS: [&str; 12] = [ + "jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec", + ]; + + let mut day: Option = None; + let mut month: Option = None; + let mut year: Option = None; + let mut time: Option<(i64, i64, i64)> = None; + + for token in normalized.split_whitespace() { + if token.contains(':') && time.is_none() { + let mut it = token.split(':'); + let h = it.next()?.parse::().ok()?; + let m = it.next()?.parse::().ok()?; + let sec = it.next().and_then(|v| v.parse::().ok()).unwrap_or(0); + time = Some((h, m, sec)); + continue; + } + if month.is_none() && token.len() >= 3 { + let prefix = token[..3].to_ascii_lowercase(); + if let Some(i) = MONTHS.iter().position(|m| *m == prefix) { + month = Some(i as i64 + 1); + continue; + } + } + if let Ok(n) = token.parse::() { + // One or two digits is a day-of-month if we still need one; anything + // else (or a second number) is the year. + if day.is_none() && token.len() <= 2 && (1..=31).contains(&n) { + day = Some(n); + } else if year.is_none() { + year = Some(if token.len() <= 2 { + if n < 70 { 2000 + n } else { 1900 + n } + } else { + n + }); + } + } + } + + let (day, month, year) = (day?, month?, year?); + let (h, m, sec) = time.unwrap_or((0, 0, 0)); + Some(days_from_civil(year, month, day) * 86400 + h * 3600 + m * 60 + sec) +} + +/// Days since 1970-01-01 for a proleptic-Gregorian date (Howard Hinnant's +/// `days_from_civil`). Avoids pulling in a date crate for one calculation. +fn days_from_civil(y: i64, m: i64, d: i64) -> i64 { + let y = if m <= 2 { y - 1 } else { y }; + let era = if y >= 0 { y } else { y - 399 } / 400; + let yoe = y - era * 400; + let mp = if m > 2 { m - 3 } else { m + 9 }; + let doy = (153 * mp + 2) / 5 + d - 1; + let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + era * 146097 + doe - 719468 +} + +#[cfg(test)] +mod tests { + use super::*; + + fn uri(s: &str) -> http::Uri { + s.parse().unwrap() + } + + fn set(jar: &mut CookieJar, url: &str, values: &[&str]) { + let headers: Vec<(String, String)> = values + .iter() + .map(|v| ("set-cookie".to_string(), v.to_string())) + .collect(); + jar.store(&headers, &uri(url)); + } + + #[test] + fn cookie_set_on_redirect_is_sent_to_next_hop() { + let mut jar = CookieJar::new(); + set( + &mut jar, + "https://example.com/login", + &["session=abc123; Path=/"], + ); + assert_eq!( + jar.header_for(&uri("https://example.com/dashboard")), + Some("session=abc123".to_string()) + ); + } + + #[test] + fn host_only_cookie_does_not_reach_subdomains_or_siblings() { + let mut jar = CookieJar::new(); + // No Domain attribute -> host-only. + set(&mut jar, "https://example.com/", &["a=1"]); + assert!(jar.header_for(&uri("https://www.example.com/")).is_none()); + assert!(jar.header_for(&uri("https://evil.com/")).is_none()); + assert!(jar.header_for(&uri("https://example.com/")).is_some()); + } + + #[test] + fn domain_attribute_covers_subdomains() { + let mut jar = CookieJar::new(); + set( + &mut jar, + "https://www.example.com/", + &["a=1; Domain=example.com"], + ); + assert!(jar.header_for(&uri("https://example.com/")).is_some()); + assert!(jar.header_for(&uri("https://api.example.com/")).is_some()); + // Suffix match must land on a label boundary. + assert!(jar.header_for(&uri("https://notexample.com/")).is_none()); + } + + #[test] + fn domain_not_covering_the_setting_host_is_rejected() { + let mut jar = CookieJar::new(); + // A redirect target must not be able to set cookies for elsewhere. + set(&mut jar, "https://evil.com/", &["a=1; Domain=example.com"]); + assert!(jar.is_empty()); + assert!(jar.header_for(&uri("https://example.com/")).is_none()); + } + + #[test] + fn ip_host_cannot_widen_via_domain() { + let mut jar = CookieJar::new(); + set(&mut jar, "http://10.0.0.1/", &["a=1; Domain=0.0.1"]); + assert!(jar.is_empty()); + } + + #[test] + fn secure_cookie_is_withheld_over_plaintext() { + let mut jar = CookieJar::new(); + set(&mut jar, "https://example.com/", &["s=1; Secure", "p=2"]); + assert_eq!( + jar.header_for(&uri("http://example.com/")), + Some("p=2".to_string()) + ); + assert!( + jar.header_for(&uri("https://example.com/")) + .unwrap() + .contains("s=1") + ); + } + + #[test] + fn path_scoping() { + let mut jar = CookieJar::new(); + set(&mut jar, "https://example.com/", &["a=1; Path=/admin"]); + assert!(jar.header_for(&uri("https://example.com/admin")).is_some()); + assert!( + jar.header_for(&uri("https://example.com/admin/users")) + .is_some() + ); + // Prefix must break on a boundary, not mid-segment. + assert!( + jar.header_for(&uri("https://example.com/administrator")) + .is_none() + ); + assert!(jar.header_for(&uri("https://example.com/other")).is_none()); + } + + #[test] + fn default_path_is_the_requests_directory() { + assert_eq!(default_path("/a/b/c"), "/a/b"); + assert_eq!(default_path("/a"), "/"); + assert_eq!(default_path("/"), "/"); + assert_eq!(default_path(""), "/"); + } + + #[test] + fn longer_paths_are_sent_first() { + let mut jar = CookieJar::new(); + set( + &mut jar, + "https://example.com/", + &["broad=1; Path=/", "narrow=2; Path=/admin/panel"], + ); + assert_eq!( + jar.header_for(&uri("https://example.com/admin/panel")), + Some("narrow=2; broad=1".to_string()) + ); + } + + #[test] + fn resetting_the_same_cookie_replaces_it() { + let mut jar = CookieJar::new(); + set(&mut jar, "https://example.com/", &["a=1"]); + set(&mut jar, "https://example.com/", &["a=2"]); + assert_eq!( + jar.header_for(&uri("https://example.com/")), + Some("a=2".to_string()) + ); + } + + #[test] + fn expired_cookie_deletes_instead_of_storing() { + let mut jar = CookieJar::new(); + set(&mut jar, "https://example.com/", &["a=1"]); + set(&mut jar, "https://example.com/", &["a=; Max-Age=0"]); + assert!(jar.header_for(&uri("https://example.com/")).is_none()); + + set(&mut jar, "https://example.com/", &["b=1"]); + set( + &mut jar, + "https://example.com/", + &["b=; Expires=Thu, 01 Jan 1970 00:00:00 GMT"], + ); + assert!(jar.header_for(&uri("https://example.com/")).is_none()); + } + + #[test] + fn future_expiry_is_kept() { + let mut jar = CookieJar::new(); + set( + &mut jar, + "https://example.com/", + &["a=1; Expires=Tue, 01 Jan 2999 00:00:00 GMT"], + ); + assert!(jar.header_for(&uri("https://example.com/")).is_some()); + } + + #[test] + fn malformed_set_cookie_is_ignored() { + let mut jar = CookieJar::new(); + set( + &mut jar, + "https://example.com/", + &["novalue", "=noname", ""], + ); + assert!(jar.is_empty()); + } + + #[test] + fn host_matching_is_case_and_trailing_dot_insensitive() { + let mut jar = CookieJar::new(); + set(&mut jar, "https://Example.COM./", &["a=1"]); + assert!(jar.header_for(&uri("https://example.com/")).is_some()); + } + + #[test] + fn empty_value_is_preserved() { + let mut jar = CookieJar::new(); + set(&mut jar, "https://example.com/", &["a="]); + assert_eq!( + jar.header_for(&uri("https://example.com/")), + Some("a=".to_string()) + ); + } + + #[test] + fn http_date_formats() { + // IMF-fixdate. + assert_eq!( + parse_http_date("Sun, 06 Nov 1994 08:49:37 GMT"), + Some(784111777) + ); + // RFC 850, two-digit year. + assert_eq!( + parse_http_date("Sunday, 06-Nov-94 08:49:37 GMT"), + Some(784111777) + ); + // asctime. + assert_eq!(parse_http_date("Sun Nov 6 08:49:37 1994"), Some(784111777)); + assert_eq!(parse_http_date("Thu, 01 Jan 1970 00:00:00 GMT"), Some(0)); + assert_eq!(parse_http_date("garbage"), None); + } +} diff --git a/src/lib.rs b/src/lib.rs index f45d10d..a2f112b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,7 @@ pub mod batch; pub mod client; pub mod config; +pub mod cookies; pub mod debug; pub mod h2; pub mod response; diff --git a/src/main.rs b/src/main.rs index 24a5219..691c41d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -52,6 +52,10 @@ struct Cli { #[arg(long, default_value = "10")] max_redirects: u32, + /// Don't apply cookies a redirect sets to the hops that follow it + #[arg(long)] + no_redirect_cookies: bool, + /// Request timeout in seconds #[arg(short, long)] timeout: Option, @@ -111,6 +115,7 @@ fn build_config(cli: &Cli, url: String) -> RequestConfig { config.verify_certs = Some(cli.verify); config.follow_redirects = Some(cli.follow_redirects); config.max_redirects = Some(cli.max_redirects); + config.redirect_cookies = Some(!cli.no_redirect_cookies); config.timeout_seconds = cli.timeout; config.max_body_size = cli.max_body_size; config.proxy = cli.proxy.clone(); diff --git a/src/mock.rs b/src/mock.rs index d7a63de..133d263 100644 --- a/src/mock.rs +++ b/src/mock.rs @@ -838,7 +838,7 @@ impl PyBlasthttpMock { Some(body_obj) }; let files_obj = bound.getattr("files").ok(); - let files = files_obj.and_then(|f| if f.is_none() { None } else { Some(f) }); + let files = files_obj.filter(|f| !f.is_none()); let (body_bytes, final_headers) = crate::python::apply_body_and_files(body, files, headers)?; mock_entries.push(MockBatchEntry { diff --git a/src/python.rs b/src/python.rs index 8d4168f..24e8ef5 100644 --- a/src/python.rs +++ b/src/python.rs @@ -869,6 +869,7 @@ impl BlastHTTP { timeout=None, follow_redirects=None, max_redirects=None, + redirect_cookies=None, verify_certs=None, proxy=None, no_proxy=None, @@ -895,6 +896,7 @@ impl BlastHTTP { timeout: Option, follow_redirects: Option, max_redirects: Option, + redirect_cookies: Option, verify_certs: Option, proxy: Option, no_proxy: Option>, @@ -919,6 +921,7 @@ impl BlastHTTP { max_body_size, follow_redirects, max_redirects, + redirect_cookies, verify_certs, proxy, no_proxy: no_proxy.unwrap_or_default(), @@ -1085,6 +1088,7 @@ impl BlastHTTP { no_proxy=None, headers=None, retries=None, + redirect_cookies=None, ))] #[allow(clippy::too_many_arguments)] fn download<'py>( @@ -1099,6 +1103,7 @@ impl BlastHTTP { no_proxy: Option>, headers: Option>, retries: Option, + redirect_cookies: Option, ) -> PyResult> { let config = RequestConfig { url, @@ -1109,6 +1114,7 @@ impl BlastHTTP { max_body_size: max_size, follow_redirects: Some(true), max_redirects: Some(10), + redirect_cookies, verify_certs, proxy, no_proxy: no_proxy.unwrap_or_default(), @@ -1338,6 +1344,8 @@ struct PyBatchConfig { #[pyo3(get, set)] max_redirects: Option, #[pyo3(get, set)] + redirect_cookies: Option, + #[pyo3(get, set)] verify_certs: Option, #[pyo3(get, set)] proxy: Option, @@ -1375,6 +1383,7 @@ impl PyBatchConfig { timeout=None, follow_redirects=None, max_redirects=None, + redirect_cookies=None, verify_certs=None, proxy=None, no_proxy=None, @@ -1398,6 +1407,7 @@ impl PyBatchConfig { timeout: Option, follow_redirects: Option, max_redirects: Option, + redirect_cookies: Option, verify_certs: Option, proxy: Option, no_proxy: Option>, @@ -1420,6 +1430,7 @@ impl PyBatchConfig { timeout, follow_redirects, max_redirects, + redirect_cookies, verify_certs, proxy, no_proxy, @@ -1447,6 +1458,7 @@ impl Clone for PyBatchConfig { timeout: self.timeout, follow_redirects: self.follow_redirects, max_redirects: self.max_redirects, + redirect_cookies: self.redirect_cookies, verify_certs: self.verify_certs, proxy: self.proxy.clone(), no_proxy: self.no_proxy.clone(), @@ -1477,6 +1489,7 @@ impl PyBatchConfig { max_body_size: None, follow_redirects: self.follow_redirects, max_redirects: self.max_redirects, + redirect_cookies: self.redirect_cookies, verify_certs: self.verify_certs, proxy: self.proxy, no_proxy: self.no_proxy.unwrap_or_default(), diff --git a/tests/redirect_cookies.rs b/tests/redirect_cookies.rs new file mode 100644 index 0000000..0cb680c --- /dev/null +++ b/tests/redirect_cookies.rs @@ -0,0 +1,152 @@ +//! End-to-end checks that cookies set mid-redirect reach the next hop. +//! +//! The jar itself is unit-tested in `src/cookies.rs`; these tests prove the +//! wiring, i.e. that a real request through the client picks the cookie off +//! a 302 and puts it on the wire for the hop that follows. + +use blasthttp::client::HttpClient; +use blasthttp::client::hyper::HyperClient; +use blasthttp::config::RequestConfig; +use std::io::{BufRead, BufReader, Write}; +use std::net::{TcpListener, TcpStream}; +use std::sync::mpsc; +use std::thread; + +/// Read one request's headers off a socket and hand back the raw lines. +fn read_request(stream: &mut TcpStream) -> Vec { + let mut reader = BufReader::new(stream.try_clone().unwrap()); + let mut lines = Vec::new(); + loop { + let mut line = String::new(); + if reader.read_line(&mut line).unwrap_or(0) == 0 { + break; + } + let trimmed = line.trim_end_matches(['\r', '\n']).to_string(); + if trimmed.is_empty() { + break; + } + lines.push(trimmed); + } + lines +} + +/// Serve `/start` as a 302 to `/end` that sets a cookie, then serve `/end`. +/// Returns the port and a channel carrying the request lines for each hop. +fn spawn_server(set_cookie: &'static str) -> (u16, mpsc::Receiver>) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + let (tx, rx) = mpsc::channel(); + + thread::spawn(move || { + // Two hops: the redirect, then the destination. The client may open a + // fresh connection for the second, so accept in a loop. + for _ in 0..2 { + let Ok((mut stream, _)) = listener.accept() else { + return; + }; + let lines = read_request(&mut stream); + let target = lines + .first() + .and_then(|l| l.split_whitespace().nth(1)) + .unwrap_or("/") + .to_string(); + let _ = tx.send(lines); + + let response = if target.ends_with("/start") { + format!( + "HTTP/1.1 302 Found\r\nLocation: /end\r\nSet-Cookie: {}\r\n\ + Content-Length: 0\r\nConnection: close\r\n\r\n", + set_cookie + ) + } else { + "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok".to_string() + }; + let _ = stream.write_all(response.as_bytes()); + let _ = stream.flush(); + } + }); + + (port, rx) +} + +/// The `Cookie` header the second hop received, if any. +fn second_hop_cookie(rx: &mpsc::Receiver>) -> Option { + let _first = rx + .recv_timeout(std::time::Duration::from_secs(5)) + .expect("no first hop"); + let second = rx + .recv_timeout(std::time::Duration::from_secs(5)) + .expect("no second hop"); + second + .iter() + .find(|l| l.to_ascii_lowercase().starts_with("cookie:")) + .map(|l| l[7..].trim().to_string()) +} + +async fn run(port: u16, mutate: impl FnOnce(&mut RequestConfig)) { + let mut config = RequestConfig::new(format!("http://127.0.0.1:{}/start", port)); + config.follow_redirects = Some(true); + mutate(&mut config); + let client = HyperClient::new(); + let resp = client.send(&config).await.expect("request failed"); + assert_eq!(resp.status, 200, "should have landed on /end"); +} + +#[tokio::test] +async fn cookie_from_redirect_is_sent_on_the_next_hop() { + let (port, rx) = spawn_server("session=abc123; Path=/"); + run(port, |_| {}).await; + assert_eq!(second_hop_cookie(&rx).as_deref(), Some("session=abc123")); +} + +#[tokio::test] +async fn opting_out_restores_the_old_behavior() { + let (port, rx) = spawn_server("session=abc123; Path=/"); + run(port, |c| c.redirect_cookies = Some(false)).await; + assert_eq!(second_hop_cookie(&rx), None); +} + +#[tokio::test] +async fn secure_cookie_is_withheld_from_a_plaintext_hop() { + let (port, rx) = spawn_server("session=abc123; Path=/; Secure"); + run(port, |_| {}).await; + assert_eq!(second_hop_cookie(&rx), None); +} + +#[tokio::test] +async fn cookie_scoped_to_another_path_is_not_sent() { + let (port, rx) = spawn_server("session=abc123; Path=/somewhere-else"); + run(port, |_| {}).await; + assert_eq!(second_hop_cookie(&rx), None); +} + +#[tokio::test] +async fn cookie_for_an_unrelated_domain_is_rejected() { + let (port, rx) = spawn_server("session=abc123; Domain=example.com"); + run(port, |_| {}).await; + assert_eq!(second_hop_cookie(&rx), None); +} + +#[tokio::test] +async fn chain_cookie_merges_with_a_caller_supplied_cookie_header() { + let (port, rx) = spawn_server("session=abc123; Path=/"); + run(port, |c| { + c.headers = Some(vec![("Cookie".to_string(), "mine=1".to_string())]); + }) + .await; + // One header carrying both, not two Cookie headers. + assert_eq!( + second_hop_cookie(&rx).as_deref(), + Some("mine=1; session=abc123") + ); +} + +#[tokio::test] +async fn caller_cookie_survives_when_the_chain_adds_nothing() { + let (port, rx) = spawn_server("bad; no-equals-sign"); + run(port, |c| { + c.headers = Some(vec![("Cookie".to_string(), "mine=1".to_string())]); + }) + .await; + assert_eq!(second_hop_cookie(&rx).as_deref(), Some("mine=1")); +}