From 9cf8b91c24fffaf0ca63ad0d546a7a42598ae049 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 08:45:12 +0000 Subject: [PATCH 1/3] Traffic statistics: counts, top clients and a request log as Parquet in the bucket Every instance counts the requests it answers in memory (by minute, surface, repository, method and status, with a latency histogram), keeps the top client networks of each minute in bounded memory, and logs each request. Once a minute it writes Parquet files of its own under traffic/v1/ in Forge's bucket; the admin API lists and reads every instance's files from there, so Admin -> Traffic shows all instances without calls between them. - crates/traffic (forge-traffic): the recorder, the request-scoped slot the principal layer and the dispatchers note the client, account and repository in, the Parquet rows (pure-Rust parquet, Snappy), the file layout, flushing, pruning and the queries. - BlobStore: put_bytes, get_bytes and list (ListObjectsV2). - /admin/traffic, /admin/traffic/client and /admin/settings/traffic; the traffic worker flushes every minute and prunes every hour, and the last counts are written on shutdown. - Admin -> Traffic in the web UI: totals, an SVG chart or table over time, top clients with their request logs, repositories, surfaces, statuses, and the settings. Fixes #709 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012e6DJbpgZmQnhz5Jjjhug6 --- Cargo.lock | 130 +++ Cargo.toml | 7 + crates/server/Cargo.toml | 1 + crates/server/src/admin/mod.rs | 8 + crates/server/src/admin/traffic.rs | 362 +++++++ crates/server/src/app.rs | 6 +- crates/server/src/images/dispatch.rs | 8 +- crates/server/src/lib.rs | 1 + crates/server/src/main.rs | 3 + crates/server/src/middleware/principal.rs | 7 +- crates/server/src/plugins.rs | 6 +- crates/server/src/state.rs | 11 +- crates/server/src/telemetry.rs | 65 +- crates/server/src/traffic.rs | 233 +++++ crates/server/src/workers.rs | 18 + crates/server/tests/traffic.rs | 238 +++++ crates/storage/src/lib.rs | 2 +- crates/storage/src/s3.rs | 105 ++ crates/telemetry/src/lib.rs | 1 + crates/traffic/Cargo.toml | 29 + crates/traffic/src/layout.rs | 693 ++++++++++++ crates/traffic/src/lib.rs | 104 ++ crates/traffic/src/objects.rs | 127 +++ crates/traffic/src/query.rs | 985 ++++++++++++++++++ crates/traffic/src/record.rs | 377 +++++++ crates/traffic/src/rows.rs | 312 ++++++ crates/traffic/src/settings.rs | 65 ++ crates/traffic/src/slot.rs | 112 ++ crates/traffic/src/top.rs | 190 ++++ docs/changes/2.2.0/709.feature.md | 12 + docs/deployment.md | 55 +- web/src/api/traffic.ts | 233 +++++ web/src/components/AppRail.tsx | 1 + web/src/components/TrafficChart.tsx | 249 +++++ web/src/components/TrafficFilters.tsx | 50 + web/src/components/ui/icons.tsx | 1 + web/src/lib/traffic.ts | 86 ++ web/src/queries/useTraffic.ts | 65 ++ web/src/routeTree.gen.ts | 63 ++ web/src/routes.ts | 10 +- web/src/routes/admin/AdminHome.tsx | 15 +- web/src/routes/admin/traffic/Client.tsx | 271 +++++ web/src/routes/admin/traffic/Settings.tsx | 209 ++++ web/src/routes/admin/traffic/Traffic.tsx | 332 ++++++ web/src/routes/admin/traffic/traffic.test.tsx | 309 ++++++ web/src/styles/forge-ui.css | 19 + 46 files changed, 6165 insertions(+), 21 deletions(-) create mode 100644 crates/server/src/admin/traffic.rs create mode 100644 crates/server/src/traffic.rs create mode 100644 crates/server/tests/traffic.rs create mode 100644 crates/traffic/Cargo.toml create mode 100644 crates/traffic/src/layout.rs create mode 100644 crates/traffic/src/lib.rs create mode 100644 crates/traffic/src/objects.rs create mode 100644 crates/traffic/src/query.rs create mode 100644 crates/traffic/src/record.rs create mode 100644 crates/traffic/src/rows.rs create mode 100644 crates/traffic/src/settings.rs create mode 100644 crates/traffic/src/slot.rs create mode 100644 crates/traffic/src/top.rs create mode 100644 docs/changes/2.2.0/709.feature.md create mode 100644 web/src/api/traffic.ts create mode 100644 web/src/components/TrafficChart.tsx create mode 100644 web/src/components/TrafficFilters.tsx create mode 100644 web/src/lib/traffic.ts create mode 100644 web/src/queries/useTraffic.ts create mode 100644 web/src/routes/admin/traffic/Client.tsx create mode 100644 web/src/routes/admin/traffic/Settings.tsx create mode 100644 web/src/routes/admin/traffic/Traffic.tsx create mode 100644 web/src/routes/admin/traffic/traffic.test.tsx diff --git a/Cargo.lock b/Cargo.lock index d8dad922..97265e15 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -45,6 +45,20 @@ dependencies = [ "zeroize", ] +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "const-random", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.5" @@ -1083,6 +1097,26 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -1206,6 +1240,12 @@ version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-common" version = "0.1.7" @@ -1846,6 +1886,7 @@ dependencies = [ "forge-plugin", "forge-storage", "forge-telemetry", + "forge-traffic", "futures-util", "hex", "http", @@ -1933,6 +1974,26 @@ dependencies = [ "tracing", ] +[[package]] +name = "forge-traffic" +version = "2.1.0" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "chrono", + "forge-storage", + "futures-util", + "moka", + "parquet", + "parquet_derive", + "rand 0.10.3", + "serde", + "serde_json", + "tokio", + "tracing", +] + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -2138,6 +2199,18 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -3267,6 +3340,36 @@ dependencies = [ "windows-link", ] +[[package]] +name = "parquet" +version = "60.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "806eaa0f1c0436dcb43bee8e6a82bf2efff90521a60172f8281cb9ef577b43a4" +dependencies = [ + "ahash", + "bytes", + "chrono", + "half", + "hashbrown 0.17.1", + "num-bigint 0.5.1", + "num-integer", + "num-traits", + "seq-macro", + "snap", + "twox-hash", +] + +[[package]] +name = "parquet_derive" +version = "60.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c6a0cc5516d0c382595f124ba4f88fdfff723adec2adf58c7d34730d0c30b53" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "parse-display" version = "0.9.1" @@ -4115,6 +4218,12 @@ version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + [[package]] name = "serde" version = "1.0.229" @@ -4368,6 +4477,12 @@ dependencies = [ "serde", ] +[[package]] +name = "snap" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "199905e6153d6405f9728fe44daace35f8f837bbf830bb6e85fbd5828709a886" + [[package]] name = "socket2" version = "0.6.5" @@ -4856,6 +4971,15 @@ dependencies = [ "time-core", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tinystr" version = "0.8.4" @@ -5172,6 +5296,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "twox-hash" +version = "2.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86a801b3cea342a06d468c8710662aa29e5e05e4f5c0d62f00bbb7f2ad7941c2" + [[package]] name = "typed-path" version = "0.12.3" diff --git a/Cargo.toml b/Cargo.toml index f5717aac..1f23f7ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "crates/core", "crates/crypto", "crates/telemetry", + "crates/traffic", "crates/plugin-api", "crates/plugin", "crates/server", @@ -192,6 +193,12 @@ jsonwebtoken = { version = "11", default-features = false, features = ["use_pem" # already links, used directly where `jsonwebtoken` has no key generation. aws-lc-rs = { version = "1.18" } +# Traffic statistics (crates/traffic): Parquet files in the object store. +# Without the Arrow layer (its record API suffices) and with Snappy alone, a +# pure-Rust codec. +parquet = { version = "60", default-features = false, features = ["snap"] } +parquet_derive = { version = "60" } + # Temp files tempfile = { version = "3" } diff --git a/crates/server/Cargo.toml b/crates/server/Cargo.toml index 1b7da79a..5f9049af 100644 --- a/crates/server/Cargo.toml +++ b/crates/server/Cargo.toml @@ -22,6 +22,7 @@ forge-core = { path = "../core" } forge-crypto = { path = "../crypto" } forge-plugin = { path = "../plugin" } forge-telemetry = { path = "../telemetry" } +forge-traffic = { path = "../traffic" } # The built-in format plugins (crates/formats/docker, crates/formats/nuget, # crates/formats/npm, crates/formats/raw), linked in. forge-format-docker = { path = "../formats/docker" } diff --git a/crates/server/src/admin/mod.rs b/crates/server/src/admin/mod.rs index c17790d8..a5285087 100644 --- a/crates/server/src/admin/mod.rs +++ b/crates/server/src/admin/mod.rs @@ -69,6 +69,7 @@ pub mod roles; pub mod settings; pub mod token_scope; pub mod tokens; +pub mod traffic; pub mod trusted_publishers; pub mod user_roles; pub mod users; @@ -350,6 +351,13 @@ pub fn admin_router(state: AppState) -> Router { .route("/crypto/keys", get(crypto::list_keys)) // --- local blob cache ---------------------------------------------- .route("/blob-cache", get(blob_cache::stats)) + // --- traffic statistics -------------------------------------------- + .route("/traffic", get(traffic::overview)) + .route("/traffic/client", get(traffic::client)) + .route( + "/settings/traffic", + get(traffic::get_settings).put(traffic::put_settings), + ) // --- the email service --------------------------------------------- .route( "/settings/email", diff --git a/crates/server/src/admin/traffic.rs b/crates/server/src/admin/traffic.rs new file mode 100644 index 00000000..2abf83d7 --- /dev/null +++ b/crates/server/src/admin/traffic.rs @@ -0,0 +1,362 @@ +//! The traffic statistics (Admin → Traffic). **Admin-only.** +//! +//! * `GET /admin/traffic?range=&instance=&repository=` — the traffic of a +//! range ([`forge_traffic::query::Overview`]) over every instance or the +//! one `instance` names: counts by time bucket, surface, repository and +//! status, latency, and the top clients. `range` is one of [`RANGES`] +//! (default `24h`), or `from` and `to` (RFC 3339) give the range. +//! * `GET /admin/traffic/client?network=&range=&instance=` — one client +//! network's traffic and its requests in the request log +//! ([`forge_traffic::query::ClientDetail`]). +//! * `GET /admin/settings/traffic` answers the [`TrafficSettings`]; `PUT` +//! changes the fields a body names, for every instance (each follows with +//! its next flush). +//! +//! The files are read from the object store, so the answers cover every +//! instance; a store that cannot be read answers 503. + +use axum::extract::{Query, State}; +use axum::http::StatusCode; +use axum::response::{IntoResponse, Json, Response}; +use chrono::{DateTime, Duration, Utc}; +use forge_core::auth::principal::Principal; +use forge_traffic::query::{self, Filter, Span}; +use forge_traffic::TrafficSettings; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +use crate::state::AppState; +use crate::traffic::{read_settings, write_settings}; + +/// The ranges `range` names, and their length. +pub const RANGES: [(&str, i64); 6] = [ + ("1h", 3_600), + ("6h", 6 * 3_600), + ("24h", 24 * 3_600), + ("7d", 7 * 86_400), + ("30d", 30 * 86_400), + ("90d", 90 * 86_400), +]; + +/// The longest range `from` and `to` may give. +const MAX_RANGE_DAYS: i64 = 400; + +/// The most users whose names an answer looks up. +const MAX_USERNAMES: usize = 50; + +/// The query of both views. +#[derive(Debug, Default, Deserialize)] +pub struct RangeQuery { + pub range: Option, + pub from: Option>, + pub to: Option>, + /// Only this instance's files (an id of the answer's `instances`). + pub instance: Option, + /// The overview: only this repository's counts. + pub repository: Option, + /// The client view: the network ([`crate::traffic::network_label`]). + pub network: Option, +} + +fn bad_request(msg: impl Into) -> Response { + (StatusCode::BAD_REQUEST, msg.into()).into_response() +} + +fn unavailable(e: anyhow::Error) -> Response { + tracing::warn!(error = %format!("{e:#}"), "cannot read the traffic statistics"); + ( + StatusCode::SERVICE_UNAVAILABLE, + "cannot read the traffic statistics from the object store", + ) + .into_response() +} + +impl RangeQuery { + fn span(&self, now: DateTime) -> Result { + let (from, to) = match (self.from, self.to) { + (Some(from), Some(to)) => { + if to <= from { + return Err(bad_request("to must be after from")); + } + if to - from > Duration::days(MAX_RANGE_DAYS) { + return Err(bad_request(format!( + "a range is at most {MAX_RANGE_DAYS} days" + ))); + } + (from, to) + } + (None, None) => { + let name = self.range.as_deref().unwrap_or("24h"); + let Some((_, secs)) = RANGES.iter().find(|(n, _)| *n == name) else { + let names: Vec<&str> = RANGES.iter().map(|(n, _)| *n).collect(); + return Err(bad_request(format!( + "range must be one of {}", + names.join(", ") + ))); + }; + (now - Duration::seconds(*secs), now) + } + _ => return Err(bad_request("give both from and to, or neither")), + }; + Ok(Span::new(from, to, now)) + } + + fn instance(&self) -> Option { + self.instance.clone().filter(|i| !i.is_empty()) + } +} + +/// The names of the users `ids` name, as far as they still exist. +async fn usernames(state: &AppState, ids: impl Iterator) -> HashMap { + let mut wanted: Vec = ids.collect(); + wanted.sort_unstable(); + wanted.dedup(); + wanted.truncate(MAX_USERNAMES); + let mut names = HashMap::new(); + for id in wanted { + match forge_db::user_dao::find_user_by_id(&state.pool, id).await { + Ok(Some(user)) => { + names.insert(id, user.username); + } + Ok(None) => {} + Err(e) => { + forge_db::observe_error(&e); + tracing::warn!(error = %e, "cannot look up the users of the traffic statistics"); + break; + } + } + } + names +} + +/// What the views say of the settings, so the page can tell why something +/// is missing. +#[derive(Debug, Serialize)] +pub struct Recording { + pub enabled: bool, + pub client_addresses: bool, + pub request_log: bool, +} + +impl From<&TrafficSettings> for Recording { + fn from(s: &TrafficSettings) -> Self { + Recording { + enabled: s.enabled, + client_addresses: s.client_addresses, + request_log: s.request_log, + } + } +} + +/// `GET /admin/traffic`. +#[derive(Debug, Serialize)] +pub struct OverviewView { + #[serde(flatten)] + pub overview: query::Overview, + pub recording: Recording, +} + +/// `GET /admin/traffic` +pub async fn overview(State(state): State, Query(q): Query) -> Response { + let span = match q.span(Utc::now()) { + Ok(span) => span, + Err(resp) => return resp, + }; + let settings = match stored(&state).await { + Ok(settings) => settings, + Err(resp) => return resp, + }; + let filter = Filter { + instance: q.instance(), + repository: q.repository.clone().filter(|r| !r.is_empty()), + }; + let traffic = &state.traffic; + let mut overview = match query::overview(&*traffic.objects, &traffic.cache, span, &filter).await + { + Ok(overview) => overview, + Err(e) => return unavailable(e), + }; + let names = usernames(&state, overview.clients.iter().filter_map(|c| c.user_id)).await; + for client in &mut overview.clients { + client.username = client.user_id.and_then(|id| names.get(&id).cloned()); + } + Json(OverviewView { + overview, + recording: Recording::from(&settings), + }) + .into_response() +} + +/// `GET /admin/traffic/client`. +#[derive(Debug, Serialize)] +pub struct ClientView { + #[serde(flatten)] + pub detail: query::ClientDetail, + pub recording: Recording, +} + +/// `GET /admin/traffic/client` +pub async fn client(State(state): State, Query(q): Query) -> Response { + let Some(network) = q.network.clone().filter(|n| !n.is_empty()) else { + return bad_request("network is required"); + }; + let span = match q.span(Utc::now()) { + Ok(span) => span, + Err(resp) => return resp, + }; + let settings = match stored(&state).await { + Ok(settings) => settings, + Err(resp) => return resp, + }; + let traffic = &state.traffic; + let instance = q.instance(); + let mut detail = match query::client( + &*traffic.objects, + &traffic.cache, + span, + instance.as_deref(), + &network, + ) + .await + { + Ok(detail) => detail, + Err(e) => return unavailable(e), + }; + let ids = detail + .totals + .iter() + .filter_map(|t| t.user_id) + .chain(detail.log.entries.iter().filter_map(|e| e.user_id)) + .chain(detail.log.accounts.iter().filter_map(|a| a.user_id)); + let names = usernames(&state, ids).await; + if let Some(totals) = &mut detail.totals { + totals.username = totals.user_id.and_then(|id| names.get(&id).cloned()); + } + for entry in &mut detail.log.entries { + entry.username = entry.user_id.and_then(|id| names.get(&id).cloned()); + } + for account in &mut detail.log.accounts { + account.username = account.user_id.and_then(|id| names.get(&id).cloned()); + } + Json(ClientView { + detail, + recording: Recording::from(&settings), + }) + .into_response() +} + +/// The stored settings, or the 500 answer when they cannot be read. +async fn stored(state: &AppState) -> Result { + read_settings(&state.pool).await.map_err(|e| { + tracing::error!(error = %e, "cannot read the traffic settings"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + "cannot read the traffic settings", + ) + .into_response() + }) +} + +/// `PUT /admin/settings/traffic` body: the fields to change. Unknown fields +/// are refused, so a misspelled one is not dropped. +#[derive(Debug, Default, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TrafficSettingsUpdate { + pub enabled: Option, + pub client_addresses: Option, + pub request_log: Option, + pub statistics_retention_days: Option, + pub client_retention_days: Option, + pub request_log_retention_days: Option, +} + +/// `GET /admin/settings/traffic` +pub async fn get_settings(State(state): State) -> Response { + match stored(&state).await { + Ok(settings) => Json(settings).into_response(), + Err(resp) => resp, + } +} + +/// `PUT /admin/settings/traffic` — change the fields the body names. This +/// instance applies a change at once, the others with their next flush. +pub async fn put_settings( + State(state): State, + axum::Extension(principal): axum::Extension, + Json(update): Json, +) -> Response { + let current = match stored(&state).await { + Ok(settings) => settings, + Err(resp) => return resp, + }; + let mut next = current.clone(); + let TrafficSettingsUpdate { + enabled, + client_addresses, + request_log, + statistics_retention_days, + client_retention_days, + request_log_retention_days, + } = update; + next.enabled = enabled.unwrap_or(next.enabled); + next.client_addresses = client_addresses.unwrap_or(next.client_addresses); + next.request_log = request_log.unwrap_or(next.request_log); + next.statistics_retention_days = + statistics_retention_days.unwrap_or(next.statistics_retention_days); + next.client_retention_days = client_retention_days.unwrap_or(next.client_retention_days); + next.request_log_retention_days = + request_log_retention_days.unwrap_or(next.request_log_retention_days); + if let Err(msg) = next.check() { + return bad_request(msg); + } + if next != current { + if let Err(e) = write_settings(&state.pool, &next).await { + return super::db_error(e); + } + state.traffic.recorder.apply(&next); + tracing::info!( + target: "forge::audit", + settings = ?next, + principal = %principal.audit_label(), + "traffic settings changed" + ); + } + Json(next).into_response() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn at(s: &str) -> DateTime { + DateTime::parse_from_rfc3339(s).unwrap().to_utc() + } + + #[test] + fn ranges_are_named_or_given() { + let now = at("2026-10-30T14:10:00Z"); + let span = RangeQuery::default().span(now).unwrap(); + assert_eq!(span.from, at("2026-10-29T14:10:00Z"), "24 hours by default"); + let q = RangeQuery { + range: Some("7d".into()), + ..RangeQuery::default() + }; + assert_eq!(q.span(now).unwrap().resolution, "hour"); + let q = RangeQuery { + range: Some("2h".into()), + ..RangeQuery::default() + }; + assert!(q.span(now).is_err()); + let q = RangeQuery { + from: Some(at("2026-10-30T10:00:00Z")), + to: Some(at("2026-10-30T11:00:00Z")), + ..RangeQuery::default() + }; + assert_eq!(q.span(now).unwrap().to, at("2026-10-30T11:00:00Z")); + let q = RangeQuery { + from: Some(at("2026-10-30T10:00:00Z")), + ..RangeQuery::default() + }; + assert!(q.span(now).is_err(), "from alone"); + } +} diff --git a/crates/server/src/app.rs b/crates/server/src/app.rs index dac8076e..49c0e169 100644 --- a/crates/server/src/app.rs +++ b/crates/server/src/app.rs @@ -178,6 +178,7 @@ pub fn build_router(state: AppState, layer: PrincipalLayer) -> Router { let app = app.fallback_service(spa_service(&state.config.web)); let hsts = state.cookie_secure; + let recorder = state.traffic.recorder.clone(); let public_base_url = state.config.public_base_url(); let images = images.map(|p| p.implementation().clone()); let server = state.config.server.clone(); @@ -249,7 +250,10 @@ pub fn build_router(state: AppState, layer: PrincipalLayer) -> Router { // The request's OpenTelemetry span and metrics, outermost so they // time and count every answer, the ones the layers above give // themselves (a timeout) included. - .layer(axum::middleware::from_fn(crate::telemetry::track)) + .layer(axum::middleware::from_fn_with_state( + recorder, + crate::telemetry::track, + )) .with_state(state) } diff --git a/crates/server/src/images/dispatch.rs b/crates/server/src/images/dispatch.rs index d97aa34e..642a61bb 100644 --- a/crates/server/src/images/dispatch.rs +++ b/crates/server/src/images/dispatch.rs @@ -290,6 +290,7 @@ impl Registry { .refresh_principal(principal) .await? .ok_or_else(invalid)?; + crate::traffic::note_principal(&principal); Ok(Caller { principal, claims: Some(claims), @@ -379,6 +380,7 @@ impl Registry { /// grammar already. pub(crate) async fn resolve(&self, site: &Site, name: &str) -> ImgResult> { if let Some(repo) = &site.host_repo { + forge_traffic::note_repository(&repo.name); return Ok(Some(Target { repo: repo.clone(), image: name.to_string(), @@ -392,7 +394,11 @@ impl Registry { })); }; let repo = repository_dao::find_repository_by_name(&self.inner.pool, repo_name).await?; - Ok(repo.filter(|r| self.serves(r)).map(|repo| Target { + let repo = repo.filter(|r| self.serves(r)); + if let Some(repo) = &repo { + forge_traffic::note_repository(&repo.name); + } + Ok(repo.map(|repo| Target { repo, image: image.to_string(), name: name.to_string(), diff --git a/crates/server/src/lib.rs b/crates/server/src/lib.rs index 04ece27e..8901b2ef 100644 --- a/crates/server/src/lib.rs +++ b/crates/server/src/lib.rs @@ -38,5 +38,6 @@ pub mod registry; pub mod serve; pub mod state; pub mod telemetry; +pub mod traffic; pub mod web_ui; pub mod workers; diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs index e641369a..ada10c33 100644 --- a/crates/server/src/main.rs +++ b/crates/server/src/main.rs @@ -181,6 +181,7 @@ async fn main() -> anyhow::Result<()> { let workers = workers::spawn_all(&state); let downloads = state.downloads.clone(); + let (traffic, pool) = (state.traffic.clone(), state.pool.clone()); let app = build_router(state, layer); let listener = tokio::net::TcpListener::bind(bind_addr) @@ -196,6 +197,8 @@ async fn main() -> anyhow::Result<()> { workers.abort(); // Download counts not written yet would be lost with the process. downloads.write_pending().await; + // So would the traffic counted since the last minute's flush. + forge_server::traffic::flush_on_shutdown(&traffic, &pool).await; tracing::info!("forge-server stopped"); // Export what the telemetry still buffers. tokio::task::spawn_blocking(move || telemetry.shutdown()) diff --git a/crates/server/src/middleware/principal.rs b/crates/server/src/middleware/principal.rs index b7f5d00c..39b4e99b 100644 --- a/crates/server/src/middleware/principal.rs +++ b/crates/server/src/middleware/principal.rs @@ -278,6 +278,9 @@ pub async fn attach_principal( next: Next, ) -> Response { let peer = peer_ip(&req); + let xff = forwarded_for(req.headers()); + let client_ip = resolve_client_ip(peer, xff.as_deref(), &layer.trusted_proxies); + crate::traffic::note_client(client_ip); if !req.method().is_safe() && cross_site_browser_request(req.headers(), layer.public_origin.as_deref()) @@ -296,9 +299,6 @@ pub async fn attach_principal( return next.run(req).await; } - let xff = forwarded_for(req.headers()); - let client_ip = resolve_client_ip(peer, xff.as_deref(), &layer.trusted_proxies); - let anon = match &layer.anon { AnonSource::Fixed(policy) => *policy, AnonSource::Settings(cache) => cache.current().await.anon, @@ -342,6 +342,7 @@ pub async fn attach_principal( .. } ); + crate::traffic::note_principal(&resolved.principal); req.extensions_mut().insert(resolved.principal); let mut resp = next.run(req).await; if named_by_headers { diff --git a/crates/server/src/plugins.rs b/crates/server/src/plugins.rs index 5fe19579..ae893403 100644 --- a/crates/server/src/plugins.rs +++ b/crates/server/src/plugins.rs @@ -266,12 +266,16 @@ async fn dispatch_request(state: AppState, req: Request) -> Response { Ok(principal) => principal, Err(response) => return response, }; + crate::traffic::note_principal(&principal); let Some((repo_name, rest)) = split_path(uri.path()) else { return (StatusCode::NOT_FOUND, "repository not found").into_response(); }; let repo = match repository_dao::find_repository_by_name(&state.pool, repo_name).await { - Ok(Some(r)) => r, + Ok(Some(r)) => { + forge_traffic::note_repository(&r.name); + r + } // As for the NuGet feed: an unknown name answers like a repository // the caller may not read, except to a global admin. Ok(None) if state.authz.is_global_admin(&principal).await => { diff --git a/crates/server/src/state.rs b/crates/server/src/state.rs index f771a682..c838cea8 100644 --- a/crates/server/src/state.rs +++ b/crates/server/src/state.rs @@ -101,6 +101,9 @@ pub struct AppState { /// This process among the instances of an install, as per-instance /// reports (the local blob cache's statistics) name it. pub instance: Arc, + /// The traffic statistics (Admin → Traffic): this instance's counts, + /// written to the object store by the traffic worker. + pub traffic: forge_traffic::Traffic, /// Forge's own OIDC issuer: the keys a `federated_credential` OIDC realm /// signs its client assertions with, published under `/.well-known/`. pub assertion_issuer: AssertionIssuer, @@ -349,6 +352,11 @@ impl AppState { )); let (pool_for_mailer, config_for_mailer) = (pool.clone(), config.clone()); + let instance = Arc::new(Instance::current()); + let traffic = forge_traffic::Traffic::new( + Arc::new(blob_store.clone()), + forge_traffic::instance_id(instance.host.as_deref(), instance.started_at), + ); AppState { downloads: DownloadTally::new(pool.clone()), pool, @@ -375,7 +383,8 @@ impl AppState { rand::rng().fill_bytes(&mut key); key }]), - instance: Arc::new(Instance::current()), + instance, + traffic, mailer: Arc::new(crate::email::Mailer::new( pool_for_mailer, envelope.clone(), diff --git a/crates/server/src/telemetry.rs b/crates/server/src/telemetry.rs index 12a2f061..7990ca13 100644 --- a/crates/server/src/telemetry.rs +++ b/crates/server/src/telemetry.rs @@ -13,7 +13,9 @@ //! every request (joining the caller's trace from its `traceparent`) and //! the HTTP metrics: request duration by surface (from the route, never //! the raw path), method and status, the requests in flight and the -//! response body sizes. +//! response body sizes. It also counts the request in the traffic +//! statistics ([`crate::traffic`]), which the metrics' bounded attributes +//! leave out: the repository, the client address, the path. //! * [`install_live_source`]: what the SDK reads from the process when it //! collects (the connection pool, the blob cache's counters, those of `GET //! /admin/blob-cache`, the circuit breaker, the staging space, the @@ -27,18 +29,20 @@ use std::borrow::Cow; use std::pin::Pin; +use std::sync::Arc; use std::task::{Context, Poll}; use std::time::{Duration, Instant}; use anyhow::Context as _; use axum::body::{Body, Bytes}; -use axum::extract::{MatchedPath, Request}; +use axum::extract::{MatchedPath, Request, State}; use axum::http::{header, HeaderMap}; use axum::middleware::Next; use axum::response::Response; use forge_db::model::RepoKind; use forge_telemetry::http::Surface; use forge_telemetry::{inventory, live, storage}; +use forge_traffic::Recorder; use http_body::{Frame, SizeHint}; use opentelemetry::metrics::MeterProvider as _; use opentelemetry::propagation::Extractor; @@ -476,8 +480,9 @@ fn unsampled() -> opentelemetry::Context { } /// Middleware of the main listener: the request's server span and its -/// metrics, and the size of its response body. -pub async fn track(req: Request, next: Next) -> Response { +/// metrics, the size of its response body, and its count in the traffic +/// statistics ([`forge_traffic`]). +pub async fn track(State(recorder): State>, req: Request, next: Next) -> Response { let surface = surface_of(req.uri().path()); let method = forge_telemetry::http::method(req.method().as_str()); let route = req @@ -485,9 +490,29 @@ pub async fn track(req: Request, next: Next) -> Response { .get::() .map(|p| p.as_str().to_owned()); let span = request_span(&req, surface, route.as_deref()); + // Health probes are no traffic. + let counted = (surface != Surface::Health && recorder.enabled()).then(|| Counted { + path: if recorder.logs_requests() { + recorded_path(req.uri().path()).into_owned() + } else { + String::new() + }, + user_agent: req + .headers() + .get(header::USER_AGENT) + .and_then(|v| v.to_str().ok()) + .map(str::to_string), + slot: forge_traffic::Slot::new(), + recorder, + elapsed: Duration::ZERO, + }); let started = Instant::now(); let active = forge_telemetry::http::Active::start(method); - let mut response = next.run(req).instrument(span.clone()).await; + let run = next.run(req).instrument(span.clone()); + let mut response = match &counted { + Some(counted) => counted.slot.scope(run).await, + None => run.await, + }; drop(active); let surface = response .extensions_mut() @@ -499,7 +524,8 @@ pub async fn track(req: Request, next: Next) -> Response { if status >= 500 { span.record("otel.status_code", "ERROR"); } - forge_telemetry::http::request(surface, method, status, started.elapsed()); + let elapsed = started.elapsed(); + forge_telemetry::http::request(surface, method, status, elapsed); response.map(|inner| { Body::new(CountingBody { inner, @@ -507,11 +533,24 @@ pub async fn track(req: Request, next: Next) -> Response { method, status, bytes: 0, + counted: counted.map(|c| Counted { elapsed, ..c }), _span: span, }) }) } +/// A request to count in the traffic statistics once its body is sent. +struct Counted { + recorder: Arc, + /// What the layers inside noted. + slot: forge_traffic::Slot, + /// For the request log; empty when it is off. + path: String, + user_agent: Option, + /// Time to the response headers. + elapsed: Duration, +} + /// A response body that counts the bytes it yields: their total is /// recorded, and the request's span ends, when the body is done or /// dropped (a client that went away). @@ -521,6 +560,7 @@ struct CountingBody { method: &'static str, status: u16, bytes: u64, + counted: Option, _span: tracing::Span, } @@ -554,6 +594,19 @@ impl http_body::Body for CountingBody { impl Drop for CountingBody { fn drop(&mut self) { forge_telemetry::http::response_size(self.surface, self.method, self.status, self.bytes); + if let Some(counted) = self.counted.take() { + counted.recorder.record(forge_traffic::Event { + at: chrono::Utc::now(), + surface: self.surface.as_str(), + method: self.method, + status: self.status, + duration: counted.elapsed, + bytes: self.bytes, + path: counted.path, + user_agent: counted.user_agent, + noted: counted.slot.take(), + }); + } } } diff --git a/crates/server/src/traffic.rs b/crates/server/src/traffic.rs new file mode 100644 index 00000000..de2a000c --- /dev/null +++ b/crates/server/src/traffic.rs @@ -0,0 +1,233 @@ +//! The traffic statistics of Admin → Traffic ([`forge_traffic`]): what the +//! server notes about a request for its count, the settings, and the worker +//! that writes the files and deletes the expired ones. +//! +//! * The client address is the one the principal layer resolves (the socket +//! peer, or what a trusted proxy forwarded), counted under its network as +//! the login lockout counts it: an IPv6 address by its /64 +//! ([`client_network`]). +//! * The settings ([`TrafficSettings`]) are kept in `forge_setting` under +//! [`SETTINGS_KEY`]; every instance follows a change with its next flush. +//! * The worker flushes once a minute, a couple of seconds after the minute +//! turns, and prunes about once an hour, at a random offset per instance. + +use std::net::IpAddr; +use std::time::Duration; + +use chrono::{DateTime, Utc}; +use forge_core::auth::login_throttle::client_network; +use forge_core::auth::principal::{AuthMethod, Principal}; +use forge_traffic::{Account, Traffic, TrafficSettings}; +use sqlx::PgPool; +use tokio::task::JoinHandle; +use tracing::Instrument; + +use crate::state::AppState; + +/// The `forge_setting` key of the [`TrafficSettings`]. +pub const SETTINGS_KEY: &str = "traffic"; + +/// The worker's name (`forge.worker`). +pub const WORKER: &str = "traffic"; + +/// How often the worker prunes. +const PRUNE_INTERVAL: Duration = Duration::from_secs(3_600); + +/// How long the last flush on shutdown may take. +const SHUTDOWN_FLUSH: Duration = Duration::from_secs(10); + +/// The network `ip` is counted under, as text: the address itself for IPv4, +/// `/64` for IPv6. +pub fn network_label(ip: IpAddr) -> String { + match client_network(ip) { + IpAddr::V6(prefix) => format!("{prefix}/64"), + v4 => v4.to_string(), + } +} + +/// Note the request's client address. +pub fn note_client(ip: IpAddr) { + forge_traffic::note_client(ip.to_canonical(), network_label(ip)); +} + +/// The account `principal` is, as the statistics name it. +pub fn account_of(principal: &Principal) -> Account { + match principal { + Principal::Unauthenticated | Principal::Anonymous { .. } => Account { + kind: "anonymous", + user_id: None, + }, + Principal::User { + user_id, + token_id, + method, + } => Account { + kind: match (token_id, method) { + (Some(_), _) | (None, AuthMethod::Token) => "token", + (None, AuthMethod::Session) => "session", + (None, AuthMethod::TrustedHeader) => "trusted_header", + (None, AuthMethod::Password) => "password", + }, + user_id: Some(*user_id), + }, + Principal::Workload { .. } => Account { + kind: "workload", + user_id: None, + }, + } +} + +/// Note the account the request runs as. +pub fn note_principal(principal: &Principal) { + forge_traffic::note_account(account_of(principal)); +} + +/// The stored [`TrafficSettings`], or the defaults when never saved. +pub async fn read_settings(pool: &PgPool) -> sqlx::Result { + match forge_db::authz_dao::get_setting(pool, SETTINGS_KEY).await? { + None => Ok(TrafficSettings::default()), + Some(raw) => serde_json::from_str(&raw).map_err(|e| sqlx::Error::Decode(Box::new(e))), + } +} + +/// Store the [`TrafficSettings`] for every instance. +pub async fn write_settings(pool: &PgPool, settings: &TrafficSettings) -> sqlx::Result<()> { + let json = serde_json::to_string(settings).map_err(|e| sqlx::Error::Encode(Box::new(e)))?; + forge_db::authz_dao::set_setting(pool, SETTINGS_KEY, &json).await +} + +/// The stored settings, applied to this instance's recorder; the ones +/// applied before when they cannot be read. +async fn follow_settings( + pool: &PgPool, + traffic: &Traffic, + last: &TrafficSettings, +) -> TrafficSettings { + match read_settings(pool).await { + Ok(settings) => { + traffic.recorder.apply(&settings); + settings + } + Err(e) => { + forge_db::observe_error(&e); + tracing::warn!(error = %e, "cannot read the traffic settings; keeping the ones in force"); + last.clone() + } + } +} + +/// How long until `now`'s next minute is two seconds old. +fn until_next_flush(now: DateTime) -> Duration { + let into_minute = + now.timestamp().rem_euclid(60) * 1_000 + i64::from(now.timestamp_subsec_millis()); + Duration::from_millis(u64::try_from(62_000 - into_minute).unwrap_or(0)) +} + +/// The worker: flush every minute, prune every hour. +pub fn spawn_worker(state: AppState) -> JoinHandle<()> { + tokio::spawn(async move { + let traffic = state.traffic.clone(); + let mut settings = + follow_settings(&state.pool, &traffic, &TrafficSettings::default()).await; + // Instances started together prune at different times. + let mut next_prune = tokio::time::Instant::now() + + Duration::from_secs(rand::random_range(60..PRUNE_INTERVAL.as_secs())); + loop { + tokio::time::sleep(until_next_flush(Utc::now())).await; + let run = forge_telemetry::workers::Run::start(WORKER); + let ok = async { + settings = follow_settings(&state.pool, &traffic, &settings).await; + let report = traffic.flush(&settings, Utc::now()).await; + let mut ok = report.failures == 0; + if tokio::time::Instant::now() >= next_prune { + next_prune = tokio::time::Instant::now() + PRUNE_INTERVAL; + match forge_traffic::layout::prune(&*traffic.objects, &settings, Utc::now()) + .await + { + Ok(0) => {} + Ok(deleted) => { + tracing::info!(deleted, "deleted expired traffic statistics"); + } + Err(e) => { + ok = false; + tracing::warn!(error = %e, "cannot delete expired traffic statistics"); + } + } + } + ok + } + .instrument(run.span()) + .await; + run.finish(ok); + } + }) +} + +/// Write what this instance counted and has not written yet, before it +/// exits: at most [`SHUTDOWN_FLUSH`]. +pub async fn flush_on_shutdown(traffic: &Traffic, pool: &PgPool) { + let flush = async { + let settings = read_settings(pool).await.unwrap_or_default(); + traffic.flush_all(&settings).await + }; + match tokio::time::timeout(SHUTDOWN_FLUSH, flush).await { + Ok(report) if report.failures > 0 => { + tracing::warn!( + ?report, + "could not write all the traffic statistics on shutdown" + ); + } + Ok(_) => {} + Err(_) => tracing::warn!("writing the traffic statistics on shutdown timed out"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ipv6_clients_are_counted_by_their_64() { + assert_eq!(network_label("192.0.2.7".parse().unwrap()), "192.0.2.7"); + assert_eq!( + network_label("::ffff:192.0.2.7".parse().unwrap()), + "192.0.2.7" + ); + assert_eq!( + network_label("2001:db8:1:2:3:4:5:6".parse().unwrap()), + "2001:db8:1:2::/64" + ); + } + + #[test] + fn the_flush_waits_for_the_minute_to_turn() { + let at = |s: &str| DateTime::parse_from_rfc3339(s).unwrap().to_utc(); + assert_eq!( + until_next_flush(at("2026-10-30T14:00:00Z")), + Duration::from_secs(62) + ); + assert_eq!( + until_next_flush(at("2026-10-30T14:00:59.500Z")), + Duration::from_millis(2_500) + ); + } + + #[test] + fn accounts_name_how_the_caller_signed_in() { + let user = |token_id, method| Principal::User { + user_id: 9, + token_id, + method, + }; + assert_eq!( + account_of(&user(Some(1), AuthMethod::Password)).kind, + "token" + ); + assert_eq!(account_of(&user(None, AuthMethod::Session)).kind, "session"); + assert_eq!( + account_of(&user(None, AuthMethod::Session)).user_id, + Some(9) + ); + assert_eq!(account_of(&Principal::Unauthenticated).kind, "anonymous"); + } +} diff --git a/crates/server/src/workers.rs b/crates/server/src/workers.rs index c7e1698c..7bfee1bb 100644 --- a/crates/server/src/workers.rs +++ b/crates/server/src/workers.rs @@ -60,6 +60,11 @@ //! bytes, and the outbox backlog for the metrics //! ([`crate::telemetry::spawn_inventory`]). Read-only, per instance. //! +//! 10. **Traffic statistics** — writes this instance's request counts, top +//! clients and request log to the object store every minute, and deletes +//! the files past their retention every hour ([`crate::traffic`]). Each +//! instance writes only files of its own; deleting is idempotent. +//! //! Every loop reports to the metrics (`forge.worker.*`): its restarts, and //! for the periodic ones each pass's outcome and duration, the time of the //! last successful one and the interval it runs at. Each pass is a trace of @@ -121,6 +126,8 @@ pub struct WorkerHandles { /// Counts the inventory for the metrics. Present only with the metrics /// exported (`telemetry.metrics`). pub metrics_inventory: Option>, + /// Writes and prunes the traffic statistics. + pub traffic: JoinHandle<()>, } impl WorkerHandles { @@ -147,6 +154,7 @@ impl WorkerHandles { if let Some(h) = &self.metrics_inventory { h.abort(); } + self.traffic.abort(); } } @@ -337,6 +345,15 @@ pub fn spawn_all(state: &AppState) -> WorkerHandles { }) }); + // The traffic statistics: every instance writes its own files. + let traffic = { + let state = state.clone(); + forge_telemetry::workers::set_interval(crate::traffic::WORKER, Duration::from_secs(60)); + supervise(crate::traffic::WORKER, move || { + crate::traffic::spawn_worker(state.clone()) + }) + }; + WorkerHandles { outbox, reconciler, @@ -351,6 +368,7 @@ pub fn spawn_all(state: &AppState) -> WorkerHandles { blob_cache, nexus_import, metrics_inventory, + traffic, } } diff --git a/crates/server/tests/traffic.rs b/crates/server/tests/traffic.rs new file mode 100644 index 00000000..e63a953d --- /dev/null +++ b/crates/server/tests/traffic.rs @@ -0,0 +1,238 @@ +//! E2E: the traffic statistics (Admin → Traffic). Requires Docker +//! (Postgres + versitygw via the shared harness), or +//! `FORGE_TEST_DATABASE_URL` / `FORGE_TEST_S3_ENDPOINT`. +//! +//! Requests are counted with what the layers know of them (the repository, +//! the client address, the account), written to the object store by a flush +//! and read back by the admin API: the overview, a client's view with its +//! request log, and the settings. + +mod common; + +use common::{unique, TestServer}; +use forge_traffic::TrafficSettings; +use reqwest::StatusCode; +use serde_json::{json, Value}; + +async fn server() -> TestServer { + common::start_with( + common::fresh_pool().await, + common::ServerOpts { + bootstrap_admin: Some((format!("admin{}@example.com", unique()), "adminpw".into())), + ..Default::default() + }, + ) + .await +} + +async fn admin_get(srv: &TestServer, path: &str) -> (StatusCode, Value) { + let (u, p) = srv.admin_basic(); + let resp = srv + .client() + .get(format!("{}{path}", srv.base_url)) + .basic_auth(&u, Some(&p)) + .send() + .await + .unwrap(); + let status = resp.status(); + let text = resp.text().await.unwrap(); + ( + status, + serde_json::from_str(&text).unwrap_or(Value::String(text)), + ) +} + +async fn put_settings(srv: &TestServer, body: Value) -> (StatusCode, Value) { + let (u, p) = srv.admin_basic(); + let resp = srv + .client() + .put(format!("{}/admin/settings/traffic", srv.base_url)) + .basic_auth(&u, Some(&p)) + .json(&body) + .send() + .await + .unwrap(); + let status = resp.status(); + let text = resp.text().await.unwrap(); + ( + status, + serde_json::from_str(&text).unwrap_or(Value::String(text)), + ) +} + +#[tokio::test] +async fn requests_are_counted_written_and_read_back() { + let srv = server().await; + let (u, p) = srv.admin_basic(); + let client = srv.client(); + + let name = format!("traffic{}", unique()); + let resp = client + .post(format!("{}/admin/repositories", srv.base_url)) + .basic_auth(&u, Some(&p)) + .json(&json!({"name": name, "kind": "hosted", "format": "raw"})) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::CREATED, + "{}", + resp.text().await.unwrap() + ); + + // Two answers from the repository, one refused, a health probe that is + // not counted, and a repository that does not exist (its name is not + // recorded). + let file = format!("{}/repository/{name}/dir/hello.txt", srv.base_url); + let resp = client + .put(&file) + .basic_auth(&u, Some(&p)) + .header("user-agent", "curl/8.10") + .body("hello") + .send() + .await + .unwrap(); + assert!(resp.status().is_success(), "upload: {}", resp.status()); + let resp = client + .get(&file) + .basic_auth(&u, Some(&p)) + .header("user-agent", "curl/8.10") + .send() + .await + .unwrap(); + assert_eq!(resp.text().await.unwrap(), "hello"); + let resp = client.get(&file).send().await.unwrap(); + assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); + client + .get(format!("{}/healthz", srv.base_url)) + .send() + .await + .unwrap(); + client + .get(format!( + "{}/repository/no-such-{}/x", + srv.base_url, + unique() + )) + .send() + .await + .unwrap(); + + // What a minute's flush writes, written now. + let traffic = &srv.state.traffic; + let report = traffic.flush_all(&TrafficSettings::default()).await; + assert_eq!(report.failures, 0, "{report:?}"); + assert!(report.log_rows >= 5, "{report:?}"); + let instance = traffic.instance.to_string(); + + let (status, overview) = admin_get( + &srv, + &format!("/admin/traffic?range=1h&instance={instance}"), + ) + .await; + assert_eq!(status, StatusCode::OK, "{overview}"); + assert_eq!(overview["resolution"], "minute"); + assert!( + overview["instances"] + .as_array() + .unwrap() + .iter() + .any(|i| i["id"] == instance.as_str()), + "{overview}" + ); + let repositories = overview["repositories"].as_array().unwrap(); + assert_eq!( + repositories.len(), + 1, + "only the repository that exists: {overview}" + ); + assert_eq!(repositories[0]["name"], name.as_str()); + assert_eq!(repositories[0]["requests"], 3); + assert_eq!(repositories[0]["client_errors"], 1); + let surfaces: Vec<&str> = overview["surfaces"] + .as_array() + .unwrap() + .iter() + .map(|s| s["surface"].as_str().unwrap()) + .collect(); + assert!(surfaces.contains(&"plugin_raw"), "{surfaces:?}"); + assert!(!surfaces.contains(&"health"), "{surfaces:?}"); + assert!(overview["totals"]["denied"].as_u64().unwrap() >= 1); + let client_row = &overview["clients"][0]; + assert_eq!(client_row["network"], "127.0.0.1"); + assert_eq!(overview["recording"]["enabled"], true); + + // Only this repository's counts. + let (_, only) = admin_get( + &srv, + &format!("/admin/traffic?range=1h&instance={instance}&repository={name}"), + ) + .await; + assert_eq!(only["totals"]["requests"], 3); + + // The client's view: its requests in the request log, newest first. + let (status, detail) = admin_get( + &srv, + &format!("/admin/traffic/client?network=127.0.0.1&range=1h&instance={instance}"), + ) + .await; + assert_eq!(status, StatusCode::OK, "{detail}"); + assert!(detail["log"]["matched"].as_u64().unwrap() >= 5, "{detail}"); + let entries = detail["log"]["entries"].as_array().unwrap(); + let get = entries + .iter() + .find(|e| e["method"] == "GET" && e["status"] == 200 && e["repository"] == name.as_str()) + .expect("the download is in the log"); + assert_eq!(get["path"], format!("/repository/{name}/dir/hello.txt")); + assert_eq!(get["user_agent"], "curl/8.10"); + assert_eq!(get["bytes"], 5); + assert_eq!(get["account"], "token"); + assert_eq!(get["username"], u.as_str()); + + let (status, _) = admin_get(&srv, "/admin/traffic/client?range=1h").await; + assert_eq!(status, StatusCode::BAD_REQUEST, "network is required"); + let (status, _) = admin_get(&srv, "/admin/traffic?range=2h").await; + assert_eq!(status, StatusCode::BAD_REQUEST); + + // Admins only. + let resp = client + .get(format!("{}/admin/traffic", srv.base_url)) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); +} + +#[tokio::test] +async fn settings_switch_parts_off_and_are_checked() { + let srv = server().await; + let (status, settings) = admin_get(&srv, "/admin/settings/traffic").await; + assert_eq!(status, StatusCode::OK); + assert_eq!( + serde_json::from_value::(settings).unwrap(), + TrafficSettings::default() + ); + + let (status, settings) = put_settings( + &srv, + json!({"request_log": false, "client_retention_days": 7}), + ) + .await; + assert_eq!(status, StatusCode::OK, "{settings}"); + assert_eq!(settings["request_log"], false); + assert_eq!(settings["client_retention_days"], 7); + assert_eq!(settings["client_addresses"], true, "untouched"); + assert!( + !srv.state.traffic.recorder.logs_requests(), + "applied at once" + ); + + let (status, _) = put_settings(&srv, json!({"request_log_retention_days": 0})).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + let (status, _) = put_settings(&srv, json!({"unknown": true})).await; + assert!(status.is_client_error()); + + let (_, settings) = admin_get(&srv, "/admin/settings/traffic").await; + assert_eq!(settings["request_log"], false, "stored"); +} diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index 4b9c4f96..efa14ae3 100644 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -31,4 +31,4 @@ pub use blob_cache::{BlobCache, BlobCacheConfig, BlobCacheStats, BlobStream, Cac pub use cas::{Cas, CasError, SizedStage, StageReader, StagedBlob, Staging, StoredBlob}; pub use credentials::{S3Credentials, WebIdentity}; pub use read::BlobRef; -pub use s3::{BlobStore, ConditionalWrites, S3Config, S3Error}; +pub use s3::{BlobStore, ConditionalWrites, ListedObject, Listing, S3Config, S3Error}; diff --git a/crates/storage/src/s3.rs b/crates/storage/src/s3.rs index b2e239c7..308e8b22 100644 --- a/crates/storage/src/s3.rs +++ b/crates/storage/src/s3.rs @@ -89,6 +89,23 @@ pub enum ConditionalWrites { Off, } +/// The answer of [`BlobStore::list`]. +#[derive(Debug, Clone, Default)] +pub struct Listing { + pub objects: Vec, + /// The common prefixes, each ending with the delimiter. + pub prefixes: Vec, +} + +/// An object [`BlobStore::list`] found. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ListedObject { + pub key: String, + pub size: u64, + /// Changes whenever the object is replaced. + pub etag: Option, +} + /// Connection + behavior configuration for [`BlobStore`]. `Debug` omits the /// secret key. #[derive(Debug, Clone)] @@ -667,6 +684,94 @@ impl BlobStore { Ok(out.body) } + /// Store `body` at `key`, replacing what is there: for objects Forge + /// rewrites (the traffic statistics), never for content-addressed blobs + /// ([`Self::put_if_absent`]). + pub async fn put_bytes(&self, key: &str, body: Vec) -> Result<(), S3Error> { + self.admit(S3Op::Put)?; + self.call( + S3Op::Put, + self.client + .put_object() + .bucket(&self.bucket) + .key(key) + .body(ByteStream::from(body)) + .send(), + ) + .await + .map(|_| ()) + } + + /// The whole object at `key`, or `None` when there is none. + pub async fn get_bytes(&self, key: &str) -> Result, S3Error> { + self.admit(S3Op::Get)?; + let call = S3Call::start(S3Op::Get, &self.bucket); + let result = self + .client + .get_object() + .bucket(&self.bucket) + .key(key) + .send() + .instrument(call.span()) + .await; + self.observe(call, &result); + let out = match result { + Ok(out) => out, + Err(e) if is_not_found(&e) => return Ok(None), + Err(e) => return Err(self.error(e)), + }; + let body = out + .body + .collect() + .await + .map_err(|e| S3Error::S3(format!("read {key}: {e}")))?; + Ok(Some(body.into_bytes())) + } + + /// The objects whose keys start with `prefix`. With `delimiter`, keys + /// that continue past the next delimiter after the prefix are listed + /// once each, as a common prefix (a "directory"), instead. + pub async fn list(&self, prefix: &str, delimiter: Option<&str>) -> Result { + let mut listing = Listing::default(); + let mut token: Option = None; + loop { + self.admit(S3Op::List)?; + let page = self + .call( + S3Op::List, + self.client + .list_objects_v2() + .bucket(&self.bucket) + .prefix(prefix) + .set_delimiter(delimiter.map(str::to_string)) + .set_continuation_token(token.take()) + .send(), + ) + .await?; + listing + .objects + .extend(page.contents().iter().filter_map(|object| { + Some(ListedObject { + key: object.key()?.to_string(), + size: object + .size() + .and_then(|s| u64::try_from(s).ok()) + .unwrap_or(0), + etag: object.e_tag().map(str::to_string), + }) + })); + listing.prefixes.extend( + page.common_prefixes() + .iter() + .filter_map(|p| p.prefix().map(str::to_string)), + ); + match page.next_continuation_token() { + Some(next) if page.is_truncated() == Some(true) => token = Some(next.to_string()), + _ => return Ok(listing), + } + } + } + /// Delete the object at `key`. Idempotent: a missing object is `Ok(())`. pub async fn delete(&self, key: &str) -> Result<(), S3Error> { self.admit(S3Op::Delete)?; diff --git a/crates/telemetry/src/lib.rs b/crates/telemetry/src/lib.rs index 76d0815f..9af9a6d3 100644 --- a/crates/telemetry/src/lib.rs +++ b/crates/telemetry/src/lib.rs @@ -641,6 +641,7 @@ pub mod storage { CompleteMultipart => "CompleteMultipartUpload", AbortMultipart => "AbortMultipartUpload", Get => "GetObject", + List => "ListObjectsV2", Delete => "DeleteObject", } } diff --git a/crates/traffic/Cargo.toml b/crates/traffic/Cargo.toml new file mode 100644 index 00000000..a86bc92b --- /dev/null +++ b/crates/traffic/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "forge-traffic" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +# The object store the statistics are kept in (BlobStore implements +# `Objects`). +forge-storage = { path = "../storage" } + +tokio = { workspace = true, features = ["sync", "time", "rt"] } +parquet = { workspace = true } +parquet_derive = { workspace = true } +bytes = "1" +chrono = { workspace = true, features = ["serde"] } +serde = { workspace = true } +anyhow = { workspace = true } +async-trait = { workspace = true } +futures-util = { workspace = true } +moka = { workspace = true } +tracing = { workspace = true } +rand = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["rt-multi-thread", "macros", "sync", "time"] } +serde_json = { workspace = true } diff --git a/crates/traffic/src/layout.rs b/crates/traffic/src/layout.rs new file mode 100644 index 00000000..b1383fc8 --- /dev/null +++ b/crates/traffic/src/layout.rs @@ -0,0 +1,693 @@ +//! Where the files are, what writes them and what deletes them. +//! +//! Every key is under `traffic/v1/` in Forge's bucket, by area, then date +//! and hour (UTC), then the writing instance ([`instance_id`]): +//! +//! * `minutes/{date}/{hour}/{instance}.buckets.parquet` and +//! `….clients.parquet`: one instance's counts and top clients of an hour, +//! by minute; rewritten every minute while the hour lasts. Kept +//! [`MINUTE_RETENTION_DAYS`] (the clients no longer than the clients' +//! retention). +//! * `hours/{date}/{instance}.buckets.parquet` and `….clients.parquet`: +//! the same of a day, by hour; rewritten every minute while the day lasts. +//! Kept as long as the settings say ([`TrafficSettings`]). +//! * `requests/{date}/{hour}/{instance}.{unix ms}.parquet`: the request +//! log's rows of one flush. +//! +//! Each instance writes only files named after itself, so two instances +//! never write the same object and a restarted process (a new name) never +//! overwrites what its predecessor wrote. + +use std::collections::BTreeMap; +use std::sync::Arc; + +use chrono::{DateTime, NaiveDate, TimeZone, Utc}; +use futures_util::{stream, StreamExt}; + +use crate::objects::Objects; +use crate::record::Drained; +use crate::rows::{encode, BucketKey, BucketRow, ClientRow, Counts}; +use crate::top::{ClientCounts, TopClients}; +use crate::TrafficSettings; + +/// The prefix of every key. +pub const ROOT: &str = "traffic/v1/"; +/// How long the counts by minute are kept: long enough for the views of the +/// last 48 hours. +pub const MINUTE_RETENTION_DAYS: u32 = 3; +/// The top clients a file keeps per time bucket. +pub const TOP_PER_BUCKET: usize = 100; +/// The networks an hour of a day file keeps track of before its top list +/// is cut to [`TOP_PER_BUCKET`]. +const HOUR_CLIENTS: usize = 4_096; +/// Deletes and reads in flight at once. +pub(crate) const CONCURRENCY: usize = 16; + +const HOUR: i64 = 3_600; +const DAY: i64 = 86_400; + +/// The three areas of files. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Area { + /// By minute, a file per instance and hour. + Minutes, + /// By hour, a file per instance and day. + Hours, + /// The request log. + Requests, +} + +impl Area { + /// The prefix of the area's keys. + pub fn prefix(self) -> &'static str { + match self { + Area::Minutes => "traffic/v1/minutes/", + Area::Hours => "traffic/v1/hours/", + Area::Requests => "traffic/v1/requests/", + } + } + + /// How long one of its files covers, in seconds. + pub fn span(self) -> i64 { + match self { + Area::Minutes | Area::Requests => HOUR, + Area::Hours => DAY, + } + } + + /// The prefix of its files of the day starting at `day` (Unix seconds). + pub fn day_prefix(self, day: i64) -> String { + format!("{}{}/", self.prefix(), date(day)) + } +} + +/// What a file holds. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FileKind { + Buckets, + Clients, + Requests, +} + +/// A file, as its key names it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FileKey { + pub area: Area, + pub kind: FileKind, + /// The start of the time it covers ([`Area::span`]), Unix seconds. + pub start: i64, + /// The instance that wrote it ([`instance_id`]). + pub instance: String, +} + +fn date(start: i64) -> String { + Utc.timestamp_opt(start, 0) + .single() + .map(|t| t.format("%Y-%m-%d").to_string()) + .unwrap_or_default() +} + +fn hour_of_day(start: i64) -> String { + format!("{:02}", start.rem_euclid(DAY) / HOUR) +} + +fn kind_suffix(kind: FileKind) -> &'static str { + match kind { + FileKind::Buckets => "buckets", + FileKind::Clients => "clients", + FileKind::Requests => "requests", + } +} + +/// The key of the counts file (`kind` buckets or clients) of `instance` +/// covering the hour (minutes area) or day (hours area) starting at +/// `start`. +pub fn counts_key(area: Area, start: i64, instance: &str, kind: FileKind) -> String { + let suffix = kind_suffix(kind); + match area { + Area::Minutes => format!( + "{}{}/{}/{instance}.{suffix}.parquet", + area.prefix(), + date(start), + hour_of_day(start) + ), + Area::Hours | Area::Requests => { + format!( + "{}{}/{instance}.{suffix}.parquet", + area.prefix(), + date(start) + ) + } + } +} + +/// The key of the request log file `instance` flushes at `at`. +pub fn log_key(at: DateTime, instance: &str) -> String { + let start = at.timestamp(); + format!( + "{}{}/{}/{instance}.{}.parquet", + Area::Requests.prefix(), + date(start), + hour_of_day(start), + at.timestamp_millis() + ) +} + +/// What `key`, a key of `area`, names; `None` for a key Forge did not +/// write. +pub fn parse_key(area: Area, key: &str) -> Option { + let rest = key.strip_prefix(area.prefix())?; + let mut parts = rest.split('/'); + let day = NaiveDate::parse_from_str(parts.next()?, "%Y-%m-%d").ok()?; + let mut start = day.and_hms_opt(0, 0, 0)?.and_utc().timestamp(); + if area != Area::Hours { + let hour: i64 = parts.next()?.parse().ok()?; + if !(0..24).contains(&hour) { + return None; + } + start += hour * HOUR; + } + let name = parts.next()?.strip_suffix(".parquet")?; + if parts.next().is_some() { + return None; + } + let (instance, suffix) = name.split_once('.')?; + let kind = match (area, suffix) { + (Area::Requests, millis) if millis.parse::().is_ok() => FileKind::Requests, + (Area::Minutes | Area::Hours, "buckets") => FileKind::Buckets, + (Area::Minutes | Area::Hours, "clients") => FileKind::Clients, + _ => return None, + }; + Some(FileKey { + area, + kind, + start, + instance: instance.to_string(), + }) +} + +/// This process's name in the files: its host name (`-` for anything a +/// host name does not hold), when it started and a random suffix, so that +/// two processes never share one. No `.` or `_` but the separators. +pub fn instance_id(host: Option<&str>, started_at: DateTime) -> String { + let host: String = host + .unwrap_or("forge") + .chars() + .map(|c| { + if c.is_ascii_alphanumeric() || c == '-' { + c + } else { + '-' + } + }) + .take(63) + .collect(); + let host = if host.is_empty() { + "forge".into() + } else { + host + }; + format!( + "{host}_{}_{:04x}", + started_at.format("%Y%m%dT%H%M%SZ"), + rand::random::() + ) +} + +/// The host name and start time an [`instance_id`] holds. +pub fn instance_parts(id: &str) -> (String, Option>) { + let mut parts = id.split('_'); + let host = parts.next().unwrap_or_default().to_string(); + let started = parts.next().and_then(|t| { + chrono::NaiveDateTime::parse_from_str(t, "%Y%m%dT%H%M%SZ") + .ok() + .map(|t| t.and_utc()) + }); + (host, started) +} + +/// One instance's counts of an hour (by minute) or a day (by hour), as +/// written to its files. +#[derive(Debug, Default)] +struct Period { + buckets: BTreeMap<(i64, BucketKey), Counts>, + clients: BTreeMap, + dirty: bool, +} + +impl Period { + fn bucket_rows(&self) -> Vec { + self.buckets + .iter() + .map(|((start, key), counts)| BucketRow::new(*start, key, counts)) + .collect() + } + + fn client_rows(&self) -> Vec { + let mut rows = Vec::new(); + for (start, top) in &self.clients { + for (network, c) in top.top(TOP_PER_BUCKET) { + rows.push(client_row(*start, network, c)); + } + } + rows + } +} + +fn client_row(start: i64, network: &str, c: &ClientCounts) -> ClientRow { + let signed = |n: u64| i64::try_from(n).unwrap_or(i64::MAX); + ClientRow { + start, + network: network.to_string(), + address: c.address.clone(), + requests: signed(c.requests), + bytes: signed(c.bytes), + client_errors: signed(c.client_errors), + denied: signed(c.denied), + not_found: signed(c.not_found), + throttled: signed(c.throttled), + server_errors: signed(c.server_errors), + user_agent: c.user_agent.clone(), + account: c.account.clone(), + user_id: c.user_id.unwrap_or(0), + } +} + +/// What a flush did. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct FlushReport { + /// Files written. + pub files: usize, + /// Request log rows written. + pub log_rows: usize, + /// Writes that failed (retried with the next flush, but for the log). + pub failures: usize, +} + +/// Writes one instance's files: the counts of the hours and days not over +/// yet, and the request log. +pub struct Flusher { + objects: Arc, + instance: Arc, + hours: BTreeMap, + days: BTreeMap, +} + +impl Flusher { + /// The flusher of `instance`, writing to `objects`. + pub fn new(objects: Arc, instance: Arc) -> Flusher { + Flusher { + objects, + instance, + hours: BTreeMap::new(), + days: BTreeMap::new(), + } + } + + /// Add `drained` to the hours and days it falls in and write the files + /// that changed, and the request log rows. A file that cannot be written + /// is written with the next flush; log rows that cannot be are dropped. + pub async fn flush( + &mut self, + drained: Drained, + settings: &TrafficSettings, + now: DateTime, + ) -> FlushReport { + let mut report = FlushReport::default(); + for minute in drained.minutes { + let hour = minute.start - minute.start.rem_euclid(HOUR); + let day = minute.start - minute.start.rem_euclid(DAY); + let in_hour = self.hours.entry(hour).or_default(); + let in_day = self.days.entry(day).or_default(); + for (key, counts) in &minute.buckets { + in_hour + .buckets + .entry((minute.start, key.clone())) + .or_default() + .add(counts); + in_day + .buckets + .entry((hour, key.clone())) + .or_default() + .add(counts); + } + if settings.client_addresses && !minute.clients.is_empty() { + let top = in_hour + .clients + .entry(minute.start) + .or_insert_with(|| TopClients::new(TOP_PER_BUCKET)); + top.merge(&minute.clients); + top.truncate(TOP_PER_BUCKET); + in_day + .clients + .entry(hour) + .or_insert_with(|| TopClients::new(HOUR_CLIENTS)) + .merge(&minute.clients); + } + in_hour.dirty = true; + in_day.dirty = true; + } + if !settings.client_addresses { + for period in self.hours.values_mut().chain(self.days.values_mut()) { + period.clients.clear(); + } + } + + let instance = self.instance.clone(); + for (area, periods) in [ + (Area::Minutes, &mut self.hours), + (Area::Hours, &mut self.days), + ] { + for (start, period) in periods.iter_mut().filter(|(_, p)| p.dirty) { + let mut ok = true; + let mut files = vec![(FileKind::Buckets, encode(&period.bucket_rows()))]; + if settings.client_addresses { + files.push((FileKind::Clients, encode(&period.client_rows()))); + } + for (kind, body) in files { + let key = counts_key(area, *start, &instance, kind); + match body { + Ok(body) => match self.objects.put(&key, body).await { + Ok(()) => report.files += 1, + Err(e) => { + tracing::warn!(error = %e, key, "cannot write the traffic statistics"); + ok = false; + } + }, + Err(e) => { + tracing::error!(error = %e, key, "cannot encode the traffic statistics"); + ok = false; + } + } + } + if ok { + period.dirty = false; + } else { + report.failures += 1; + } + } + } + + if drained.log_dropped > 0 { + tracing::warn!( + dropped = drained.log_dropped, + "the request log dropped rows: more requests than it holds between two flushes" + ); + } + if !drained.log.is_empty() { + let key = log_key(now, &instance); + let written = match encode(&drained.log) { + Ok(body) => self.objects.put(&key, body).await, + Err(e) => Err(e), + }; + match written { + Ok(()) => { + report.files += 1; + report.log_rows = drained.log.len(); + } + Err(e) => { + report.failures += 1; + tracing::warn!( + error = %e, + key, + rows = drained.log.len(), + "cannot write the request log; its rows are lost" + ); + } + } + } + + self.forget(now); + report + } + + /// Drop the periods that are over and written, and the ones that could + /// not be written for too long; cut the top lists of the hours that are + /// over. + fn forget(&mut self, now: DateTime) { + let now = now.timestamp(); + let this_hour = now - now.rem_euclid(HOUR); + let today = now - now.rem_euclid(DAY); + self.hours.retain(|start, period| { + *start >= this_hour || (period.dirty && *start >= this_hour - 3 * HOUR) + }); + self.days + .retain(|start, period| *start >= today || (period.dirty && *start >= today - DAY)); + for day in self.days.values_mut() { + for (hour, top) in day.clients.iter_mut() { + if *hour < this_hour { + top.truncate(TOP_PER_BUCKET); + } + } + } + } +} + +/// How long the files of `area` holding `kind` are kept, in seconds; +/// `None` to keep them. +pub fn retention(settings: &TrafficSettings, area: Area, kind: FileKind) -> Option { + let days = |d: u32| i64::from(d) * DAY; + match (area, kind) { + (Area::Minutes, FileKind::Clients) => Some(days( + MINUTE_RETENTION_DAYS.min(settings.client_retention_days), + )), + (Area::Minutes, _) => Some(days(MINUTE_RETENTION_DAYS)), + (Area::Hours, FileKind::Clients) => Some(days(settings.client_retention_days)), + (Area::Hours, _) => match settings.statistics_retention_days { + 0 => None, + d => Some(days(d)), + }, + (Area::Requests, _) => Some(days(settings.request_log_retention_days)), + } +} + +/// Delete the files past their retention ([`retention`]), whoever wrote +/// them. Idempotent and safe on every instance at once. Returns how many +/// files it deleted. +pub async fn prune( + objects: &dyn Objects, + settings: &TrafficSettings, + now: DateTime, +) -> anyhow::Result { + let now = now.timestamp(); + let mut expired = Vec::new(); + for area in [Area::Minutes, Area::Hours, Area::Requests] { + let kinds: &[FileKind] = match area { + Area::Requests => &[FileKind::Requests], + _ => &[FileKind::Buckets, FileKind::Clients], + }; + // A day can hold expired files once the shortest retention of the + // area has passed since it ended. + let Some(shortest) = kinds + .iter() + .filter_map(|kind| retention(settings, area, *kind)) + .min() + else { + continue; + }; + for dir in objects.list_dirs(area.prefix()).await? { + let Some(day) = dir + .strip_prefix(area.prefix()) + .and_then(|d| d.strip_suffix('/')) + .and_then(|d| NaiveDate::parse_from_str(d, "%Y-%m-%d").ok()) + .and_then(|d| d.and_hms_opt(0, 0, 0)) + .map(|d| d.and_utc().timestamp()) + else { + continue; + }; + if day + area.span() + shortest > now { + // Even its earliest file is within the shortest retention. + continue; + } + for listed in objects.list(&dir).await? { + let Some(file) = parse_key(area, &listed.key) else { + continue; + }; + if retention(settings, area, file.kind) + .is_some_and(|keep| file.start + area.span() + keep <= now) + { + expired.push(listed.key); + } + } + } + } + let deleted = expired.len(); + let failures: Vec = stream::iter(expired) + .map(|key| async move { objects.delete(&key).await }) + .buffer_unordered(CONCURRENCY) + .filter_map(|result| async move { result.err() }) + .collect() + .await; + match failures.into_iter().next() { + Some(e) => Err(e.context("cannot delete expired traffic statistics")), + None => Ok(deleted), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::objects::MemoryObjects; + use crate::record::{Event, Recorder}; + use crate::rows::decode; + use crate::slot::Noted; + use std::time::Duration; + + fn t(rfc3339: &str) -> DateTime { + DateTime::parse_from_rfc3339(rfc3339).unwrap().to_utc() + } + + #[test] + fn keys_name_what_they_hold() { + let start = t("2026-10-30T14:00:00Z").timestamp(); + let key = counts_key(Area::Minutes, start, "web-1_x_0001", FileKind::Clients); + assert_eq!( + key, + "traffic/v1/minutes/2026-10-30/14/web-1_x_0001.clients.parquet" + ); + let parsed = parse_key(Area::Minutes, &key).unwrap(); + assert_eq!( + (parsed.start, parsed.kind, parsed.instance.as_str()), + (start, FileKind::Clients, "web-1_x_0001") + ); + + let day = t("2026-10-30T00:00:00Z").timestamp(); + let key = counts_key(Area::Hours, day, "web-1", FileKind::Buckets); + assert_eq!(key, "traffic/v1/hours/2026-10-30/web-1.buckets.parquet"); + assert_eq!(parse_key(Area::Hours, &key).unwrap().start, day); + + let key = log_key(t("2026-10-30T14:05:01.250Z"), "web-1"); + assert_eq!( + key, + "traffic/v1/requests/2026-10-30/14/web-1.1793369101250.parquet" + ); + assert_eq!(parse_key(Area::Requests, &key).unwrap().start, start); + + assert!(parse_key( + Area::Minutes, + "traffic/v1/minutes/2026-10-30/24/a.buckets.parquet" + ) + .is_none()); + assert!(parse_key(Area::Hours, "traffic/v1/hours/2026-10-30/a.other.parquet").is_none()); + } + + #[test] + fn an_instance_id_holds_its_host_and_start() { + let started = t("2026-10-30T14:05:01Z"); + let id = instance_id(Some("forge-7d9f.pod_x"), started); + assert!(id.starts_with("forge-7d9f-pod-x_20261030T140501Z_"), "{id}"); + assert_eq!( + instance_parts(&id), + ("forge-7d9f-pod-x".to_string(), Some(started)) + ); + assert!(instance_id(None, started).starts_with("forge_")); + } + + fn event(at: DateTime, network: &str) -> Event { + Event { + at, + surface: "oci", + method: "GET", + status: 200, + duration: Duration::from_millis(3), + bytes: 10, + path: "/v2/".into(), + user_agent: None, + noted: Noted { + client: network.parse().ok(), + network: Some(network.into()), + account: None, + repository: Some("docker".into()), + }, + } + } + + #[tokio::test] + async fn a_flush_writes_the_hour_the_day_and_the_log() { + let objects = Arc::new(MemoryObjects::default()); + let mut flusher = Flusher::new(objects.clone(), "a".into()); + let settings = TrafficSettings::default(); + let recorder = Recorder::new(); + recorder.record(event(t("2026-10-30T14:00:10Z"), "192.0.2.1")); + recorder.record(event(t("2026-10-30T14:00:20Z"), "192.0.2.1")); + recorder.record(event(t("2026-10-30T14:01:20Z"), "192.0.2.2")); + let now = t("2026-10-30T14:02:00Z"); + let report = flusher.flush(recorder.drain(now), &settings, now).await; + assert_eq!( + report, + FlushReport { + files: 5, + log_rows: 3, + failures: 0 + } + ); + let keys = objects.keys(); + assert!(keys.contains(&"traffic/v1/hours/2026-10-30/a.buckets.parquet".to_string())); + + let rows: Vec = decode( + objects + .get("traffic/v1/minutes/2026-10-30/14/a.buckets.parquet") + .await + .unwrap() + .unwrap(), + ) + .unwrap(); + assert_eq!(rows.len(), 2, "one row per minute and key"); + assert_eq!(rows[0].requests, 2); + + // The next hour: the previous hour's files are not rewritten. + recorder.record(event(t("2026-10-30T15:00:05Z"), "192.0.2.1")); + let now = t("2026-10-30T15:01:00Z"); + let report = flusher.flush(recorder.drain(now), &settings, now).await; + assert_eq!( + report.files, 5, + "the new hour's two, the day's two, the log" + ); + assert_eq!(flusher.hours.len(), 1, "the hour that is over is forgotten"); + let day: Vec = decode( + objects + .get("traffic/v1/hours/2026-10-30/a.buckets.parquet") + .await + .unwrap() + .unwrap(), + ) + .unwrap(); + assert_eq!(day.len(), 2, "one row per hour"); + assert_eq!(day.iter().map(|r| r.requests).sum::(), 4); + } + + #[tokio::test] + async fn pruning_deletes_by_area_and_kind() { + let objects = MemoryObjects::default(); + let day = |d: &str| t(&format!("{d}T00:00:00Z")).timestamp(); + for key in [ + counts_key(Area::Minutes, day("2026-10-25"), "a", FileKind::Buckets), + counts_key(Area::Minutes, day("2026-10-29"), "a", FileKind::Buckets), + counts_key(Area::Hours, day("2025-01-01"), "a", FileKind::Buckets), + counts_key(Area::Hours, day("2026-09-01"), "a", FileKind::Buckets), + counts_key(Area::Hours, day("2026-09-01"), "a", FileKind::Clients), + log_key(t("2026-10-20T10:00:00Z"), "a"), + log_key(t("2026-10-29T10:00:00Z"), "a"), + "traffic/v1/hours/not-a-date/x.buckets.parquet".to_string(), + ] { + objects.put(&key, vec![1]).await.unwrap(); + } + let deleted = prune( + &objects, + &TrafficSettings::default(), + t("2026-10-30T12:00:00Z"), + ) + .await + .unwrap(); + assert_eq!(deleted, 4); + assert_eq!( + objects.keys(), + vec![ + "traffic/v1/hours/2026-09-01/a.buckets.parquet", + "traffic/v1/hours/not-a-date/x.buckets.parquet", + "traffic/v1/minutes/2026-10-29/00/a.buckets.parquet", + "traffic/v1/requests/2026-10-29/10/a.1793268000000.parquet", + ] + ); + } +} diff --git a/crates/traffic/src/lib.rs b/crates/traffic/src/lib.rs new file mode 100644 index 00000000..6e7d547d --- /dev/null +++ b/crates/traffic/src/lib.rs @@ -0,0 +1,104 @@ +//! Forge's traffic statistics: what every instance answered, counted in time +//! buckets and kept as Parquet files in the object store, so that any +//! instance can show the traffic of all of them (Admin → Traffic). +//! +//! * **Recording** ([`Recorder`]) is a counter update in memory, behind one +//! short lock: no I/O on the request path. A request is counted in its +//! minute by surface, repository, method and status (requests, response +//! bytes, a latency histogram), its client address in that minute's top +//! list ([`TopClients`], bounded whatever the number of addresses), and, +//! with the request log on, as a row of the log. +//! * **What a request is** comes from the layers that know it: the server's +//! outermost middleware opens a [`Slot`] around the request, and the +//! principal layer ([`note_client`], [`note_account`]) and the handlers +//! that resolve a repository ([`note_repository`]) fill it in. +//! * **Flushing** ([`Flusher`]) runs once a minute on every instance. Each +//! instance writes files of its own ([`layout`]), named after the process +//! ([`instance_id`]), so instances never write the same object: +//! * per hour, its buckets and top clients by minute, rewritten every +//! minute while the hour lasts; +//! * per day, the same by hour; +//! * every minute, the rows of the request log since the last flush. +//! * **Reading** ([`query`]) lists the files of a range in the object store, +//! reads them (closed files are cached) and sums them: the view over all +//! instances needs no call between them. +//! * **Pruning** ([`layout::prune`]) deletes the files past their retention +//! ([`TrafficSettings`]); it is idempotent and runs on every instance. +//! +//! Client addresses are personal data: they are kept apart from the counts +//! (the `clients` files and the request log), with a retention of their own, +//! and can be switched off. + +pub mod layout; +pub mod objects; +pub mod query; +mod record; +mod rows; +mod settings; +mod slot; +mod top; + +use std::sync::Arc; + +pub use layout::{instance_id, FlushReport, Flusher}; +pub use objects::{Listed, MemoryObjects, Objects}; +pub use record::{Drained, Event, Minute, Recorder}; +pub use rows::{BucketKey, Counts, LATENCY_BOUNDS_MS}; +pub use settings::{TrafficSettings, MAX_RETENTION_DAYS}; +pub use slot::{note_account, note_client, note_repository, Account, Noted, Slot}; +pub use top::{ClientCounts, TopClients}; + +/// What the server holds of the traffic statistics: this instance's +/// [`Recorder`] and [`Flusher`], the object store the files go to, and the +/// cache of the files read back. +#[derive(Clone)] +pub struct Traffic { + /// Counts this instance's requests. + pub recorder: Arc, + /// Where the files are written and read. + pub objects: Arc, + /// This process's name in the files ([`instance_id`]). + pub instance: Arc, + /// Files read back, decoded ([`query`]). + pub cache: query::FileCache, + /// Writes this instance's files. Shared, so that it outlives a restart + /// of the worker and the last flush on shutdown adds to what the worker + /// wrote instead of replacing it. + flusher: Arc>, +} + +impl Traffic { + /// The statistics of the instance `instance`, kept in `objects`. + pub fn new(objects: Arc, instance: String) -> Traffic { + let instance: Arc = instance.into(); + Traffic { + recorder: Arc::new(Recorder::new()), + flusher: Arc::new(tokio::sync::Mutex::new(Flusher::new( + objects.clone(), + instance.clone(), + ))), + objects, + instance, + cache: query::FileCache::new(), + } + } + + /// Write the minutes that ended before `now`, and the request log. + pub async fn flush( + &self, + settings: &TrafficSettings, + now: chrono::DateTime, + ) -> FlushReport { + let mut flusher = self.flusher.lock().await; + flusher.flush(self.recorder.drain(now), settings, now).await + } + + /// Write everything counted, the minute still being counted included + /// (on shutdown). + pub async fn flush_all(&self, settings: &TrafficSettings) -> FlushReport { + let mut flusher = self.flusher.lock().await; + flusher + .flush(self.recorder.drain_all(), settings, chrono::Utc::now()) + .await + } +} diff --git a/crates/traffic/src/objects.rs b/crates/traffic/src/objects.rs new file mode 100644 index 00000000..b7ab713b --- /dev/null +++ b/crates/traffic/src/objects.rs @@ -0,0 +1,127 @@ +//! The object store the files live in: Forge's S3 bucket +//! ([`forge_storage::BlobStore`]), or memory in tests ([`MemoryObjects`]). + +use std::collections::BTreeMap; +use std::sync::{Mutex, PoisonError}; + +use bytes::Bytes; +use forge_storage::BlobStore; + +/// An object a listing found. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Listed { + pub key: String, + pub size: u64, + /// Changes whenever the object is replaced (the cache key of a file). + pub etag: Option, +} + +/// What the statistics need of an object store. +#[async_trait::async_trait] +pub trait Objects: Send + Sync + 'static { + /// Store `body` at `key`, replacing what is there. + async fn put(&self, key: &str, body: Vec) -> anyhow::Result<()>; + /// The object at `key`, or `None` when there is none (any more). + async fn get(&self, key: &str) -> anyhow::Result>; + /// Every object whose key starts with `prefix`. + async fn list(&self, prefix: &str) -> anyhow::Result>; + /// The "directories" right under `prefix` (which ends with `/`): the + /// distinct key prefixes up to the next `/`, each ending with it. + async fn list_dirs(&self, prefix: &str) -> anyhow::Result>; + /// Delete the object at `key`; a missing one is no error. + async fn delete(&self, key: &str) -> anyhow::Result<()>; +} + +#[async_trait::async_trait] +impl Objects for BlobStore { + async fn put(&self, key: &str, body: Vec) -> anyhow::Result<()> { + Ok(self.put_bytes(key, body).await?) + } + + async fn get(&self, key: &str) -> anyhow::Result> { + Ok(self.get_bytes(key).await?) + } + + async fn list(&self, prefix: &str) -> anyhow::Result> { + Ok(BlobStore::list(self, prefix, None) + .await? + .objects + .into_iter() + .map(|o| Listed { + key: o.key, + size: o.size, + etag: o.etag, + }) + .collect()) + } + + async fn list_dirs(&self, prefix: &str) -> anyhow::Result> { + Ok(BlobStore::list(self, prefix, Some("/")).await?.prefixes) + } + + async fn delete(&self, key: &str) -> anyhow::Result<()> { + Ok(BlobStore::delete(self, key).await?) + } +} + +/// An object store in memory, for tests. +#[derive(Debug, Default)] +pub struct MemoryObjects { + objects: Mutex>, +} + +impl MemoryObjects { + fn lock(&self) -> std::sync::MutexGuard<'_, BTreeMap> { + self.objects.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// The keys stored, in order. + pub fn keys(&self) -> Vec { + self.lock().keys().cloned().collect() + } +} + +#[async_trait::async_trait] +impl Objects for MemoryObjects { + async fn put(&self, key: &str, body: Vec) -> anyhow::Result<()> { + let mut objects = self.lock(); + let version = objects.get(key).map_or(1, |(_, v)| v + 1); + objects.insert(key.to_string(), (body.into(), version)); + Ok(()) + } + + async fn get(&self, key: &str) -> anyhow::Result> { + Ok(self.lock().get(key).map(|(body, _)| body.clone())) + } + + async fn list(&self, prefix: &str) -> anyhow::Result> { + Ok(self + .lock() + .range(prefix.to_string()..) + .take_while(|(key, _)| key.starts_with(prefix)) + .map(|(key, (body, version))| Listed { + key: key.clone(), + size: body.len() as u64, + etag: Some(version.to_string()), + }) + .collect()) + } + + async fn list_dirs(&self, prefix: &str) -> anyhow::Result> { + let mut dirs: Vec = Vec::new(); + for listed in self.list(prefix).await? { + if let Some(end) = listed.key[prefix.len()..].find('/') { + let dir = &listed.key[..prefix.len() + end + 1]; + if dirs.last().map(String::as_str) != Some(dir) { + dirs.push(dir.to_string()); + } + } + } + Ok(dirs) + } + + async fn delete(&self, key: &str) -> anyhow::Result<()> { + self.lock().remove(key); + Ok(()) + } +} diff --git a/crates/traffic/src/query.rs b/crates/traffic/src/query.rs new file mode 100644 index 00000000..0e2b7044 --- /dev/null +++ b/crates/traffic/src/query.rs @@ -0,0 +1,985 @@ +//! Reading the files back: the traffic of a time range, over all instances +//! or one, and one client's. +//! +//! A range of up to [`MINUTE_RANGE_HOURS`] hours within the minute files' +//! retention is read from them (by minute); a longer one from the files by +//! hour. The files are found by listing the days of the range, read in +//! parallel and summed. Files of counts are cached decoded, keyed by their +//! ETag, so a file rewritten since is read again. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::sync::Arc; + +use chrono::{DateTime, TimeZone, Utc}; +use futures_util::{stream, StreamExt, TryStreamExt}; +use serde::Serialize; + +use crate::layout::{instance_parts, parse_key, Area, FileKey, FileKind, CONCURRENCY}; +use crate::objects::{Listed, Objects}; +use crate::rows::{decode, BucketRow, ClientRow, Counts, LogRow, LATENCY_BOUNDS_MS}; + +/// The longest range read by minute, in hours. +pub const MINUTE_RANGE_HOURS: i64 = 48; +/// The longest stretch of the request log a client's view reads, in hours. +pub const LOG_RANGE_HOURS: i64 = 24; +/// The most request log rows a client's view returns. +pub const LOG_LIMIT: usize = 500; +/// The lists' lengths: top repositories, top clients. +pub const TOP: usize = 20; +/// The most points of a series. +const MAX_POINTS: i64 = 180; +/// The steps a series may take, in seconds. +const STEPS: [i64; 13] = [ + 60, 300, 600, 900, 1_800, 3_600, 7_200, 10_800, 21_600, 43_200, 86_400, 172_800, 604_800, +]; +/// The rows the cache of decoded files holds. +const CACHE_ROWS: u64 = 100_000; + +const HOUR: i64 = 3_600; +const DAY: i64 = 86_400; + +#[derive(Debug)] +enum Decoded { + Buckets(Vec), + Clients(Vec), +} + +impl Decoded { + fn rows(&self) -> usize { + match self { + Decoded::Buckets(rows) => rows.len(), + Decoded::Clients(rows) => rows.len(), + } + } +} + +/// Files of counts read back, decoded. +#[derive(Clone)] +pub struct FileCache(moka::future::Cache>); + +impl FileCache { + pub fn new() -> FileCache { + FileCache( + moka::future::Cache::builder() + .max_capacity(CACHE_ROWS) + .weigher(|_, d: &Arc| u32::try_from(d.rows().max(1)).unwrap_or(u32::MAX)) + .build(), + ) + } +} + +impl Default for FileCache { + fn default() -> Self { + FileCache::new() + } +} + +/// An instance, as the views name it. +#[derive(Debug, Clone, Serialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct InstanceView { + /// Its name in the files ([`crate::instance_id`]). + pub id: String, + pub host: String, + pub started_at: Option>, +} + +impl InstanceView { + fn of(id: &str) -> InstanceView { + let (host, started_at) = instance_parts(id); + InstanceView { + id: id.to_string(), + host, + started_at, + } + } +} + +/// The time range a view covers, and its points. +#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] +pub struct Span { + pub from: DateTime, + pub to: DateTime, + /// The length of a point of the series. + pub step_seconds: i64, + /// What the files read count by: `minute` or `hour`. + pub resolution: &'static str, +} + +impl Span { + /// The span of `[from, to)` read at `now`: by minute when it is short + /// and recent enough, else by hour; `from` aligned down to the step. + pub fn new(from: DateTime, to: DateTime, now: DateTime) -> Span { + // The end rounded up to the second: a request of the second the + // range ends in (the request log's times are in milliseconds) is in. + let to_s = to.timestamp() + i64::from(to.timestamp_subsec_nanos() > 0); + let (from_s, to_s) = (from.timestamp(), to_s.max(from.timestamp() + 60)); + let minute_files_from = + now.timestamp() - i64::from(crate::layout::MINUTE_RETENTION_DAYS) * DAY + HOUR; + let by_minute = to_s - from_s <= MINUTE_RANGE_HOURS * HOUR && from_s >= minute_files_from; + let resolution = if by_minute { 60 } else { HOUR }; + let step = STEPS + .into_iter() + .filter(|step| *step >= resolution) + .find(|step| (to_s - from_s).div_euclid(*step) <= MAX_POINTS) + .unwrap_or(STEPS[STEPS.len() - 1]); + let from_s = from_s - from_s.rem_euclid(step); + Span { + from: at(from_s), + to: at(to_s), + step_seconds: step, + resolution: if by_minute { "minute" } else { "hour" }, + } + } + + fn area(&self) -> Area { + if self.resolution == "minute" { + Area::Minutes + } else { + Area::Hours + } + } + + fn contains(&self, start: i64) -> bool { + (self.from.timestamp()..self.to.timestamp()).contains(&start) + } + + fn point(&self, start: i64) -> usize { + usize::try_from((start - self.from.timestamp()).div_euclid(self.step_seconds)).unwrap_or(0) + } + + fn points(&self) -> Vec { + let mut points = Vec::new(); + let mut start = self.from.timestamp(); + while start < self.to.timestamp() { + points.push(Point { + start: at(start), + ..Point::default() + }); + start += self.step_seconds; + } + points + } +} + +fn at(secs: i64) -> DateTime { + Utc.timestamp_opt(secs, 0).single().unwrap_or_default() +} + +/// A point of a series. +#[derive(Debug, Clone, Default, Serialize, PartialEq, Eq)] +pub struct Point { + pub start: DateTime, + pub requests: u64, + pub bytes: u64, + pub client_errors: u64, + pub server_errors: u64, +} + +impl Point { + fn add(&mut self, status: u16, requests: u64, bytes: u64) { + self.requests += requests; + self.bytes += bytes; + match status { + 400..=499 => self.client_errors += requests, + 500..=599 => self.server_errors += requests, + _ => {} + } + } +} + +/// The counts of a range. +#[derive(Debug, Clone, Default, Serialize, PartialEq)] +pub struct Totals { + pub requests: u64, + pub bytes: u64, + /// 4xx answers (the next three included). + pub client_errors: u64, + /// 401 and 403 answers. + pub denied: u64, + pub not_found: u64, + /// 429 answers. + pub throttled: u64, + /// 5xx answers. + pub server_errors: u64, + /// The mean time to the response headers. + pub mean_ms: Option, + /// The latency histogram's bound below which half, 95 % and 99 % of + /// the requests were answered ([`LATENCY_BOUNDS_MS`]); `None` without + /// requests, and above the last bound. + pub p50_ms: Option, + pub p95_ms: Option, + pub p99_ms: Option, +} + +impl Totals { + fn add_status(&mut self, status: u16, requests: u64) { + match status { + 401 | 403 => self.denied += requests, + 404 => self.not_found += requests, + 429 => self.throttled += requests, + _ => {} + } + match status { + 400..=499 => self.client_errors += requests, + 500..=599 => self.server_errors += requests, + _ => {} + } + } + + fn finish(&mut self, counts: &Counts) { + if counts.requests == 0 { + return; + } + self.mean_ms = Some(counts.duration_us as f64 / counts.requests as f64 / 1_000.0); + let quantile = |q: f64| { + let wanted = (counts.requests as f64 * q).ceil() as u64; + let mut seen = 0; + for (i, n) in counts.latency.iter().enumerate() { + seen += n; + if seen >= wanted { + return LATENCY_BOUNDS_MS.get(i).copied(); + } + } + None + }; + self.p50_ms = quantile(0.5); + self.p95_ms = quantile(0.95); + self.p99_ms = quantile(0.99); + } +} + +/// A surface's share of a range. +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct SurfaceShare { + pub surface: String, + pub requests: u64, + pub bytes: u64, +} + +/// A repository's share of a range. +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct RepositoryShare { + pub name: String, + pub requests: u64, + pub bytes: u64, + pub client_errors: u64, + pub server_errors: u64, +} + +/// A status's share of a range. +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct StatusShare { + pub status: u16, + pub requests: u64, +} + +/// A client network's counts in a range. +#[derive(Debug, Clone, Default, Serialize, PartialEq, Eq)] +pub struct ClientView { + pub network: String, + /// The last address of the network seen. + pub address: String, + pub requests: u64, + pub bytes: u64, + pub client_errors: u64, + pub denied: u64, + pub not_found: u64, + pub throttled: u64, + pub server_errors: u64, + pub user_agent: Option, + /// The last account kind seen. + pub account: Option, + pub user_id: Option, + /// The user's name, filled in by the server. + pub username: Option, + /// The hosts of the instances it reached. + pub hosts: Vec, + pub first_seen: Option>, + pub last_seen: Option>, +} + +/// The traffic of a range: `GET /admin/traffic`. +#[derive(Debug, Clone, Serialize)] +pub struct Overview { + #[serde(flatten)] + pub span: Span, + /// The instances whose files the range holds (whatever the instance + /// asked for). + pub instances: Vec, + pub totals: Totals, + pub series: Vec, + pub surfaces: Vec, + /// The [`TOP`] repositories with the most requests. + pub repositories: Vec, + pub statuses: Vec, + /// The [`TOP`] client networks with the most requests (by the top + /// lists of the time buckets, whatever the repository asked for). + pub clients: Vec, +} + +/// What to read. +#[derive(Debug, Clone, Default)] +pub struct Filter { + /// Only the files of this instance ([`crate::instance_id`]). + pub instance: Option, + /// Only the counts of this repository. + pub repository: Option, +} + +/// The files of `area` covering any of `[from, to)`. +async fn files( + objects: &dyn Objects, + area: Area, + from: i64, + to: i64, +) -> anyhow::Result> { + let first_day = (from - area.span() + 1).div_euclid(DAY) * DAY; + let days: Vec = (0..) + .map(|i| first_day + i * DAY) + .take_while(|day| *day < to) + .collect(); + let listings: Vec> = stream::iter(days) + .map(|day| async move { objects.list(&area.day_prefix(day)).await }) + .buffered(CONCURRENCY) + .try_collect() + .await?; + Ok(listings + .into_iter() + .flatten() + .filter_map(|listed| Some((parse_key(area, &listed.key)?, listed))) + .filter(|(file, _)| file.start < to && file.start + area.span() > from) + .collect()) +} + +/// The decoded file `listed`, from the cache when it holds this version. +async fn read_counts( + objects: &dyn Objects, + cache: &FileCache, + file: &FileKey, + listed: &Listed, +) -> anyhow::Result>> { + let cache_key = format!("{}#{}", listed.key, listed.etag.as_deref().unwrap_or("")); + if let Some(hit) = cache.0.get(&cache_key).await { + return Ok(Some(hit)); + } + let Some(body) = objects.get(&listed.key).await? else { + // Deleted since the listing (pruned). + return Ok(None); + }; + let decoded = match file.kind { + FileKind::Buckets => Decoded::Buckets(decode(body)?), + FileKind::Clients => Decoded::Clients(decode(body)?), + FileKind::Requests => return Ok(None), + }; + let decoded = Arc::new(decoded); + if listed.etag.is_some() { + cache.0.insert(cache_key, decoded.clone()).await; + } + Ok(Some(decoded)) +} + +/// The files of counts of `span` (of `filter`'s instance, if any), read, +/// with the instance that wrote each; and every instance found. +async fn read_span( + objects: &dyn Objects, + cache: &FileCache, + span: &Span, + instance: Option<&str>, + kinds: &[FileKind], +) -> anyhow::Result<(Vec<(String, Arc)>, Vec)> { + let found = files( + objects, + span.area(), + span.from.timestamp(), + span.to.timestamp(), + ) + .await?; + let instances: BTreeSet = found.iter().map(|(f, _)| f.instance.clone()).collect(); + let wanted: Vec<(FileKey, Listed)> = found + .into_iter() + .filter(|(f, _)| kinds.contains(&f.kind)) + .filter(|(f, _)| instance.is_none_or(|i| i == f.instance)) + .collect(); + let read: Vec)>> = stream::iter(wanted) + .map(|(file, listed)| async move { + let decoded = read_counts(objects, cache, &file, &listed).await?; + anyhow::Ok(decoded.map(|d| (file.instance, d))) + }) + .buffer_unordered(CONCURRENCY) + .try_collect() + .await?; + Ok(( + read.into_iter().flatten().collect(), + instances.iter().map(|id| InstanceView::of(id)).collect(), + )) +} + +/// Sum client rows into views by network. +fn sum_clients<'a>(rows: impl Iterator) -> Vec { + let mut by_network: HashMap<&str, (ClientView, i64, BTreeSet)> = HashMap::new(); + for (instance, row) in rows { + let (view, latest, hosts) = by_network.entry(&row.network).or_insert_with(|| { + ( + ClientView { + network: row.network.clone(), + ..ClientView::default() + }, + i64::MIN, + BTreeSet::new(), + ) + }); + let n = |v: i64| u64::try_from(v).unwrap_or(0); + view.requests += n(row.requests); + view.bytes += n(row.bytes); + view.client_errors += n(row.client_errors); + view.denied += n(row.denied); + view.not_found += n(row.not_found); + view.throttled += n(row.throttled); + view.server_errors += n(row.server_errors); + hosts.insert(instance_parts(instance).0); + let start = at(row.start); + view.first_seen = Some(view.first_seen.map_or(start, |s| s.min(start))); + view.last_seen = Some(view.last_seen.map_or(start, |s| s.max(start))); + if row.start >= *latest { + *latest = row.start; + let some = |s: &str| (!s.is_empty()).then(|| s.to_string()); + if !row.address.is_empty() { + view.address.clone_from(&row.address); + } + if let Some(ua) = some(&row.user_agent) { + view.user_agent = Some(ua); + } + if let Some(account) = some(&row.account) { + view.account = Some(account); + view.user_id = (row.user_id != 0).then_some(row.user_id); + } + } + } + let mut views: Vec = by_network + .into_values() + .map(|(mut view, _, hosts)| { + view.hosts = hosts.into_iter().collect(); + view + }) + .collect(); + views.sort_by(|a, b| b.requests.cmp(&a.requests).then(a.network.cmp(&b.network))); + views +} + +/// The traffic of `span` over `objects`' files. +pub async fn overview( + objects: &dyn Objects, + cache: &FileCache, + span: Span, + filter: &Filter, +) -> anyhow::Result { + let (files, instances) = read_span( + objects, + cache, + &span, + filter.instance.as_deref(), + &[FileKind::Buckets, FileKind::Clients], + ) + .await?; + + let mut series = span.points(); + let mut totals = Totals::default(); + let mut all = Counts::default(); + let mut surfaces: BTreeMap<&str, (u64, u64)> = BTreeMap::new(); + let mut repositories: HashMap<&str, RepositoryShare> = HashMap::new(); + let mut statuses: BTreeMap = BTreeMap::new(); + let mut clients = Vec::new(); + for (instance, decoded) in &files { + match &**decoded { + Decoded::Buckets(rows) => { + for row in rows { + if !span.contains(row.start) + || filter + .repository + .as_deref() + .is_some_and(|r| r != row.repository) + { + continue; + } + let counts = row.counts(); + let status = u16::try_from(row.status).unwrap_or(0); + all.add(&counts); + totals.add_status(status, counts.requests); + if let Some(point) = series.get_mut(span.point(row.start)) { + point.add(status, counts.requests, counts.bytes); + } + let surface = surfaces.entry(&row.surface).or_default(); + surface.0 += counts.requests; + surface.1 += counts.bytes; + *statuses.entry(status).or_default() += counts.requests; + if !row.repository.is_empty() { + let share = repositories.entry(&row.repository).or_insert_with(|| { + RepositoryShare { + name: row.repository.clone(), + requests: 0, + bytes: 0, + client_errors: 0, + server_errors: 0, + } + }); + share.requests += counts.requests; + share.bytes += counts.bytes; + match status { + 400..=499 => share.client_errors += counts.requests, + 500..=599 => share.server_errors += counts.requests, + _ => {} + } + } + } + } + Decoded::Clients(rows) => clients.extend( + rows.iter() + .filter(|row| span.contains(row.start)) + .map(|row| (instance.as_str(), row)), + ), + } + } + totals.requests = all.requests; + totals.bytes = all.bytes; + totals.finish(&all); + + let mut surfaces: Vec = surfaces + .into_iter() + .map(|(surface, (requests, bytes))| SurfaceShare { + surface: surface.to_string(), + requests, + bytes, + }) + .collect(); + surfaces.sort_by(|a, b| b.requests.cmp(&a.requests).then(a.surface.cmp(&b.surface))); + let mut repositories: Vec = repositories.into_values().collect(); + repositories.sort_by(|a, b| b.requests.cmp(&a.requests).then(a.name.cmp(&b.name))); + repositories.truncate(TOP); + let mut clients = sum_clients(clients.into_iter()); + clients.truncate(TOP); + + Ok(Overview { + span, + instances, + totals, + series, + surfaces, + repositories, + statuses: statuses + .into_iter() + .map(|(status, requests)| StatusShare { status, requests }) + .collect(), + clients, + }) +} + +/// A request of the request log. +#[derive(Debug, Clone, Serialize, PartialEq)] +pub struct LogEntry { + pub time: DateTime, + pub address: String, + pub method: String, + pub path: String, + pub surface: String, + pub repository: Option, + pub status: u16, + pub bytes: u64, + /// Time to the response headers. + pub duration_ms: f64, + pub user_agent: Option, + pub account: Option, + pub user_id: Option, + /// The user's name, filled in by the server. + pub username: Option, + /// The host of the instance that answered. + pub host: String, +} + +/// How often a value occurs in a client's log. +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct Tally { + pub value: String, + pub requests: u64, +} + +/// How often an account occurs in a client's log. +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct AccountTally { + /// The account kind ([`crate::Account::kind`]); `None` when unknown. + pub account: Option, + pub user_id: Option, + /// The user's name, filled in by the server. + pub username: Option, + pub requests: u64, +} + +/// The request log's part of a client's view. +#[derive(Debug, Clone, Serialize)] +pub struct ClientLog { + /// The stretch read: the last [`LOG_RANGE_HOURS`] of the range at most. + pub from: DateTime, + pub to: DateTime, + /// The client's requests in it. + pub matched: u64, + /// The newest [`LOG_LIMIT`] of them, newest first. + pub entries: Vec, + /// The paths, user agents, accounts and statuses of all of them, most + /// frequent first ([`TOP`] each). + pub paths: Vec, + pub user_agents: Vec, + pub accounts: Vec, + pub statuses: Vec, +} + +/// One client network's traffic: `GET /admin/traffic/client`. +#[derive(Debug, Clone, Serialize)] +pub struct ClientDetail { + pub network: String, + #[serde(flatten)] + pub span: Span, + /// The instances whose files the range holds. + pub instances: Vec, + /// Its counts, from the time buckets it was among the top clients of. + pub totals: Option, + pub series: Vec, + pub log: ClientLog, +} + +/// `network`'s traffic in `span`: its counts in the top lists, and its +/// requests in the request log. +pub async fn client( + objects: &dyn Objects, + cache: &FileCache, + span: Span, + instance: Option<&str>, + network: &str, +) -> anyhow::Result { + let (files, instances) = + read_span(objects, cache, &span, instance, &[FileKind::Clients]).await?; + let mut series = span.points(); + let mut rows = Vec::new(); + for (instance, decoded) in &files { + if let Decoded::Clients(found) = &**decoded { + for row in found { + if row.network == network && span.contains(row.start) { + if let Some(point) = series.get_mut(span.point(row.start)) { + let n = |v: i64| u64::try_from(v).unwrap_or(0); + point.requests += n(row.requests); + point.bytes += n(row.bytes); + point.client_errors += n(row.client_errors); + point.server_errors += n(row.server_errors); + } + rows.push((instance.as_str(), row)); + } + } + } + } + let totals = sum_clients(rows.into_iter()).into_iter().next(); + let log = client_log(objects, &span, instance, network).await?; + Ok(ClientDetail { + network: network.to_string(), + span, + instances, + totals, + series, + log, + }) +} + +fn tallies(counts: HashMap) -> Vec { + let mut tallies: Vec = counts + .into_iter() + .map(|(value, requests)| Tally { value, requests }) + .collect(); + tallies.sort_by(|a, b| b.requests.cmp(&a.requests).then(a.value.cmp(&b.value))); + tallies.truncate(TOP); + tallies +} + +async fn client_log( + objects: &dyn Objects, + span: &Span, + instance: Option<&str>, + network: &str, +) -> anyhow::Result { + let to = span.to.timestamp(); + let from = span.from.timestamp().max(to - LOG_RANGE_HOURS * HOUR); + // A file is named after the hour of its flush, which comes up to a + // minute after its rows. + let wanted: Vec<(FileKey, Listed)> = files(objects, Area::Requests, from, to + 120) + .await? + .into_iter() + .filter(|(f, _)| instance.is_none_or(|i| i == f.instance)) + .collect(); + let matched: Vec> = stream::iter(wanted) + .map(|(file, listed)| async move { + let Some(body) = objects.get(&listed.key).await? else { + return anyhow::Ok(Vec::new()); + }; + let rows: Vec = decode(body)?; + Ok(rows + .into_iter() + .filter(|row| row.network == network) + .filter(|row| (from * 1_000..to * 1_000).contains(&row.time_ms)) + .map(|row| (file.instance.clone(), row)) + .collect()) + }) + .buffer_unordered(CONCURRENCY) + .try_collect() + .await?; + let mut matched: Vec<(String, LogRow)> = matched.into_iter().flatten().collect(); + matched.sort_by_key(|(_, row)| std::cmp::Reverse(row.time_ms)); + + let (mut paths, mut user_agents, mut accounts, mut statuses) = ( + HashMap::new(), + HashMap::new(), + HashMap::new(), + HashMap::new(), + ); + for (_, row) in &matched { + *paths.entry(row.path.clone()).or_default() += 1; + if !row.user_agent.is_empty() { + *user_agents.entry(row.user_agent.clone()).or_default() += 1; + } + *accounts + .entry((row.account.clone(), row.user_id)) + .or_insert(0u64) += 1; + *statuses.entry(row.status.to_string()).or_default() += 1; + } + let some = |s: String| (!s.is_empty()).then_some(s); + Ok(ClientLog { + from: at(from), + to: at(to), + matched: matched.len() as u64, + paths: tallies(paths), + user_agents: tallies(user_agents), + accounts: { + let mut accounts: Vec = accounts + .into_iter() + .map(|((account, user_id), requests)| AccountTally { + account: (!account.is_empty()).then_some(account), + user_id: (user_id != 0).then_some(user_id), + username: None, + requests, + }) + .collect(); + accounts.sort_by(|a, b| { + b.requests + .cmp(&a.requests) + .then_with(|| (&a.account, a.user_id).cmp(&(&b.account, b.user_id))) + }); + accounts.truncate(TOP); + accounts + }, + statuses: tallies(statuses), + entries: matched + .into_iter() + .take(LOG_LIMIT) + .map(|(instance, row)| LogEntry { + time: Utc + .timestamp_millis_opt(row.time_ms) + .single() + .unwrap_or_default(), + address: row.address, + method: row.method, + path: row.path, + surface: row.surface, + repository: some(row.repository), + status: u16::try_from(row.status).unwrap_or(0), + bytes: u64::try_from(row.bytes).unwrap_or(0), + duration_ms: row.duration_us as f64 / 1_000.0, + user_agent: some(row.user_agent), + account: some(row.account), + user_id: (row.user_id != 0).then_some(row.user_id), + username: None, + host: instance_parts(&instance).0, + }) + .collect(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::layout::Flusher; + use crate::objects::MemoryObjects; + use crate::record::{Event, Recorder}; + use crate::slot::{Account, Noted}; + use crate::TrafficSettings; + use std::time::Duration; + + fn t(rfc3339: &str) -> DateTime { + DateTime::parse_from_rfc3339(rfc3339).unwrap().to_utc() + } + + fn event(at: &str, network: &str, repository: &str, status: u16, ms: u64) -> Event { + Event { + at: t(at), + surface: "plugin_npm", + method: "GET", + status, + duration: Duration::from_millis(ms), + bytes: 1_000, + path: format!("/repository/{repository}/left-pad"), + user_agent: Some("npm/10".into()), + noted: Noted { + client: network.parse().ok(), + network: Some(network.into()), + account: Some(Account { + kind: "token", + user_id: Some(3), + }), + repository: Some(repository.into()), + }, + } + } + + /// Two instances, each with its own files. + async fn two_instances() -> Arc { + let objects = Arc::new(MemoryObjects::default()); + let settings = TrafficSettings::default(); + let now = t("2026-10-30T14:10:00Z"); + for (instance, events) in [ + ( + "a_20261030T100000Z_0001", + vec![ + event("2026-10-30T14:00:01Z", "192.0.2.1", "npm", 200, 20), + event("2026-10-30T14:00:02Z", "192.0.2.1", "npm", 404, 2), + event("2026-10-30T14:05:00Z", "192.0.2.2", "npm-proxy", 500, 900), + ], + ), + ( + "b_20261030T110000Z_0002", + vec![ + event("2026-10-30T14:00:30Z", "192.0.2.1", "npm", 200, 40), + event("2026-10-30T13:59:59Z", "192.0.2.3", "npm", 401, 1), + ], + ), + ] { + let recorder = Recorder::new(); + for e in events { + recorder.record(e); + } + let mut flusher = Flusher::new(objects.clone(), instance.into()); + flusher.flush(recorder.drain(now), &settings, now).await; + } + objects + } + + #[tokio::test] + async fn the_overview_sums_every_instance() { + let objects = two_instances().await; + let cache = FileCache::new(); + let now = t("2026-10-30T14:10:00Z"); + let span = Span::new(t("2026-10-30T13:10:00Z"), now, now); + assert_eq!((span.resolution, span.step_seconds), ("minute", 60)); + + let all = overview(&*objects, &cache, span, &Filter::default()) + .await + .unwrap(); + assert_eq!(all.instances.len(), 2); + assert_eq!(all.instances[0].host, "a"); + assert_eq!(all.totals.requests, 5); + assert_eq!( + ( + all.totals.client_errors, + all.totals.denied, + all.totals.not_found + ), + (2, 1, 1) + ); + assert_eq!(all.totals.server_errors, 1); + assert_eq!(all.totals.p50_ms, Some(25)); + assert_eq!(all.totals.p99_ms, Some(1_000)); + assert_eq!(all.series.len(), 60); + assert_eq!(all.series.iter().map(|p| p.requests).sum::(), 5); + assert_eq!(all.repositories[0].name, "npm"); + assert_eq!(all.repositories[0].requests, 4); + let top = &all.clients[0]; + assert_eq!((top.network.as_str(), top.requests), ("192.0.2.1", 3)); + assert_eq!(top.hosts, vec!["a", "b"]); + assert_eq!(top.user_id, Some(3)); + + let one = overview( + &*objects, + &cache, + span, + &Filter { + instance: Some("b_20261030T110000Z_0002".into()), + repository: None, + }, + ) + .await + .unwrap(); + assert_eq!(one.totals.requests, 2); + assert_eq!(one.instances.len(), 2, "the selector still lists both"); + + let repo = overview( + &*objects, + &cache, + span, + &Filter { + instance: None, + repository: Some("npm-proxy".into()), + }, + ) + .await + .unwrap(); + assert_eq!(repo.totals.requests, 1); + + // A week reads the files by hour. + let week = Span::new(t("2026-10-23T14:10:00Z"), now, now); + assert_eq!((week.resolution, week.step_seconds), ("hour", 3_600)); + let week = overview(&*objects, &cache, week, &Filter::default()) + .await + .unwrap(); + assert_eq!(week.totals.requests, 5); + } + + #[tokio::test] + async fn a_client_view_reads_its_log() { + let objects = two_instances().await; + let now = t("2026-10-30T14:10:00Z"); + let span = Span::new(t("2026-10-30T13:10:00Z"), now, now); + let detail = client(&*objects, &FileCache::new(), span, None, "192.0.2.1") + .await + .unwrap(); + assert_eq!(detail.totals.as_ref().unwrap().requests, 3); + assert_eq!(detail.log.matched, 3); + assert_eq!( + detail.log.entries[0].time, + t("2026-10-30T14:00:30Z"), + "newest first" + ); + assert_eq!(detail.log.entries[0].host, "b"); + assert_eq!(detail.log.entries[0].repository.as_deref(), Some("npm")); + assert_eq!(detail.log.paths[0].value, "/repository/npm/left-pad"); + assert_eq!( + detail.log.statuses[0], + Tally { + value: "200".into(), + requests: 2 + } + ); + } + + #[test] + fn a_span_ends_after_the_second_it_ends_in() { + let now = t("2026-10-30T14:10:00.250Z"); + let span = Span::new(t("2026-10-30T13:10:00Z"), now, now); + assert_eq!(span.to, t("2026-10-30T14:10:01Z")); + } + + #[test] + fn spans_pick_their_resolution_and_step() { + let now = t("2026-10-30T14:10:00Z"); + let day = Span::new(t("2026-10-29T14:10:00Z"), now, now); + assert_eq!((day.resolution, day.step_seconds), ("minute", 600)); + assert_eq!(day.from, t("2026-10-29T14:10:00Z")); + let month = Span::new(t("2026-09-30T14:10:00Z"), now, now); + assert_eq!((month.resolution, month.step_seconds), ("hour", 21_600)); + // Older than the minute files: by hour. + let old = Span::new(t("2026-10-20T00:00:00Z"), t("2026-10-20T01:00:00Z"), now); + assert_eq!(old.resolution, "hour"); + } +} diff --git a/crates/traffic/src/record.rs b/crates/traffic/src/record.rs new file mode 100644 index 00000000..c69e73ed --- /dev/null +++ b/crates/traffic/src/record.rs @@ -0,0 +1,377 @@ +//! Counting requests in memory, minute by minute. + +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Mutex, PoisonError}; +use std::time::Duration; + +use chrono::{DateTime, Utc}; + +use crate::rows::{BucketKey, Counts, LogRow}; +use crate::slot::Noted; +use crate::top::TopClients; +use crate::TrafficSettings; + +/// The networks a minute's top list keeps track of ([`TopClients`]). +const MINUTE_CLIENTS: usize = 1_024; +/// The keys a minute counts under before new repositories are counted +/// without their name: a bound whatever the handlers note. +const MAX_KEYS_PER_MINUTE: usize = 10_000; +/// The request log rows kept between two flushes; more are dropped (and +/// counted in the log of the flush). +const MAX_LOG_ROWS: usize = 250_000; +/// The closed minutes kept while flushes fail; older ones are dropped. +const MAX_CLOSED_MINUTES: usize = 180; +/// The longest path and user agent the request log keeps, in bytes. +const MAX_PATH_BYTES: usize = 1_024; +const MAX_USER_AGENT_BYTES: usize = 256; + +/// One answered request. +#[derive(Debug, Clone)] +pub struct Event { + /// When the response was sent (its body done or cut off). + pub at: DateTime, + /// The surface (`forge_telemetry::http::Surface`). + pub surface: &'static str, + pub method: &'static str, + pub status: u16, + /// Time to the response headers. + pub duration: Duration, + /// Response body bytes sent. + pub bytes: u64, + /// The path as the log keeps it (secrets replaced); empty when the log + /// is off ([`Recorder::logs_requests`]). + pub path: String, + pub user_agent: Option, + /// What the layers of the request noted ([`crate::Slot`]). + pub noted: Noted, +} + +/// The counts of one minute. +#[derive(Debug, Clone)] +pub struct Minute { + /// Its start, in Unix seconds. + pub start: i64, + pub buckets: HashMap, + pub clients: TopClients, +} + +impl Minute { + fn new(start: i64) -> Minute { + Minute { + start, + buckets: HashMap::new(), + clients: TopClients::new(MINUTE_CLIENTS), + } + } + + fn is_empty(&self) -> bool { + self.buckets.is_empty() + } +} + +/// What a flush takes from the [`Recorder`]. +#[derive(Debug, Default)] +pub struct Drained { + /// The minutes counted to the end. + pub minutes: Vec, + pub(crate) log: Vec, + /// Log rows dropped for want of room. + pub log_dropped: u64, +} + +#[derive(Debug)] +struct Pending { + current: Minute, + closed: Vec, + log: Vec, +} + +/// Counts this instance's requests until the next flush. +#[derive(Debug)] +pub struct Recorder { + enabled: AtomicBool, + addresses: AtomicBool, + log: AtomicBool, + log_dropped: AtomicU64, + pending: Mutex, +} + +impl Default for Recorder { + fn default() -> Self { + Recorder::new() + } +} + +/// `s` cut to at most `max` bytes, on a character boundary. +fn clip(mut s: String, max: usize) -> String { + if s.len() > max { + let mut end = max; + while !s.is_char_boundary(end) { + end -= 1; + } + s.truncate(end); + } + s +} + +impl Recorder { + /// A recorder with the default settings ([`TrafficSettings::default`]), + /// until [`Self::apply`] says otherwise. + pub fn new() -> Recorder { + let defaults = TrafficSettings::default(); + Recorder { + enabled: AtomicBool::new(defaults.enabled), + addresses: AtomicBool::new(defaults.client_addresses), + log: AtomicBool::new(defaults.request_log), + log_dropped: AtomicU64::new(0), + pending: Mutex::new(Pending { + current: Minute::new(0), + closed: Vec::new(), + log: Vec::new(), + }), + } + } + + /// Follow `settings` from the next request on. + pub fn apply(&self, settings: &TrafficSettings) { + self.enabled.store(settings.enabled, Ordering::Relaxed); + self.addresses + .store(settings.client_addresses, Ordering::Relaxed); + self.log.store(settings.request_log, Ordering::Relaxed); + } + + /// Whether requests are counted at all. + pub fn enabled(&self) -> bool { + self.enabled.load(Ordering::Relaxed) + } + + /// Whether requests go to the request log: only then does an + /// [`Event`] need its path. + pub fn logs_requests(&self) -> bool { + self.enabled() && self.log.load(Ordering::Relaxed) + } + + fn lock(&self) -> std::sync::MutexGuard<'_, Pending> { + self.pending.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// Count `event`. + pub fn record(&self, event: Event) { + if !self.enabled() { + return; + } + let addresses = self.addresses.load(Ordering::Relaxed); + let Event { + at, + surface, + method, + status, + duration, + bytes, + path, + user_agent, + noted, + } = event; + let user_agent = user_agent.map(|ua| clip(ua, MAX_USER_AGENT_BYTES)); + let (network, address) = match (addresses, noted.network, noted.client) { + (true, Some(network), client) => { + (network, client.map(|c| c.to_string()).unwrap_or_default()) + } + _ => (String::new(), String::new()), + }; + let (account, user_id) = noted + .account + .map(|a| (a.kind, a.user_id)) + .unwrap_or(("", None)); + let repository = noted.repository.unwrap_or_default(); + let row = self.log.load(Ordering::Relaxed).then(|| LogRow { + time_ms: at.timestamp_millis(), + network: network.clone(), + address: address.clone(), + method: method.to_string(), + path: clip(path, MAX_PATH_BYTES), + surface: surface.to_string(), + repository: repository.clone(), + status: i32::from(status), + bytes: i64::try_from(bytes).unwrap_or(i64::MAX), + duration_us: i64::try_from(duration.as_micros()).unwrap_or(i64::MAX), + user_agent: user_agent.clone().unwrap_or_default(), + account: account.to_string(), + user_id: user_id.unwrap_or(0), + }); + let mut key = BucketKey { + surface: surface.to_string(), + repository, + method: method.to_string(), + status, + }; + + let minute = at.timestamp().div_euclid(60) * 60; + let mut pending = self.lock(); + if minute > pending.current.start { + let done = std::mem::replace(&mut pending.current, Minute::new(minute)); + if !done.is_empty() { + pending.closed.push(done); + if pending.closed.len() > MAX_CLOSED_MINUTES { + pending.closed.remove(0); + } + } + } + // A request that ended in an earlier minute than one already counted + // (the clock stepped back) counts in the current one. + let current = &mut pending.current; + if current.buckets.len() >= MAX_KEYS_PER_MINUTE && !current.buckets.contains_key(&key) { + key.repository.clear(); + } + current + .buckets + .entry(key) + .or_default() + .observe(duration, bytes); + if !network.is_empty() { + let client = current.clients.entry(&network); + client.observe(status, bytes); + client.address = address; + if let Some(ua) = user_agent { + client.user_agent = ua; + } + if !account.is_empty() { + client.account = account.to_string(); + client.user_id = user_id; + } + } + if let Some(row) = row { + if pending.log.len() < MAX_LOG_ROWS { + pending.log.push(row); + } else { + self.log_dropped.fetch_add(1, Ordering::Relaxed); + } + } + } + + /// Take the minutes that ended before `now`, and the request log. + pub fn drain(&self, now: DateTime) -> Drained { + self.take(Some(now.timestamp().div_euclid(60) * 60)) + } + + /// Take everything, the minute still being counted included (on + /// shutdown). + pub fn drain_all(&self) -> Drained { + self.take(None) + } + + fn take(&self, before: Option) -> Drained { + let mut pending = self.lock(); + let close = before.is_none_or(|start| pending.current.start < start); + if close && !pending.current.is_empty() { + let next = Minute::new(before.unwrap_or(pending.current.start)); + let done = std::mem::replace(&mut pending.current, next); + pending.closed.push(done); + } + Drained { + minutes: std::mem::take(&mut pending.closed), + log: std::mem::take(&mut pending.log), + log_dropped: self.log_dropped.swap(0, Ordering::Relaxed), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::slot::Account; + use chrono::TimeZone; + + fn at(secs: i64) -> DateTime { + Utc.timestamp_opt(1_700_000_000 + secs, 0).unwrap() + } + + fn event(secs: i64, status: u16) -> Event { + Event { + at: at(secs), + surface: "nuget", + method: "GET", + status, + duration: Duration::from_millis(12), + bytes: 100, + path: "/repository/nuget/v3/index.json".into(), + user_agent: Some("NuGet Command Line/6.0".into()), + noted: Noted { + client: Some("192.0.2.10".parse().unwrap()), + network: Some("192.0.2.10".into()), + account: Some(Account { + kind: "token", + user_id: Some(7), + }), + repository: Some("nuget".into()), + }, + } + } + + #[test] + fn minutes_close_when_the_next_one_starts() { + let r = Recorder::new(); + // 1_700_000_000 is 20 s into its minute: the first two requests + // share it, the third starts the next one. + r.record(event(0, 200)); + r.record(event(30, 404)); + r.record(event(45, 200)); + let early = r.drain(at(41)); + assert_eq!(early.minutes.len(), 1, "the first minute ended"); + assert_eq!(early.minutes[0].start, 1_699_999_980); + assert_eq!(early.log.len(), 3, "the log goes with every flush"); + let minute = &early.minutes[0]; + let ok = BucketKey { + surface: "nuget".into(), + repository: "nuget".into(), + method: "GET".into(), + status: 200, + }; + assert_eq!(minute.buckets[&ok].requests, 1); + let top = minute.clients.top(1); + assert_eq!(top[0].0, "192.0.2.10"); + assert_eq!(top[0].1.user_id, Some(7)); + + let rest = r.drain_all(); + assert_eq!(rest.minutes.len(), 1); + assert_eq!(rest.minutes[0].start, 1_700_000_040); + assert_eq!( + rest.minutes[0] + .buckets + .values() + .map(|c| c.requests) + .sum::(), + 1 + ); + assert!(r.drain_all().minutes.is_empty()); + } + + #[test] + fn settings_switch_parts_off() { + let r = Recorder::new(); + r.apply(&TrafficSettings { + client_addresses: false, + request_log: false, + ..TrafficSettings::default() + }); + assert!(!r.logs_requests()); + r.record(event(0, 200)); + let d = r.drain_all(); + assert!(d.log.is_empty()); + assert!(d.minutes[0].clients.is_empty(), "no addresses"); + + r.apply(&TrafficSettings { + enabled: false, + ..TrafficSettings::default() + }); + r.record(event(0, 200)); + assert!(r.drain_all().minutes.is_empty()); + } + + #[test] + fn long_values_are_clipped_on_a_char_boundary() { + assert_eq!(clip("äää".into(), 3), "ä"); + assert_eq!(clip("abc".into(), 10), "abc"); + } +} diff --git a/crates/traffic/src/rows.rs b/crates/traffic/src/rows.rs new file mode 100644 index 00000000..6f39f6c4 --- /dev/null +++ b/crates/traffic/src/rows.rs @@ -0,0 +1,312 @@ +//! The counts and the rows of the Parquet files. +//! +//! The files are plain Parquet, readable by any tool (DuckDB, pandas). A +//! column the record API cannot leave empty holds `""` or `0` for none: the +//! derived reader takes no `Option` field. Times are Unix seconds (`start`) +//! or milliseconds (`time_ms`), in UTC. + +use std::sync::Arc; + +use bytes::Bytes; +use parquet::basic::Compression; +use parquet::file::properties::WriterProperties; +use parquet::file::reader::{FileReader, SerializedFileReader}; +use parquet::file::writer::SerializedFileWriter; +use parquet::record::{RecordReader, RecordWriter}; +use parquet_derive::{ParquetRecordReader, ParquetRecordWriter}; + +/// The upper bounds of the latency histogram, in milliseconds: a request is +/// counted in the first bucket whose bound it does not exceed, or in the +/// last, above all of them. +pub const LATENCY_BOUNDS_MS: [u64; 13] = [ + 5, 10, 25, 50, 100, 250, 500, 1_000, 2_500, 5_000, 10_000, 30_000, 60_000, +]; + +/// The buckets of the latency histogram: one per bound, and one above. +pub const LATENCY_BUCKETS: usize = LATENCY_BOUNDS_MS.len() + 1; + +/// What requests are counted under within a time bucket. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct BucketKey { + /// The surface (`forge_telemetry::http::Surface`), narrowed to the + /// format for a repository's requests. + pub surface: String, + /// The repository the request addressed; `""` for none. + pub repository: String, + /// The request method. + pub method: String, + /// The response status. + pub status: u16, +} + +/// The counts of a [`BucketKey`] in a time bucket. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Counts { + pub requests: u64, + /// Response body bytes sent. + pub bytes: u64, + /// Time to the response headers, summed, in microseconds. + pub duration_us: u64, + /// The latency histogram ([`LATENCY_BOUNDS_MS`]). + pub latency: [u64; LATENCY_BUCKETS], +} + +impl Counts { + /// Count one request. + pub fn observe(&mut self, duration: std::time::Duration, bytes: u64) { + self.requests += 1; + self.bytes += bytes; + let us = u64::try_from(duration.as_micros()).unwrap_or(u64::MAX); + self.duration_us = self.duration_us.saturating_add(us); + let bucket = LATENCY_BOUNDS_MS + .iter() + .position(|&bound| us <= bound * 1_000) + .unwrap_or(LATENCY_BOUNDS_MS.len()); + self.latency[bucket] += 1; + } + + /// Add `other`'s counts. + pub fn add(&mut self, other: &Counts) { + self.requests += other.requests; + self.bytes += other.bytes; + self.duration_us = self.duration_us.saturating_add(other.duration_us); + for (mine, theirs) in self.latency.iter_mut().zip(other.latency) { + *mine += theirs; + } + } +} + +/// A row of a `buckets` file: a key's counts in the bucket starting at +/// `start` (a minute or an hour). +#[derive(Debug, Clone, Default, PartialEq, Eq, ParquetRecordWriter, ParquetRecordReader)] +pub(crate) struct BucketRow { + pub start: i64, + pub surface: String, + pub repository: String, + pub method: String, + pub status: i32, + pub requests: i64, + pub bytes: i64, + pub duration_us: i64, + pub le_5ms: i64, + pub le_10ms: i64, + pub le_25ms: i64, + pub le_50ms: i64, + pub le_100ms: i64, + pub le_250ms: i64, + pub le_500ms: i64, + pub le_1s: i64, + pub le_2500ms: i64, + pub le_5s: i64, + pub le_10s: i64, + pub le_30s: i64, + pub le_60s: i64, + pub over_60s: i64, +} + +fn signed(n: u64) -> i64 { + i64::try_from(n).unwrap_or(i64::MAX) +} + +fn unsigned(n: i64) -> u64 { + u64::try_from(n).unwrap_or(0) +} + +impl BucketRow { + pub fn new(start: i64, key: &BucketKey, counts: &Counts) -> BucketRow { + let l = counts.latency.map(signed); + BucketRow { + start, + surface: key.surface.clone(), + repository: key.repository.clone(), + method: key.method.clone(), + status: i32::from(key.status), + requests: signed(counts.requests), + bytes: signed(counts.bytes), + duration_us: signed(counts.duration_us), + le_5ms: l[0], + le_10ms: l[1], + le_25ms: l[2], + le_50ms: l[3], + le_100ms: l[4], + le_250ms: l[5], + le_500ms: l[6], + le_1s: l[7], + le_2500ms: l[8], + le_5s: l[9], + le_10s: l[10], + le_30s: l[11], + le_60s: l[12], + over_60s: l[13], + } + } + + pub fn counts(&self) -> Counts { + Counts { + requests: unsigned(self.requests), + bytes: unsigned(self.bytes), + duration_us: unsigned(self.duration_us), + latency: [ + self.le_5ms, + self.le_10ms, + self.le_25ms, + self.le_50ms, + self.le_100ms, + self.le_250ms, + self.le_500ms, + self.le_1s, + self.le_2500ms, + self.le_5s, + self.le_10s, + self.le_30s, + self.le_60s, + self.over_60s, + ] + .map(unsigned), + } + } +} + +/// A row of a `clients` file: a client network's counts in the bucket +/// starting at `start`, as one of the bucket's top clients. +#[derive(Debug, Clone, Default, PartialEq, Eq, ParquetRecordWriter, ParquetRecordReader)] +pub(crate) struct ClientRow { + pub start: i64, + /// The network the requests are counted under. + pub network: String, + /// The last address of the network seen. + pub address: String, + pub requests: i64, + pub bytes: i64, + /// 4xx answers, the next three included. + pub client_errors: i64, + /// 401 and 403 answers. + pub denied: i64, + pub not_found: i64, + /// 429 answers. + pub throttled: i64, + /// 5xx answers. + pub server_errors: i64, + /// The last user agent seen; `""` for none. + pub user_agent: String, + /// The last account kind seen ([`crate::Account::kind`]). + pub account: String, + /// The last account's user; `0` for none. + pub user_id: i64, +} + +/// A row of a `requests` file: one request of the request log. +#[derive(Debug, Clone, Default, PartialEq, Eq, ParquetRecordWriter, ParquetRecordReader)] +pub(crate) struct LogRow { + /// When the response was sent, in Unix milliseconds. + pub time_ms: i64, + /// The client network; `""` with client addresses switched off. + pub network: String, + /// The client address; `""` with client addresses switched off. + pub address: String, + pub method: String, + /// The path, without the query and with secrets replaced + /// (`forge-server`'s `recorded_path`). + pub path: String, + pub surface: String, + /// `""` for none. + pub repository: String, + pub status: i32, + pub bytes: i64, + /// Time to the response headers, in microseconds. + pub duration_us: i64, + /// `""` for none. + pub user_agent: String, + pub account: String, + /// `0` for none. + pub user_id: i64, +} + +/// `rows` as a Parquet file, in one row group. +pub(crate) fn encode(rows: &[T]) -> anyhow::Result> +where + for<'a> &'a [T]: RecordWriter, +{ + let schema = rows.schema()?; + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .set_created_by(format!("forge {}", env!("CARGO_PKG_VERSION"))) + .build(); + let mut writer = SerializedFileWriter::new(Vec::new(), schema, Arc::new(props))?; + let mut group = writer.next_row_group()?; + rows.write_to_row_group(&mut group)?; + group.close()?; + Ok(writer.into_inner()?) +} + +/// The rows of the Parquet file `bytes`. +pub(crate) fn decode(bytes: Bytes) -> anyhow::Result> +where + Vec: RecordReader, +{ + let reader = SerializedFileReader::new(bytes)?; + let mut rows = Vec::new(); + for i in 0..reader.num_row_groups() { + let mut group = reader.get_row_group(i)?; + let n = usize::try_from(group.metadata().num_rows()).unwrap_or(0); + if n > 0 { + rows.read_from_row_group(&mut *group, n)?; + } + } + Ok(rows) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + + #[test] + fn latency_lands_in_its_bucket() { + let mut c = Counts::default(); + c.observe(Duration::from_micros(4_999), 1); + c.observe(Duration::from_millis(5), 1); + c.observe(Duration::from_micros(5_001), 1); + c.observe(Duration::from_secs(61), 1); + assert_eq!(c.latency[0], 2, "up to and including 5 ms"); + assert_eq!(c.latency[1], 1); + assert_eq!(c.latency[LATENCY_BUCKETS - 1], 1, "above the last bound"); + assert_eq!(c.requests, 4); + assert_eq!(c.bytes, 4); + } + + #[test] + fn rows_round_trip_through_parquet() { + let key = BucketKey { + surface: "nuget".into(), + repository: "nuget-hosted".into(), + method: "GET".into(), + status: 200, + }; + let mut counts = Counts::default(); + counts.observe(Duration::from_millis(30), 1234); + counts.observe(Duration::from_millis(3), 10); + let rows = vec![ + BucketRow::new(1_700_000_040, &key, &counts), + BucketRow::new(1_700_000_100, &key, &Counts::default()), + ]; + let back: Vec = decode(encode(&rows).unwrap().into()).unwrap(); + assert_eq!(back, rows); + assert_eq!(back[0].counts(), counts); + + let log = vec![LogRow { + time_ms: 1_700_000_040_123, + network: "2001:db8::/64".into(), + address: "2001:db8::7".into(), + method: "GET".into(), + path: "/v2/".into(), + status: 401, + ..LogRow::default() + }]; + let back: Vec = decode(encode(&log).unwrap().into()).unwrap(); + assert_eq!(back, log); + + let none: Vec = decode(encode::(&[]).unwrap().into()).unwrap(); + assert!(none.is_empty()); + } +} diff --git a/crates/traffic/src/settings.rs b/crates/traffic/src/settings.rs new file mode 100644 index 00000000..42754dfa --- /dev/null +++ b/crates/traffic/src/settings.rs @@ -0,0 +1,65 @@ +//! What the statistics record and how long they are kept. + +/// The longest retention accepted, in days (100 years). +pub const MAX_RETENTION_DAYS: u32 = 36_500; + +/// The traffic settings of every instance, set in the web UI (Admin → +/// Traffic) or through `/admin/settings/traffic` and kept in the database. +/// An install that never saved them uses [`TrafficSettings::default`]. +/// +/// Read leniently, like the other stored settings: a key left out takes its +/// default and one this version does not know is ignored. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(default)] +pub struct TrafficSettings { + /// Whether requests are counted at all. Off, nothing is recorded or + /// written; what was written stays until it expires. + pub enabled: bool, + /// Whether client addresses are recorded: the top clients and the + /// request log's addresses. + pub client_addresses: bool, + /// Whether every request is written to the request log. + pub request_log: bool, + /// How long the counts by hour are kept, in days; `0` keeps them. + pub statistics_retention_days: u32, + /// How long the top clients are kept, in days. + pub client_retention_days: u32, + /// How long the request log is kept, in days. + pub request_log_retention_days: u32, +} + +impl Default for TrafficSettings { + fn default() -> Self { + TrafficSettings { + enabled: true, + client_addresses: true, + request_log: true, + statistics_retention_days: 400, + client_retention_days: 30, + request_log_retention_days: 7, + } + } +} + +impl TrafficSettings { + /// What is wrong with these settings, if anything. + pub fn check(&self) -> Result<(), String> { + if self.statistics_retention_days > MAX_RETENTION_DAYS { + return Err(format!( + "statistics_retention_days must be at most {MAX_RETENTION_DAYS}, or 0 to keep the statistics" + )); + } + for (name, days) in [ + ("client_retention_days", self.client_retention_days), + ( + "request_log_retention_days", + self.request_log_retention_days, + ), + ] { + if !(1..=MAX_RETENTION_DAYS).contains(&days) { + return Err(format!("{name} must be between 1 and {MAX_RETENTION_DAYS}")); + } + } + Ok(()) + } +} diff --git a/crates/traffic/src/slot.rs b/crates/traffic/src/slot.rs new file mode 100644 index 00000000..f565a502 --- /dev/null +++ b/crates/traffic/src/slot.rs @@ -0,0 +1,112 @@ +//! What the layers of a request learn about it, for its count: the client +//! address, the account it ran as and the repository it addressed. +//! +//! The outermost middleware opens a [`Slot`] around the request +//! ([`Slot::scope`]); code running in the request's task fills it in with +//! [`note_client`], [`note_account`] and [`note_repository`] without the +//! values being threaded through every handler. Outside a slot (a worker, a +//! test calling a handler directly) noting does nothing. + +use std::future::Future; +use std::net::IpAddr; +use std::sync::{Arc, Mutex, PoisonError}; + +tokio::task_local! { + static SLOT: Arc>; +} + +/// The account a request ran as, as the statistics name it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Account { + /// How the caller authenticated: `anonymous`, `session`, `token`, + /// `password`, `trusted_header` or `workload`. + pub kind: &'static str, + /// The user, for the kinds that have one. + pub user_id: Option, +} + +/// What was noted about a request. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Noted { + /// The client address: the socket peer, or the one a trusted proxy + /// forwarded. + pub client: Option, + /// The network the address is counted under (the address itself, an + /// IPv6 address's /64), as text. + pub network: Option, + /// The account the request ran as. + pub account: Option, + /// The repository the request addressed, once it was found. + pub repository: Option, +} + +/// The slot of one request. +#[derive(Debug, Clone, Default)] +pub struct Slot(Arc>); + +impl Slot { + /// An empty slot. + pub fn new() -> Slot { + Slot::default() + } + + /// Run `fut` with this slot open: what it notes lands here. + pub async fn scope(&self, fut: F) -> F::Output { + SLOT.scope(self.0.clone(), fut).await + } + + /// What was noted so far. + pub fn take(&self) -> Noted { + std::mem::take(&mut *self.0.lock().unwrap_or_else(PoisonError::into_inner)) + } +} + +fn note(f: impl FnOnce(&mut Noted)) { + let _ = SLOT.try_with(|slot| f(&mut slot.lock().unwrap_or_else(PoisonError::into_inner))); +} + +/// Note the request's client `address`, counted under `network`. +pub fn note_client(address: IpAddr, network: String) { + note(|n| { + n.client = Some(address); + n.network = Some(network); + }); +} + +/// Note the account the request runs as. A later note replaces an earlier +/// one (the registry's own token names the caller after the principal layer +/// found none). +pub fn note_account(account: Account) { + note(|n| n.account = Some(account)); +} + +/// Note the repository the request addresses, once it is known to exist. +pub fn note_repository(name: &str) { + note(|n| n.repository = Some(name.to_string())); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn notes_land_in_the_open_slot_only() { + note_repository("outside"); + let slot = Slot::new(); + slot.scope(async { + note_client("192.0.2.7".parse().unwrap(), "192.0.2.7".into()); + tokio::task::yield_now().await; + note_repository("nuget"); + note_account(Account { + kind: "token", + user_id: Some(4), + }); + }) + .await; + let noted = slot.take(); + assert_eq!(noted.repository.as_deref(), Some("nuget")); + assert_eq!(noted.network.as_deref(), Some("192.0.2.7")); + assert_eq!(noted.account.unwrap().user_id, Some(4)); + assert_eq!(slot.take(), Noted::default(), "taken once"); + } +} diff --git a/crates/traffic/src/top.rs b/crates/traffic/src/top.rs new file mode 100644 index 00000000..9ca5b4cc --- /dev/null +++ b/crates/traffic/src/top.rs @@ -0,0 +1,190 @@ +//! The clients with the most requests, counted in bounded memory. +//! +//! [`TopClients`] keeps at most twice its capacity of networks; when full, +//! it drops the half with the fewest requests. Whatever the number of +//! addresses a flood comes from, the memory stays bounded, and a network +//! with many requests stays in: it is dropped only while it has fewer +//! requests than half the networks kept. A network dropped and seen again +//! starts counting from zero, so a count can only be low, never high. + +use std::collections::HashMap; + +/// A client network's counts. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ClientCounts { + pub requests: u64, + pub bytes: u64, + /// 4xx answers (the next three included). + pub client_errors: u64, + /// 401 and 403 answers. + pub denied: u64, + pub not_found: u64, + /// 429 answers. + pub throttled: u64, + /// 5xx answers. + pub server_errors: u64, + /// The last address of the network seen. + pub address: String, + /// The last user agent seen; `""` for none. + pub user_agent: String, + /// The last account kind seen; `""` for none. + pub account: String, + /// The last account's user. + pub user_id: Option, +} + +impl ClientCounts { + /// Count one request answered with `status`. + pub fn observe(&mut self, status: u16, bytes: u64) { + self.requests += 1; + self.bytes += bytes; + match status { + 401 | 403 => self.denied += 1, + 404 => self.not_found += 1, + 429 => self.throttled += 1, + _ => {} + } + match status { + 400..=499 => self.client_errors += 1, + 500..=599 => self.server_errors += 1, + _ => {} + } + } + + /// Add `other`'s counts; its last-seen values win where it has them. + pub fn add(&mut self, other: &ClientCounts) { + self.requests += other.requests; + self.bytes += other.bytes; + self.client_errors += other.client_errors; + self.denied += other.denied; + self.not_found += other.not_found; + self.throttled += other.throttled; + self.server_errors += other.server_errors; + if !other.address.is_empty() { + self.address.clone_from(&other.address); + } + if !other.user_agent.is_empty() { + self.user_agent.clone_from(&other.user_agent); + } + if !other.account.is_empty() { + self.account.clone_from(&other.account); + self.user_id = other.user_id; + } + } +} + +/// The networks with the most requests, about `capacity` of them. +#[derive(Debug, Clone)] +pub struct TopClients { + capacity: usize, + counts: HashMap, +} + +impl TopClients { + /// Keep about `capacity` networks (at least one). + pub fn new(capacity: usize) -> TopClients { + TopClients { + capacity: capacity.max(1), + counts: HashMap::new(), + } + } + + /// The counts of `network`, added when new. + pub fn entry(&mut self, network: &str) -> &mut ClientCounts { + if !self.counts.contains_key(network) { + if self.counts.len() >= self.capacity * 2 { + self.shrink(); + } + self.counts + .insert(network.to_string(), ClientCounts::default()); + } + self.counts.get_mut(network).expect("just inserted") + } + + /// Add all of `other`'s networks. + pub fn merge(&mut self, other: &TopClients) { + for (network, counts) in &other.counts { + self.entry(network).add(counts); + } + } + + /// Whether no network was counted. + pub fn is_empty(&self) -> bool { + self.counts.is_empty() + } + + /// The `n` networks with the most requests, most first (ties by + /// network, for a stable order). + pub fn top(&self, n: usize) -> Vec<(&str, &ClientCounts)> { + let mut all: Vec<_> = self + .counts + .iter() + .map(|(network, counts)| (network.as_str(), counts)) + .collect(); + all.sort_by(|a, b| b.1.requests.cmp(&a.1.requests).then(a.0.cmp(b.0))); + all.truncate(n); + all + } + + /// Keep only the `n` networks with the most requests. + pub fn truncate(&mut self, n: usize) { + if self.counts.len() > n { + let keep: Vec = self + .top(n) + .into_iter() + .map(|(k, _)| k.to_string()) + .collect(); + let mut kept = HashMap::with_capacity(keep.len()); + for network in keep { + if let Some(counts) = self.counts.remove(&network) { + kept.insert(network, counts); + } + } + self.counts = kept; + } + } + + fn shrink(&mut self) { + self.truncate(self.capacity); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn heavy_clients_survive_a_flood_of_light_ones() { + let mut top = TopClients::new(8); + for round in 0..1_000 { + for heavy in ["192.0.2.1", "192.0.2.2"] { + top.entry(heavy).observe(200, 10); + } + top.entry(&format!("10.0.{}.{}", round / 250, round % 250)) + .observe(404, 1); + } + let best = top.top(2); + assert_eq!(best[0], ("192.0.2.1", &best[0].1.clone())); + assert_eq!(best[0].1.requests, 1_000); + assert_eq!(best[1].0, "192.0.2.2"); + assert!(top.counts.len() <= 16, "bounded: {}", top.counts.len()); + } + + #[test] + fn status_classes_are_counted() { + let mut c = ClientCounts::default(); + for status in [200, 401, 403, 404, 429, 500, 503, 302] { + c.observe(status, 0); + } + assert_eq!( + ( + c.client_errors, + c.denied, + c.not_found, + c.throttled, + c.server_errors + ), + (4, 2, 1, 1, 2) + ); + } +} diff --git a/docs/changes/2.2.0/709.feature.md b/docs/changes/2.2.0/709.feature.md new file mode 100644 index 00000000..67a08546 --- /dev/null +++ b/docs/changes/2.2.0/709.feature.md @@ -0,0 +1,12 @@ +**Admin → Traffic** shows what every instance answered: requests over time, +data sent, 4xx and 5xx answers and latency, by repository, surface and +status, and the 20 client networks with the most requests. Each client opens +its own view with its requests from a request log. Every instance writes its +counts to Forge's bucket once a minute, as Parquet files under `traffic/v1/` +that DuckDB or pandas read too, so the page covers all instances without +Redis. What is recorded (client addresses and the request log can be +switched off) and how long each part is kept (statistics 400 days, top +clients 30, request log 7 by default) is set under **Admin → Traffic → +Settings** (`/admin/settings/traffic`). The listing of these files makes +Forge call `ListObjectsV2`, which the documented `s3:ListBucket` permission +already allows. diff --git a/docs/deployment.md b/docs/deployment.md index d27bd9dd..c8840a01 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -69,7 +69,9 @@ for the bucket `forge`; MinIO and Ceph RGW use the same action names): These are the S3 calls behind the policy: `HeadBucket`, `CreateBucket` (only when the bucket is missing), `HeadObject`, `GetObject`, `PutObject`, `DeleteObject`, `CreateMultipartUpload`, `UploadPart`, -`CompleteMultipartUpload` and `AbortMultipartUpload`. The policy is the same +`CompleteMultipartUpload`, `AbortMultipartUpload` and `ListObjectsV2` (the +[traffic statistics](#traffic-statistics) list their files, under +`traffic/`). The policy is the same for static keys and for the role behind short-lived credentials. ### Short-lived S3 credentials @@ -1297,6 +1299,54 @@ console log is) is exported too, each record with the trace and span it was written in, so a backend links a request's log lines to its trace. The console log stays as it is. +## Traffic statistics + +**Admin → Traffic** (`/manage/traffic`) shows what every instance answered: +requests over time, data sent, 4xx and 5xx answers and latency, by +repository, surface and status, and the 20 client networks with the most +requests. A client opens its own view with its requests from the request +log: time, method, path, status, size, duration, account, user agent and +the instance that answered. Health probes are not counted. + +Each instance counts its requests in memory and writes them to Forge's +bucket once a minute, as Parquet files under `traffic/v1/`. Every instance +writes files of its own, so the page reads the files of all of them from the +bucket and sums them: it needs no Redis and no call between instances, and +it shows every instance, or one, whichever answers it. The files are plain +Parquet, so DuckDB, pandas or Spark read them too: + +| Files | What they hold | Kept | +| --- | --- | --- | +| `minutes///.buckets.parquet` | An hour's requests by minute, repository, surface, method and status: count, bytes, a latency histogram | 3 days (the views of the last 48 hours) | +| `hours//.buckets.parquet` | The same of a day, by hour | **Statistics**, 400 days by default | +| `…/.clients.parquet` (both) | The top 100 client networks of each minute or hour | **Top clients**, 30 days by default | +| `requests///..parquet` | The request log: a file per instance and minute | **Request log**, 7 days by default | + +What is recorded and how long it is kept is set under **Admin → Traffic → +Settings** (`/admin/settings/traffic`), for every instance at once. Each +instance deletes the files past their retention about once an hour, +whichever instance wrote them. + +- **Client addresses are personal data.** The address is the one Forge + resolves for the login lockout: the socket peer, or the `X-Forwarded-For` + of a [trusted proxy](#reverse-proxy). An IPv6 client is counted by its + /64. Addresses are kept only in the top clients and the request log, with + retentions of their own; **Client addresses** off records neither, and + **Request log** off keeps the counts and the top clients without the log. +- **Size.** The counts are a few small files per instance and hour. The + request log grows with the traffic: a row per request, with its path + (secrets in a path, such as a command-line login's, are replaced) and user + agent, compressed with Snappy. +- **Accuracy.** The last minute shows once the instances have written it. + An instance that stops writes what it counted on a graceful shutdown; one + that is killed loses its last minute. The top lists are counted in bounded + memory: under a flood of addresses a client's count can come out low, + never high. A client's view reads the request log of at most the last 24 + hours of its range, and shows the newest 500 requests. + +The statistics are separate from the [OpenTelemetry metrics](#telemetry), +which carry no address, path or user and need a backend of their own. + ## Running more than one instance Instances coordinate through PostgreSQL (row locks and leases), so any number @@ -1578,7 +1628,8 @@ incomplete multipart uploads, for example after one day: (`aws s3api put-bucket-lifecycle-configuration --bucket forge --lifecycle-configuration file://lifecycle.json`; MinIO and Ceph RGW accept the same rule.) Do not add expiration rules for current objects: Forge's own -garbage collection decides when a blob may go. +garbage collection decides when a blob may go, and Forge deletes the +[traffic statistics](#traffic-statistics) past their retention itself. To restore: stop every Forge instance, restore PostgreSQL, restore the bucket as described above, then start Forge. Practise this before you need it. diff --git a/web/src/api/traffic.ts b/web/src/api/traffic.ts new file mode 100644 index 00000000..3ac30293 --- /dev/null +++ b/web/src/api/traffic.ts @@ -0,0 +1,233 @@ +import { httpClient } from '~/lib/httpClient' + +/** + * Admin traffic API: the traffic statistics every instance writes to the + * object store (`crates/server/src/admin/traffic.rs`, `crates/traffic`). + * + * - `GET /admin/traffic` answers the traffic of a range over every instance + * (or the one `instance` names): counts by time bucket, surface, + * repository and status, latency, and the top 20 client networks. + * - `GET /admin/traffic/client` answers one client network's counts and + * its requests in the request log (the last 24 hours of the range at + * most, the newest 500). + * - `GET` / `PUT /admin/settings/traffic`: what is recorded and how long + * it is kept; `PUT` changes the fields it names. + */ + +/** The ranges the views offer. */ +export type TrafficRange = '1h' | '6h' | '24h' | '7d' | '30d' | '90d' + +export const TRAFFIC_RANGES: { id: TrafficRange; label: string }[] = [ + { id: '1h', label: '1 hour' }, + { id: '6h', label: '6 hours' }, + { id: '24h', label: '24 hours' }, + { id: '7d', label: '7 days' }, + { id: '30d', label: '30 days' }, + { id: '90d', label: '90 days' }, +] + +export function isTrafficRange(value: unknown): value is TrafficRange { + return TRAFFIC_RANGES.some((r) => r.id === value) +} + +/** An instance whose files the range holds. */ +export interface TrafficInstance { + id: string + host: string + started_at: string | null +} + +/** A point of a series: the counts of one step. */ +export interface TrafficPoint { + start: string + requests: number + bytes: number + client_errors: number + server_errors: number +} + +/** What is recorded now (the settings), so a view can say why data is missing. */ +export interface TrafficRecording { + enabled: boolean + client_addresses: boolean + request_log: boolean +} + +/** The span a view covers. */ +interface TrafficSpan { + from: string + to: string + step_seconds: number + /** What the files read count by. */ + resolution: 'minute' | 'hour' +} + +export interface TrafficTotals { + requests: number + bytes: number + /** 4xx answers (the next three included). */ + client_errors: number + /** 401 and 403 answers. */ + denied: number + not_found: number + /** 429 answers. */ + throttled: number + /** 5xx answers. */ + server_errors: number + mean_ms: number | null + /** The latency bound half, 95 % and 99 % of the requests stayed within. */ + p50_ms: number | null + p95_ms: number | null + p99_ms: number | null +} + +/** A client network's counts. */ +export interface TrafficClient { + network: string + address: string + requests: number + bytes: number + client_errors: number + denied: number + not_found: number + throttled: number + server_errors: number + user_agent: string | null + account: string | null + user_id: number | null + username: string | null + hosts: string[] + first_seen: string | null + last_seen: string | null +} + +/** `GET /admin/traffic`. */ +export interface TrafficOverview extends TrafficSpan { + instances: TrafficInstance[] + totals: TrafficTotals + series: TrafficPoint[] + surfaces: { surface: string; requests: number; bytes: number }[] + repositories: { + name: string + requests: number + bytes: number + client_errors: number + server_errors: number + }[] + statuses: { status: number; requests: number }[] + clients: TrafficClient[] + recording: TrafficRecording +} + +/** A request of the request log. */ +export interface TrafficLogEntry { + time: string + address: string + method: string + path: string + surface: string + repository: string | null + status: number + bytes: number + duration_ms: number + user_agent: string | null + account: string | null + user_id: number | null + username: string | null + host: string +} + +/** How often a value occurs in a client's log. */ +export interface TrafficTally { + value: string + requests: number +} + +/** How often an account occurs in a client's log. */ +export interface TrafficAccountTally { + account: string | null + user_id: number | null + username: string | null + requests: number +} + +/** `GET /admin/traffic/client`. */ +export interface TrafficClientDetail extends TrafficSpan { + network: string + instances: TrafficInstance[] + /** Its counts in the time buckets it was among the top clients of. */ + totals: TrafficClient | null + series: TrafficPoint[] + log: { + from: string + to: string + matched: number + entries: TrafficLogEntry[] + paths: TrafficTally[] + user_agents: TrafficTally[] + accounts: TrafficAccountTally[] + statuses: TrafficTally[] + } + recording: TrafficRecording +} + +/** `GET /admin/settings/traffic`. */ +export interface TrafficSettings { + enabled: boolean + client_addresses: boolean + request_log: boolean + /** Days the counts by hour are kept; `0` keeps them. */ + statistics_retention_days: number + client_retention_days: number + request_log_retention_days: number +} + +/** `PUT /admin/settings/traffic` body: the fields to change. */ +export type TrafficSettingsUpdate = Partial + +/** What a view asks for. */ +export interface TrafficQuery { + range: TrafficRange + instance?: string + repository?: string +} + +export const trafficKeys = { + all: ['traffic'] as const, + overview: (q: TrafficQuery) => ['traffic', 'overview', q] as const, + client: (network: string, q: TrafficQuery) => ['traffic', 'client', network, q] as const, + settings: () => ['settings', 'traffic'] as const, +} + +/** `GET /admin/traffic`. */ +export function fetchTrafficOverview( + q: TrafficQuery, + signal?: AbortSignal, +): Promise { + return httpClient.get('/admin/traffic', { + signal, + query: { range: q.range, instance: q.instance, repository: q.repository }, + }) +} + +/** `GET /admin/traffic/client`. */ +export function fetchTrafficClient( + network: string, + q: TrafficQuery, + signal?: AbortSignal, +): Promise { + return httpClient.get('/admin/traffic/client', { + signal, + query: { network, range: q.range, instance: q.instance }, + }) +} + +/** `GET /admin/settings/traffic`. */ +export function fetchTrafficSettings(signal?: AbortSignal): Promise { + return httpClient.get('/admin/settings/traffic', { signal }) +} + +/** `PUT /admin/settings/traffic`. */ +export function updateTrafficSettings(body: TrafficSettingsUpdate): Promise { + return httpClient.put('/admin/settings/traffic', body) +} diff --git a/web/src/components/AppRail.tsx b/web/src/components/AppRail.tsx index 6c417049..5c1c002f 100644 --- a/web/src/components/AppRail.tsx +++ b/web/src/components/AppRail.tsx @@ -28,6 +28,7 @@ const ADMINISTRATION: RailLink[] = [ { label: 'Settings', to: '/manage/settings', icon: 'Sliders' }, { label: 'Email', to: '/manage/settings/email', icon: 'Mail' }, { label: 'Retention', to: '/manage/settings/retention', icon: 'Clock' }, + { label: 'Traffic', to: '/manage/traffic', icon: 'Activity' }, { label: 'System', to: '/manage/system', icon: 'Database' }, ] diff --git a/web/src/components/TrafficChart.tsx b/web/src/components/TrafficChart.tsx new file mode 100644 index 00000000..9afa07b9 --- /dev/null +++ b/web/src/components/TrafficChart.tsx @@ -0,0 +1,249 @@ +import { useEffect, useId, useRef, useState, type KeyboardEvent, type PointerEvent } from 'react' + +import type { TrafficPoint } from '~/api/traffic' +import { formatBytes } from '~/lib/oci' +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '~/components/ui' + +/** What the chart plots of each point. */ +export type TrafficMetric = 'requests' | 'bytes' | 'client_errors' | 'server_errors' + +export const TRAFFIC_METRICS: { id: TrafficMetric; label: string }[] = [ + { id: 'requests', label: 'Requests' }, + { id: 'bytes', label: 'Data sent' }, + { id: 'client_errors', label: '4xx' }, + { id: 'server_errors', label: '5xx' }, +] + +const METRIC_NAMES: Record = { + requests: 'Requests', + bytes: 'Data sent', + client_errors: '4xx answers', + server_errors: '5xx answers', +} + +const compact = new Intl.NumberFormat(undefined, { notation: 'compact', maximumFractionDigits: 1 }) +const full = new Intl.NumberFormat() + +/** `n` as a metric's value: bytes in binary units, counts in full. */ +export function formatMetric(metric: TrafficMetric, n: number): string { + return metric === 'bytes' ? formatBytes(n) : full.format(n) +} + +/** An axis tick: compact once the axis reaches 10,000. */ +function formatTick(metric: TrafficMetric, n: number, max: number): string { + if (metric === 'bytes') return formatBytes(n) + return max >= 10_000 ? compact.format(n) : full.format(n) +} + +/** About `count` round ticks from 0 to at least `max`. */ +function niceTicks(max: number, count = 4): number[] { + if (max <= 0) return [0, 1] + const raw = max / count + const power = 10 ** Math.floor(Math.log10(raw)) + const step = [1, 2, 2.5, 5, 10].map((m) => m * power).find((s) => s >= raw) ?? raw + const ticks: number[] = [] + for (let t = 0; t <= max + step * 0.999; t += step) ticks.push(Math.round(t * 1e6) / 1e6) + return ticks +} + +/** A point's time as the axis and the tooltip say it. */ +function timeLabel(iso: string, stepSeconds: number, withDate: boolean): string { + const d = new Date(iso) + if (stepSeconds >= 86_400) return d.toLocaleDateString(undefined, { month: 'short', day: 'numeric' }) + const time = d.toLocaleTimeString(undefined, { hour: '2-digit', minute: '2-digit' }) + return withDate + ? `${d.toLocaleDateString(undefined, { month: 'short', day: 'numeric' })} ${time}` + : time +} + +/** The intervals time labels may be apart, in seconds. */ +const LABEL_INTERVALS = [ + 60, 300, 600, 900, 1_800, 3_600, 7_200, 10_800, 21_600, 43_200, 86_400, 172_800, 604_800, +] + +const HEIGHT = 220 +const MARGIN = { top: 12, right: 12, bottom: 28, left: 64 } + +/** + * One series of a traffic view over time: an area with a 2px line, a + * crosshair that snaps to the nearest point and a tooltip with its value + * (the arrow keys step through the points), or the same points as a table. + * Drawn at the width it is given, so its text keeps its size. + */ +export function TrafficChart({ + points, + stepSeconds, + metric, + view, +}: { + points: TrafficPoint[] + stepSeconds: number + metric: TrafficMetric + view: 'chart' | 'table' +}) { + if (view === 'table') { + return + } + return +} + +function AreaChart({ + points, + stepSeconds, + metric, +}: { + points: TrafficPoint[] + stepSeconds: number + metric: TrafficMetric +}) { + const box = useRef(null) + const [width, setWidth] = useState(720) + const [active, setActive] = useState(null) + const tipId = useId() + + useEffect(() => { + const el = box.current + if (!el || typeof ResizeObserver === 'undefined') return + const observer = new ResizeObserver(([entry]) => { + const w = Math.round(entry.contentRect.width) + if (w > 0) setWidth(w) + }) + observer.observe(el) + return () => observer.disconnect() + }, []) + + const values = points.map((p) => p[metric]) + const max = Math.max(0, ...values) + const ticks = niceTicks(max) + const top = ticks[ticks.length - 1] || 1 + const plotW = Math.max(width - MARGIN.left - MARGIN.right, 1) + const plotH = HEIGHT - MARGIN.top - MARGIN.bottom + const x = (i: number) => + MARGIN.left + (points.length <= 1 ? plotW / 2 : (i / (points.length - 1)) * plotW) + const y = (v: number) => MARGIN.top + plotH - (v / top) * plotH + const line = points.map((_, i) => `${i === 0 ? 'M' : 'L'}${x(i)},${y(values[i])}`).join('') + const area = + points.length > 0 + ? `${line}L${x(points.length - 1)},${y(0)}L${x(0)},${y(0)}Z` + : '' + const multiDay = + points.length > 1 && + new Date(points[0].start).toDateString() !== + new Date(points[points.length - 1].start).toDateString() + // Room for a time label: about 6.5px a character, and a gap. + const withDate = multiDay && stepSeconds < 86_400 + const labelWidth = (points[0] ? timeLabel(points[0].start, stepSeconds, withDate).length : 5) * 6.5 + 24 + const perPoint = points.length <= 1 ? plotW : plotW / (points.length - 1) + // Labels on round times (every 3 hours, every day), as far apart as they need. + const wanted = Math.max(1, Math.ceil(labelWidth / perPoint)) * stepSeconds + const interval = + LABEL_INTERVALS.find((s) => s >= wanted && s % stepSeconds === 0) ?? wanted + const labelled = (iso: string) => (Date.parse(iso) / 1000) % interval === 0 + const total = values.reduce((a, b) => a + b, 0) + + function nearest(e: PointerEvent): number | null { + if (points.length === 0) return null + const rect = e.currentTarget.getBoundingClientRect() + const px = e.clientX - rect.left - MARGIN.left + const i = points.length <= 1 ? 0 : Math.round((px / plotW) * (points.length - 1)) + return Math.min(points.length - 1, Math.max(0, i)) + } + + function onKey(e: KeyboardEvent) { + if (points.length === 0) return + const step = { ArrowLeft: -1, ArrowRight: 1, Home: -Infinity, End: Infinity }[e.key] + if (step === undefined) return + e.preventDefault() + const from = active ?? (step > 0 ? -1 : points.length) + setActive(Math.min(points.length - 1, Math.max(0, from + step))) + } + + const tip = active !== null ? points[active] : null + const tipLeft = active !== null ? Math.min(Math.max(x(active), 80), width - 80) : 0 + + return ( +
+ setActive(nearest(e))} + onPointerLeave={() => setActive(null)} + onBlur={() => setActive(null)} + onKeyDown={onKey} + > + {ticks.map((t) => ( + + + + {formatTick(metric, t, top)} + + + ))} + {points.map((p, i) => + labelled(p.start) ? ( + + {timeLabel(p.start, stepSeconds, withDate)} + + ) : null, + )} + + + {active !== null && tip && ( + + + + + )} + + {tip && active !== null && ( +
+
+
+
+ {METRIC_NAMES[metric]}, {timeLabel(tip.start, stepSeconds, true)} +
+
+ )} +
+ ) +} + +/** The points of the chart as rows, newest first. */ +function PointTable({ + points, + stepSeconds, + metric, +}: { + points: TrafficPoint[] + stepSeconds: number + metric: TrafficMetric +}) { + return ( + + + From + Requests + Data sent + 4xx + 5xx + + + {[...points].reverse().map((p) => ( + + {timeLabel(p.start, stepSeconds, true)} + {full.format(p.requests)} + {formatBytes(p.bytes)} + {full.format(p.client_errors)} + {full.format(p.server_errors)} + + ))} + +
+ ) +} diff --git a/web/src/components/TrafficFilters.tsx b/web/src/components/TrafficFilters.tsx new file mode 100644 index 00000000..72f576ac --- /dev/null +++ b/web/src/components/TrafficFilters.tsx @@ -0,0 +1,50 @@ +import { TRAFFIC_RANGES, type TrafficInstance, type TrafficRange } from '~/api/traffic' +import { formatDateTime } from '~/lib/oci' +import { SegmentedControl, Select } from '~/components/ui' + +/** + * The filters of a traffic view, in one row above what they scope: the + * range first, then the instance (all of them, or one). + */ +export function TrafficFilters({ + range, + onRange, + instance, + instances, + onInstance, +}: { + range: TrafficRange + onRange: (range: TrafficRange) => void + instance: string | undefined + instances: TrafficInstance[] + onInstance: (instance: string | undefined) => void +}) { + // An instance named in the URL stays selectable when the range holds no + // file of it. + const known = + instance && !instances.some((i) => i.id === instance) + ? [...instances, { id: instance, host: instance, started_at: null }] + : instances + return ( +
+ ({ id: r.id, label: r.label }))} + /> + +
+ ) +} diff --git a/web/src/components/ui/icons.tsx b/web/src/components/ui/icons.tsx index f72fbbeb..0272eb0c 100644 --- a/web/src/components/ui/icons.tsx +++ b/web/src/components/ui/icons.tsx @@ -81,6 +81,7 @@ const ICONS = { Eye: [EYE, circ(10, 10, 2.5)], EyeOff: [EYE, circ(10, 10, 2.5), 'M3 3l14 14'], Clock: [circ(10, 10, 7), 'M10 6v4l2.5 2.5'], + Activity: ['M2.5 10.5h3l2.2-5.5 4.6 10 2.2-4.5h3'], Alert: ['M10 3l8 14H2z', 'M10 8.5V12', 'M10 14.6v.1'], Info: [circ(10, 10, 7), 'M10 9v5', 'M10 6.4v.1'], X: ['M5 5l10 10', 'M15 5L5 15'], diff --git a/web/src/lib/traffic.ts b/web/src/lib/traffic.ts new file mode 100644 index 00000000..8d15194d --- /dev/null +++ b/web/src/lib/traffic.ts @@ -0,0 +1,86 @@ +import type { Tone } from '~/components/ui' + +/** + * How the traffic views name what the statistics count: surfaces, accounts, + * statuses, instances. + */ + +const SURFACES: Record = { + oci: 'Docker registry', + nuget: 'NuGet', + plugin_npm: 'npm', + plugin_raw: 'Raw', + plugin_other: 'Other formats', + repository: 'Unknown repositories', + admin: 'Admin API', + browse: 'Browse API', + auth: 'Sign-in and tokens', + ui: 'Web UI', +} + +/** A surface as people call it (`plugin_npm` → npm). */ +export function surfaceLabel(surface: string): string { + return SURFACES[surface] ?? surface +} + +const ACCOUNT_KINDS: Record = { + anonymous: 'anonymous', + session: 'web session', + token: 'API token', + password: 'password', + trusted_header: 'trusted proxy', + workload: 'workload identity', +} + +/** Who a request ran as: `alice (API token)`, `anonymous`. */ +export function accountLabel( + account: string | null, + username: string | null, + userId: number | null, +): string { + const kind = account ? (ACCOUNT_KINDS[account] ?? account) : null + const who = username ?? (userId != null ? `user ${userId}` : null) + if (who && kind) return `${who} (${kind})` + return who ?? kind ?? '—' +} + +/** The tone of a status: 2xx ok, 3xx neutral, 4xx warn, 5xx danger. */ +export function statusTone(status: number): Tone { + if (status >= 500) return 'danger' + if (status >= 400) return 'warn' + if (status >= 300) return 'neutral' + return 'ok' +} + +/** What the series' points are: "per minute", "per 5 minutes", "per day". */ +export function stepLabel(seconds: number): string { + if (seconds % 86_400 === 0) return seconds === 86_400 ? 'per day' : `per ${seconds / 86_400} days` + if (seconds % 3_600 === 0) return seconds === 3_600 ? 'per hour' : `per ${seconds / 3_600} hours` + return seconds === 60 ? 'per minute' : `per ${seconds / 60} minutes` +} + +const percent = new Intl.NumberFormat(undefined, { style: 'percent', maximumFractionDigits: 1 }) + +/** `part` of `whole` as a percentage; an em dash without a whole. */ +export function share(part: number, whole: number): string { + return whole > 0 ? percent.format(part / whole) : '—' +} + +function bound(ms: number): string { + return ms >= 1_000 ? `${ms / 1_000} s` : `${ms} ms` +} + +/** + * How fast the requests were answered, from the latency histogram's bounds: + * `95 % answered within 250 ms, half within 10 ms`; `null` without requests. + */ +export function latencySummary(totals: { + requests: number + p50_ms: number | null + p95_ms: number | null +}): string | null { + if (totals.requests === 0) return null + const p95 = totals.p95_ms == null ? '5 % took over 60 s' : `95 % answered within ${bound(totals.p95_ms)}` + const p50 = totals.p50_ms == null ? 'half took over 60 s' : `half within ${bound(totals.p50_ms)}` + return `${p95}, ${p50}` +} diff --git a/web/src/queries/useTraffic.ts b/web/src/queries/useTraffic.ts new file mode 100644 index 00000000..824d970c --- /dev/null +++ b/web/src/queries/useTraffic.ts @@ -0,0 +1,65 @@ +import { keepPreviousData, useMutation, useQuery, useQueryClient } from '@tanstack/react-query' + +import type { ApiError } from '~/lib/httpClient' +import { + fetchTrafficClient, + fetchTrafficOverview, + fetchTrafficSettings, + trafficKeys, + updateTrafficSettings, + type TrafficClientDetail, + type TrafficOverview, + type TrafficQuery, + type TrafficSettings, + type TrafficSettingsUpdate, +} from '~/api/traffic' + +/** How often an open traffic view is read again: the files change every minute. */ +const REFRESH_MS = 60_000 + +/** + * The traffic of a range (`GET /admin/traffic`). While another range or + * instance loads, the previous answer stays on screen. + */ +export function useTrafficOverview(q: TrafficQuery) { + return useQuery({ + queryKey: trafficKeys.overview(q), + queryFn: ({ signal }) => fetchTrafficOverview(q, signal), + placeholderData: keepPreviousData, + refetchInterval: REFRESH_MS, + }) +} + +/** One client network's traffic and request log (`GET /admin/traffic/client`). */ +export function useTrafficClient(network: string, q: TrafficQuery) { + return useQuery({ + queryKey: trafficKeys.client(network, q), + queryFn: ({ signal }) => fetchTrafficClient(network, q, signal), + placeholderData: keepPreviousData, + refetchInterval: REFRESH_MS, + enabled: network.length > 0, + }) +} + +/** What is recorded and how long it is kept (`GET /admin/settings/traffic`). */ +export function useTrafficSettings() { + return useQuery({ + queryKey: trafficKeys.settings(), + queryFn: ({ signal }) => fetchTrafficSettings(signal), + }) +} + +/** + * Change the traffic settings; the answer replaces the cached ones, and the + * traffic views (which say what is recorded) are read again. + */ +export function useUpdateTrafficSettings() { + const queryClient = useQueryClient() + return useMutation({ + mutationFn: (body) => updateTrafficSettings(body), + onSuccess: (saved) => { + queryClient.setQueryData(trafficKeys.settings(), saved) + void queryClient.invalidateQueries({ queryKey: trafficKeys.all }) + }, + }) +} diff --git a/web/src/routeTree.gen.ts b/web/src/routeTree.gen.ts index a87b8c8a..4c7f49fd 100644 --- a/web/src/routeTree.gen.ts +++ b/web/src/routeTree.gen.ts @@ -24,6 +24,7 @@ import { Route as adminRolesListRouteImport } from './routes/admin/roles/List' import { Route as adminServiceAccountsListRouteImport } from './routes/admin/service-accounts/List' import { Route as adminSettingsSettingsRouteImport } from './routes/admin/settings/Settings' import { Route as adminSystemSystemRouteImport } from './routes/admin/system/System' +import { Route as adminTrafficTrafficRouteImport } from './routes/admin/traffic/Traffic' import { Route as adminUsersListRouteImport } from './routes/admin/users/List' import { Route as RepoBrowseRouteImport } from './routes/RepoBrowse' import { Route as ImageDetailRouteImport } from './routes/ImageDetail' @@ -38,6 +39,8 @@ import { Route as adminServiceAccountsEditRouteImport } from './routes/admin/ser import { Route as adminServiceAccountsCreateRouteImport } from './routes/admin/service-accounts/Create' import { Route as adminSettingsEmailRouteImport } from './routes/admin/settings/Email' import { Route as adminSettingsRetentionSettingsRouteImport } from './routes/admin/settings/RetentionSettings' +import { Route as adminTrafficClientRouteImport } from './routes/admin/traffic/Client' +import { Route as adminTrafficSettingsRouteImport } from './routes/admin/traffic/Settings' import { Route as adminUsersEditRouteImport } from './routes/admin/users/Edit' import { Route as adminUsersCreateRouteImport } from './routes/admin/users/Create' import { Route as PackageDetailRouteImport } from './routes/PackageDetail' @@ -120,6 +123,11 @@ const adminSystemSystemRoute = adminSystemSystemRouteImport.update({ path: '/system', getParentRoute: () => AdminLayoutRoute, } as any) +const adminTrafficTrafficRoute = adminTrafficTrafficRouteImport.update({ + id: '/traffic', + path: '/traffic', + getParentRoute: () => AdminLayoutRoute, +} as any) const adminUsersListRoute = adminUsersListRouteImport.update({ id: '/users', path: '/users', @@ -193,6 +201,16 @@ const adminSettingsRetentionSettingsRoute = path: '/settings/retention', getParentRoute: () => AdminLayoutRoute, } as any) +const adminTrafficClientRoute = adminTrafficClientRouteImport.update({ + id: '/traffic/client', + path: '/traffic/client', + getParentRoute: () => AdminLayoutRoute, +} as any) +const adminTrafficSettingsRoute = adminTrafficSettingsRouteImport.update({ + id: '/traffic/settings', + path: '/traffic/settings', + getParentRoute: () => AdminLayoutRoute, +} as any) const adminUsersEditRoute = adminUsersEditRouteImport.update({ id: '/users/$userId', path: '/users/$userId', @@ -241,6 +259,7 @@ export interface FileRoutesByFullPath { '/manage/service-accounts': typeof adminServiceAccountsListRoute '/manage/settings': typeof adminSettingsSettingsRoute '/manage/system': typeof adminSystemSystemRoute + '/manage/traffic': typeof adminTrafficTrafficRoute '/manage/users': typeof adminUsersListRoute '/repos/$repo': typeof RepoBrowseRoute '/images/$repo/$': typeof ImageDetailRoute @@ -255,6 +274,8 @@ export interface FileRoutesByFullPath { '/manage/service-accounts/new': typeof adminServiceAccountsCreateRoute '/manage/settings/email': typeof adminSettingsEmailRoute '/manage/settings/retention': typeof adminSettingsRetentionSettingsRoute + '/manage/traffic/client': typeof adminTrafficClientRoute + '/manage/traffic/settings': typeof adminTrafficSettingsRoute '/manage/users/$userId': typeof adminUsersEditRoute '/manage/users/new': typeof adminUsersCreateRoute '/packages/$repo/$id': typeof PackageDetailRoute @@ -276,6 +297,7 @@ export interface FileRoutesByTo { '/manage/service-accounts': typeof adminServiceAccountsListRoute '/manage/settings': typeof adminSettingsSettingsRoute '/manage/system': typeof adminSystemSystemRoute + '/manage/traffic': typeof adminTrafficTrafficRoute '/manage/users': typeof adminUsersListRoute '/repos/$repo': typeof RepoBrowseRoute '/images/$repo/$': typeof ImageDetailRoute @@ -290,6 +312,8 @@ export interface FileRoutesByTo { '/manage/service-accounts/new': typeof adminServiceAccountsCreateRoute '/manage/settings/email': typeof adminSettingsEmailRoute '/manage/settings/retention': typeof adminSettingsRetentionSettingsRoute + '/manage/traffic/client': typeof adminTrafficClientRoute + '/manage/traffic/settings': typeof adminTrafficSettingsRoute '/manage/users/$userId': typeof adminUsersEditRoute '/manage/users/new': typeof adminUsersCreateRoute '/packages/$repo/$id': typeof PackageDetailRoute @@ -314,6 +338,7 @@ export interface FileRoutesById { '/_app/manage/service-accounts': typeof adminServiceAccountsListRoute '/_app/manage/settings': typeof adminSettingsSettingsRoute '/_app/manage/system': typeof adminSystemSystemRoute + '/_app/manage/traffic': typeof adminTrafficTrafficRoute '/_app/manage/users': typeof adminUsersListRoute '/_app/repos/$repo': typeof RepoBrowseRoute '/_app/images/$repo/$': typeof ImageDetailRoute @@ -328,6 +353,8 @@ export interface FileRoutesById { '/_app/manage/service-accounts/new': typeof adminServiceAccountsCreateRoute '/_app/manage/settings/email': typeof adminSettingsEmailRoute '/_app/manage/settings/retention': typeof adminSettingsRetentionSettingsRoute + '/_app/manage/traffic/client': typeof adminTrafficClientRoute + '/_app/manage/traffic/settings': typeof adminTrafficSettingsRoute '/_app/manage/users/$userId': typeof adminUsersEditRoute '/_app/manage/users/new': typeof adminUsersCreateRoute '/_app/packages/$repo/$id': typeof PackageDetailRoute @@ -352,6 +379,7 @@ export interface FileRouteTypes { | '/manage/service-accounts' | '/manage/settings' | '/manage/system' + | '/manage/traffic' | '/manage/users' | '/repos/$repo' | '/images/$repo/$' @@ -366,6 +394,8 @@ export interface FileRouteTypes { | '/manage/service-accounts/new' | '/manage/settings/email' | '/manage/settings/retention' + | '/manage/traffic/client' + | '/manage/traffic/settings' | '/manage/users/$userId' | '/manage/users/new' | '/packages/$repo/$id' @@ -387,6 +417,7 @@ export interface FileRouteTypes { | '/manage/service-accounts' | '/manage/settings' | '/manage/system' + | '/manage/traffic' | '/manage/users' | '/repos/$repo' | '/images/$repo/$' @@ -401,6 +432,8 @@ export interface FileRouteTypes { | '/manage/service-accounts/new' | '/manage/settings/email' | '/manage/settings/retention' + | '/manage/traffic/client' + | '/manage/traffic/settings' | '/manage/users/$userId' | '/manage/users/new' | '/packages/$repo/$id' @@ -424,6 +457,7 @@ export interface FileRouteTypes { | '/_app/manage/service-accounts' | '/_app/manage/settings' | '/_app/manage/system' + | '/_app/manage/traffic' | '/_app/manage/users' | '/_app/repos/$repo' | '/_app/images/$repo/$' @@ -438,6 +472,8 @@ export interface FileRouteTypes { | '/_app/manage/service-accounts/new' | '/_app/manage/settings/email' | '/_app/manage/settings/retention' + | '/_app/manage/traffic/client' + | '/_app/manage/traffic/settings' | '/_app/manage/users/$userId' | '/_app/manage/users/new' | '/_app/packages/$repo/$id' @@ -559,6 +595,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof adminSystemSystemRouteImport parentRoute: typeof AdminLayoutRoute } + '/_app/manage/traffic': { + id: '/_app/manage/traffic' + path: '/traffic' + fullPath: '/manage/traffic' + preLoaderRoute: typeof adminTrafficTrafficRouteImport + parentRoute: typeof AdminLayoutRoute + } '/_app/manage/users': { id: '/_app/manage/users' path: '/users' @@ -657,6 +700,20 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof adminSettingsRetentionSettingsRouteImport parentRoute: typeof AdminLayoutRoute } + '/_app/manage/traffic/client': { + id: '/_app/manage/traffic/client' + path: '/traffic/client' + fullPath: '/manage/traffic/client' + preLoaderRoute: typeof adminTrafficClientRouteImport + parentRoute: typeof AdminLayoutRoute + } + '/_app/manage/traffic/settings': { + id: '/_app/manage/traffic/settings' + path: '/traffic/settings' + fullPath: '/manage/traffic/settings' + preLoaderRoute: typeof adminTrafficSettingsRouteImport + parentRoute: typeof AdminLayoutRoute + } '/_app/manage/users/$userId': { id: '/_app/manage/users/$userId' path: '/users/$userId' @@ -711,6 +768,7 @@ interface AdminLayoutRouteChildren { adminServiceAccountsListRoute: typeof adminServiceAccountsListRoute adminSettingsSettingsRoute: typeof adminSettingsSettingsRoute adminSystemSystemRoute: typeof adminSystemSystemRoute + adminTrafficTrafficRoute: typeof adminTrafficTrafficRoute adminUsersListRoute: typeof adminUsersListRoute adminSelectorsEditRoute: typeof adminSelectorsEditRoute adminSelectorsCreateRoute: typeof adminSelectorsCreateRoute @@ -723,6 +781,8 @@ interface AdminLayoutRouteChildren { adminServiceAccountsCreateRoute: typeof adminServiceAccountsCreateRoute adminSettingsEmailRoute: typeof adminSettingsEmailRoute adminSettingsRetentionSettingsRoute: typeof adminSettingsRetentionSettingsRoute + adminTrafficClientRoute: typeof adminTrafficClientRoute + adminTrafficSettingsRoute: typeof adminTrafficSettingsRoute adminUsersEditRoute: typeof adminUsersEditRoute adminUsersCreateRoute: typeof adminUsersCreateRoute adminRepositoriesImportRoute: typeof adminRepositoriesImportRoute @@ -739,6 +799,7 @@ const AdminLayoutRouteChildren: AdminLayoutRouteChildren = { adminServiceAccountsListRoute: adminServiceAccountsListRoute, adminSettingsSettingsRoute: adminSettingsSettingsRoute, adminSystemSystemRoute: adminSystemSystemRoute, + adminTrafficTrafficRoute: adminTrafficTrafficRoute, adminUsersListRoute: adminUsersListRoute, adminSelectorsEditRoute: adminSelectorsEditRoute, adminSelectorsCreateRoute: adminSelectorsCreateRoute, @@ -751,6 +812,8 @@ const AdminLayoutRouteChildren: AdminLayoutRouteChildren = { adminServiceAccountsCreateRoute: adminServiceAccountsCreateRoute, adminSettingsEmailRoute: adminSettingsEmailRoute, adminSettingsRetentionSettingsRoute: adminSettingsRetentionSettingsRoute, + adminTrafficClientRoute: adminTrafficClientRoute, + adminTrafficSettingsRoute: adminTrafficSettingsRoute, adminUsersEditRoute: adminUsersEditRoute, adminUsersCreateRoute: adminUsersCreateRoute, adminRepositoriesImportRoute: adminRepositoriesImportRoute, diff --git a/web/src/routes.ts b/web/src/routes.ts index 0e56848e..cb6bf333 100644 --- a/web/src/routes.ts +++ b/web/src/routes.ts @@ -23,7 +23,12 @@ import { index, layout, rootRoute, route } from '@tanstack/virtual-file-routes' * /tokens -> Tokens.tsx (self-service registry tokens) * /cli-login/$secret -> CliLogin.tsx (approve a command-line client's * web login: `npm login`) - * /manage -> AdminLayout.tsx (beforeLoad: require is_admin) + * /manage -> AdminLayout.tsx (beforeLoad: require is_admin; + * /manage/traffic: every instance's + * traffic, /traffic/client: one + * client's, with its request log, + * /traffic/settings: what is + * recorded and kept) * * The admin screens live under `/manage/*`, NOT `/admin/*`: `/admin/*` is the * admin REST API, which the server routes before the SPA fallback, so a UI @@ -72,6 +77,9 @@ export const routes = rootRoute('root.tsx', [ route('/settings', 'admin/settings/Settings.tsx'), route('/settings/email', 'admin/settings/Email.tsx'), route('/settings/retention', 'admin/settings/RetentionSettings.tsx'), + route('/traffic', 'admin/traffic/Traffic.tsx'), + route('/traffic/client', 'admin/traffic/Client.tsx'), + route('/traffic/settings', 'admin/traffic/Settings.tsx'), route('/system', 'admin/system/System.tsx'), ]), ]), diff --git a/web/src/routes/admin/AdminHome.tsx b/web/src/routes/admin/AdminHome.tsx index bc703b03..e1d4427b 100644 --- a/web/src/routes/admin/AdminHome.tsx +++ b/web/src/routes/admin/AdminHome.tsx @@ -67,6 +67,13 @@ const SECTIONS = [ description: 'Whether retention runs, for every repository, and how long its audit log is kept.', }, + { + to: '/manage/traffic', + title: 'Traffic', + icon: 'Activity', + description: + 'Requests of every instance over time, by repository and status; the top clients and their request logs.', + }, { to: '/manage/system', title: 'System', @@ -84,8 +91,8 @@ const SECTIONS = [ * Admin landing. Only reachable once the `AdminLayout` guard has confirmed * `is_admin`. A card per admin screen, with the rail's icon for it: the * repositories, users (incl. per-user tokens and roles), service accounts, roles, content - * selectors, authentication-realm, settings, email-service, retention and - * system screens. + * selectors, authentication-realm, settings, email-service, retention, + * traffic and system screens. */ function AdminHome() { return ( @@ -97,8 +104,8 @@ function AdminHome() { Manage repositories and their retention policies, users and service accounts and their API tokens, roles and their grants, content selectors, authentication realms, the sign-in and - anonymous-access settings, the email service, and the secret - encryption and blob cache. + anonymous-access settings, the email service, the traffic of every + instance, and the secret encryption and blob cache. } /> diff --git a/web/src/routes/admin/traffic/Client.tsx b/web/src/routes/admin/traffic/Client.tsx new file mode 100644 index 00000000..4005b3f2 --- /dev/null +++ b/web/src/routes/admin/traffic/Client.tsx @@ -0,0 +1,271 @@ +import { createFileRoute, useNavigate } from '@tanstack/react-router' + +import { + isTrafficRange, + type TrafficClientDetail, + type TrafficRange, + type TrafficTally, +} from '~/api/traffic' +import { formatBytes, formatDateTime } from '~/lib/oci' +import { accountLabel, share, stepLabel, statusTone, surfaceLabel } from '~/lib/traffic' +import { useTrafficClient } from '~/queries/useTraffic' +import { TrafficChart } from '~/components/TrafficChart' +import { TrafficFilters } from '~/components/TrafficFilters' +import { ErrorState } from '~/components/ErrorState' +import { + EmptyState, + Notice, + Page, + PageHeader, + Panel, + Skeleton, + Stat, + StatRow, + StatusBadge, + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from '~/components/ui' + +/** The screen's URL state: the client network, the range and an instance. */ +export interface TrafficClientSearch { + network: string + range?: TrafficRange + instance?: string +} + +export const Route = createFileRoute('/_app/manage/traffic/client')({ + validateSearch: (search: Record): TrafficClientSearch => ({ + network: typeof search.network === 'string' ? search.network : '', + range: isTrafficRange(search.range) ? search.range : undefined, + instance: + typeof search.instance === 'string' && search.instance.length > 0 + ? search.instance + : undefined, + }), + component: TrafficClientScreen, +}) + +const count = new Intl.NumberFormat() + +/** + * One client network's traffic (`/admin/traffic/client`): its requests over + * time, from the time buckets it was among the top clients of, and its + * requests in the request log — what it asked for, as whom, with which + * client, and how each was answered. + */ +function TrafficClientScreen() { + const search = Route.useSearch() + const navigate = useNavigate({ from: Route.fullPath }) + const range = search.range ?? '24h' + const detail = useTrafficClient(search.network, { range, instance: search.instance }) + const set = (next: Partial) => + void navigate({ search: (prev) => ({ ...prev, ...next }) }) + + return ( + + + {!search.network ? ( + + Open a client from the top clients of the traffic screen. + + ) : ( + <> + set({ range: r })} + instance={search.instance} + instances={detail.data?.instances ?? []} + onInstance={(i) => set({ instance: i })} + /> + {detail.isError ? ( + void detail.refetch()} + /> + ) : !detail.data ? ( + + ) : ( +
+ +
+ )} + + )} + + ) +} + +function Body({ data }: { data: TrafficClientDetail }) { + const totals = data.totals + const log = data.log + return ( + <> + {!data.recording.client_addresses && ( + + Client addresses are not recorded now: the traffic settings switch + them off. What was recorded before stays until it expires. + + )} + + + + + + + + + + {!totals ? ( + + ) : ( + + )} + + + log.entries.length ? `, the newest ${log.entries.length} shown` : '' + }.`} + flush + > + {!data.recording.request_log && ( + + The request log is switched off in the traffic settings: new + requests are not logged. + + )} + {log.entries.length === 0 ? ( + + ) : ( + + )} + + + {log.matched > 0 && ( +
+ + + ({ + value: accountLabel(a.account, a.username, a.user_id), + requests: a.requests, + }))} + total={log.matched} + /> + +
+ )} + + ) +} + +function LogTable({ data }: { data: TrafficClientDetail }) { + return ( + + + Time + Request + Status + Data sent + Took + Account + User agent + Instance + + + {data.log.entries.map((e, i) => ( + + {new Date(e.time).toLocaleString()} + + + {e.method} {e.path} + +
+ {e.repository ?? surfaceLabel(e.surface)} + {e.address && e.address !== data.network ? ` · ${e.address}` : ''} +
+ + + {String(e.status)} + + {formatBytes(e.bytes)} + + {e.duration_ms < 10 ? e.duration_ms.toFixed(1) : Math.round(e.duration_ms)} ms + + {accountLabel(e.account, e.username, e.user_id)} + + {e.user_agent ?? '—'} + + {e.host} + + ))} + +
+ ) +} + +function TallyPanel({ + title, + tallies, + total, + mono = false, +}: { + title: string + tallies: TrafficTally[] + total: number + mono?: boolean +}) { + return ( + + + + {title.replace(/s$/, '')} + Requests + Share + + + {tallies.map((t) => ( + + {mono ? {t.value} : t.value} + {count.format(t.requests)} + {share(t.requests, total)} + + ))} + +
+ + ) +} diff --git a/web/src/routes/admin/traffic/Settings.tsx b/web/src/routes/admin/traffic/Settings.tsx new file mode 100644 index 00000000..ce72986e --- /dev/null +++ b/web/src/routes/admin/traffic/Settings.tsx @@ -0,0 +1,209 @@ +import { useState, type FormEvent } from 'react' +import { createFileRoute } from '@tanstack/react-router' + +import type { TrafficSettings } from '~/api/traffic' +import { useTrafficSettings, useUpdateTrafficSettings } from '~/queries/useTraffic' +import { + Button, + Checkbox, + Notice, + Page, + PageHeader, + Panel, + Skeleton, + Switch, + TextField, +} from '~/components/ui' +import { ErrorState } from '~/components/ErrorState' + +export const Route = createFileRoute('/_app/manage/traffic/settings')({ + component: TrafficSettingsScreen, +}) + +const DESCRIPTION = + 'What the traffic statistics record and how long they are kept, for every instance at once. Each instance follows a change within a minute.' + +/** The longest retention the server accepts, in days. */ +const MAX_DAYS = 36_500 + +/** + * Traffic settings (`/admin/settings/traffic`): what is recorded (requests + * at all, client addresses, the request log) and how long each part is kept. + * Each is a form panel that saves its own fields (`PUT` changes only what + * it names), so saving one never undoes a change another administrator made + * to the other. + */ +function TrafficSettingsScreen() { + const settings = useTrafficSettings() + + return ( + + + {settings.isError ? ( + void settings.refetch()} + /> + ) : settings.isLoading || !settings.data ? ( + + ) : ( + <> + + + + )} + + ) +} + +/** What is recorded. */ +function RecordingForm({ settings }: { settings: TrafficSettings }) { + const update = useUpdateTrafficSettings() + const [enabled, setEnabled] = useState(settings.enabled) + const [addresses, setAddresses] = useState(settings.client_addresses) + const [log, setLog] = useState(settings.request_log) + + function handleSubmit(e: FormEvent) { + e.preventDefault() + update.mutate({ enabled, client_addresses: addresses, request_log: log }) + } + + return ( + + {update.isPending ? 'Saving…' : 'Save recording'} + + } + > +
+ + + + {update.isError && ( + {update.error.body || update.error.message} + )} + {update.isSuccess && Saved.} +
+ + ) +} + +/** A whole number of days, or the error to show. */ +function days(value: string, min: number): number | string { + const n = value.trim() + if (!/^\d+$/.test(n) || Number(n) < min || Number(n) > MAX_DAYS) { + return `Enter a whole number of days from ${min} to ${MAX_DAYS.toLocaleString()}.` + } + return Number(n) +} + +/** How long each part is kept. */ +function RetentionForm({ settings }: { settings: TrafficSettings }) { + const update = useUpdateTrafficSettings() + const keepsAll = settings.statistics_retention_days === 0 + const [keepAll, setKeepAll] = useState(keepsAll) + const [statistics, setStatistics] = useState(String(keepsAll ? 400 : settings.statistics_retention_days)) + const [clients, setClients] = useState(String(settings.client_retention_days)) + const [log, setLog] = useState(String(settings.request_log_retention_days)) + const [errors, setErrors] = useState>({}) + + function handleSubmit(e: FormEvent) { + e.preventDefault() + const parsed = { + statistics: keepAll ? 0 : days(statistics, 1), + clients: days(clients, 1), + log: days(log, 1), + } + const found = Object.fromEntries( + Object.entries(parsed).filter((entry): entry is [string, string] => typeof entry[1] === 'string'), + ) + setErrors(found) + if (Object.keys(found).length > 0) return + update.mutate({ + statistics_retention_days: parsed.statistics as number, + client_retention_days: parsed.clients as number, + request_log_retention_days: parsed.log as number, + }) + } + + return ( + + {update.isPending ? 'Saving…' : 'Save retention'} + + } + > +
+ + {!keepAll && ( + setStatistics(e.target.value)} + error={errors.statistics} + /> + )} + setClients(e.target.value)} + error={errors.clients} + /> + setLog(e.target.value)} + error={errors.log} + description="The largest part: a file per instance and minute." + /> + {update.isError && ( + {update.error.body || update.error.message} + )} + {update.isSuccess && Saved.} +
+ + ) +} diff --git a/web/src/routes/admin/traffic/Traffic.tsx b/web/src/routes/admin/traffic/Traffic.tsx new file mode 100644 index 00000000..559bae6a --- /dev/null +++ b/web/src/routes/admin/traffic/Traffic.tsx @@ -0,0 +1,332 @@ +import { useState } from 'react' +import { createFileRoute, Link, useNavigate } from '@tanstack/react-router' + +import { isTrafficRange, type TrafficOverview, type TrafficRange } from '~/api/traffic' +import { formatBytes, formatDateTime } from '~/lib/oci' +import { accountLabel, latencySummary, share, stepLabel, surfaceLabel } from '~/lib/traffic' +import { useTrafficOverview } from '~/queries/useTraffic' +import { TRAFFIC_METRICS, TrafficChart, type TrafficMetric } from '~/components/TrafficChart' +import { TrafficFilters } from '~/components/TrafficFilters' +import { ErrorState } from '~/components/ErrorState' +import { + Button, + Chip, + EmptyState, + Notice, + Page, + PageHeader, + Panel, + SegmentedControl, + Skeleton, + Stat, + StatRow, + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from '~/components/ui' + +/** The screen's URL state: the range, and an instance or repository to narrow to. */ +export interface TrafficSearch { + range?: TrafficRange + instance?: string + repository?: string +} + +function text(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined +} + +export const Route = createFileRoute('/_app/manage/traffic')({ + validateSearch: (search: Record): TrafficSearch => ({ + range: isTrafficRange(search.range) ? search.range : undefined, + instance: text(search.instance), + repository: text(search.repository), + }), + component: TrafficScreen, +}) + +const count = new Intl.NumberFormat() + +/** + * The traffic screen (`/admin/traffic`): what every instance answered in a + * range — requests over time, their totals and latency, the repositories, + * surfaces and statuses, and the clients with the most requests, each + * opening its own view with its request log. The range, the instance and + * the repository are in the URL, so a view can be shared. + */ +function TrafficScreen() { + const search = Route.useSearch() + const navigate = useNavigate({ from: Route.fullPath }) + const range = search.range ?? '24h' + const overview = useTrafficOverview({ + range, + instance: search.instance, + repository: search.repository, + }) + const set = (next: Partial) => + void navigate({ search: (prev) => ({ ...prev, ...next }) }) + + return ( + + + Settings + + } + /> + set({ range: r })} + instance={search.instance} + instances={overview.data?.instances ?? []} + onInstance={(i) => set({ instance: i })} + /> + {search.repository && ( +

+ Only {search.repository}{' '} + +

+ )} + {overview.isError ? ( + void overview.refetch()} + /> + ) : !overview.data ? ( + + ) : ( +
+ +
+ )} + + ) +} + +function Body({ data, repository }: { data: TrafficOverview; repository?: string }) { + const [metric, setMetric] = useState('requests') + const [view, setView] = useState<'chart' | 'table'>('chart') + const { totals, recording } = data + return ( + <> + {!recording.enabled && ( + + Recording is switched off in the traffic settings: no instance counts + requests until it is switched on again. What was written before stays + until it expires. + + )} + + + + + + + + m.id === metric)?.label}, ${stepLabel(data.step_seconds)}, from ${formatDateTime(data.from)} to ${formatDateTime(data.to)}. The last minutes fill in as the instances write them.`} + actions={ +
+ + +
+ } + > + {totals.requests === 0 ? ( + + Nothing was counted here yet. Each instance writes what it counted + once a minute. + + ) : ( + + )} + + + + {!recording.client_addresses && ( + + Client addresses are not recorded: the traffic settings switch them off. + + )} + {data.clients.length === 0 ? ( + + ) : ( + + )} + {repository && ( +

+ The top clients count every repository. +

+ )} + + +
+ + {data.repositories.length === 0 ? ( + + ) : ( + + + Repository + Requests + Data sent + 4xx + 5xx + + + {data.repositories.map((r) => ( + + + ({ ...prev, repository: r.name })} + className="fg-link" + > + {r.name} + + + {count.format(r.requests)} + {formatBytes(r.bytes)} + {count.format(r.client_errors)} + {count.format(r.server_errors)} + + ))} + +
+ )} + + + + + Surface + Requests + Data sent + + + {data.surfaces.map((s) => ( + + {surfaceLabel(s.surface)} + {count.format(s.requests)} + {formatBytes(s.bytes)} + + ))} + +
+ + + Status + Requests + Share + + + {data.statuses.map((s) => ( + + + {s.status} + + {count.format(s.requests)} + {share(s.requests, totals.requests)} + + ))} + +
+ +
+ + ) +} + +function ClientTable({ data }: { data: TrafficOverview }) { + return ( + + + Client + Requests + Data sent + 4xx + 5xx + Last account + Last user agent + Instances + Last seen + + + {data.clients.map((c) => ( + + + ({ + network: c.network, + range: prev.range, + instance: prev.instance, + })} + className="fg-link fg-mono" + > + {c.network} + + {c.address && c.address !== c.network && ( +
{c.address}
+ )} + + {count.format(c.requests)} + {formatBytes(c.bytes)} + {count.format(c.client_errors)} + {count.format(c.server_errors)} + {accountLabel(c.account, c.username, c.user_id)} + + + {c.user_agent ? truncate(c.user_agent, 40) : '—'} + + + {c.hosts.join(', ')} + {formatDateTime(c.last_seen)} + + ))} + +
+ ) +} + +function truncate(s: string, max: number): string { + return s.length > max ? `${s.slice(0, max - 1)}…` : s +} diff --git a/web/src/routes/admin/traffic/traffic.test.tsx b/web/src/routes/admin/traffic/traffic.test.tsx new file mode 100644 index 00000000..cc181abe --- /dev/null +++ b/web/src/routes/admin/traffic/traffic.test.tsx @@ -0,0 +1,309 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { screen, waitFor, within } from '@testing-library/react' +import userEvent from '@testing-library/user-event' + +import { clearSession, setSession } from '~/lib/auth' +import type { Me } from '~/api/me' +import type { TrafficClientDetail, TrafficOverview, TrafficSettings } from '~/api/traffic' +import { renderApp } from '~/test/renderApp' + +/** + * The traffic screens (`/manage/traffic`, its client view and its settings), + * driven against the real route tree and query hooks: + * - the overview's totals, top clients and repositories, with the range, + * instance and repository sent as asked and kept in the URL; + * - a client's view with its request log, accounts named by user; + * - the settings, each panel saving only its own fields. + */ + +const adminMe: Me = { + user: { id: 1, username: 'root', is_admin: true, active: true }, + is_admin: true, + capabilities: ['repo.read', 'repo.admin'], +} + +const recording = { enabled: true, client_addresses: true, request_log: true } + +const point = (start: string, requests: number) => ({ + start, + requests, + bytes: requests * 1000, + client_errors: 1, + server_errors: 0, +}) + +const overview: TrafficOverview = { + from: '2026-10-30T13:00:00Z', + to: '2026-10-30T14:00:00Z', + step_seconds: 60, + resolution: 'minute', + instances: [ + { id: 'web-1_20261030T100000Z_00a1', host: 'web-1', started_at: '2026-10-30T10:00:00Z' }, + { id: 'web-2_20261030T110000Z_00b2', host: 'web-2', started_at: '2026-10-30T11:00:00Z' }, + ], + totals: { + requests: 1234, + bytes: 5 * 1024 ** 2, + client_errors: 40, + denied: 12, + not_found: 25, + throttled: 3, + server_errors: 2, + mean_ms: 18.25, + p50_ms: 10, + p95_ms: 250, + p99_ms: 1000, + }, + series: [point('2026-10-30T13:00:00Z', 600), point('2026-10-30T13:01:00Z', 634)], + surfaces: [ + { surface: 'plugin_npm', requests: 1000, bytes: 4 * 1024 ** 2 }, + { surface: 'oci', requests: 234, bytes: 1024 ** 2 }, + ], + repositories: [ + { name: 'npm-proxy', requests: 1000, bytes: 4 * 1024 ** 2, client_errors: 30, server_errors: 2 }, + ], + statuses: [ + { status: 200, requests: 1192 }, + { status: 404, requests: 25 }, + ], + clients: [ + { + network: '2001:db8:1:2::/64', + address: '2001:db8:1:2::7', + requests: 900, + bytes: 3 * 1024 ** 2, + client_errors: 20, + denied: 5, + not_found: 15, + throttled: 0, + server_errors: 1, + user_agent: 'npm/10.8.2 node/v22', + account: 'token', + user_id: 7, + username: 'ci-bot', + hosts: ['web-1', 'web-2'], + first_seen: '2026-10-30T13:00:00Z', + last_seen: '2026-10-30T13:59:00Z', + }, + ], + recording, +} + +const clientDetail: TrafficClientDetail = { + network: '2001:db8:1:2::/64', + from: '2026-10-30T13:00:00Z', + to: '2026-10-30T14:00:00Z', + step_seconds: 60, + resolution: 'minute', + instances: overview.instances, + totals: overview.clients[0], + series: overview.series, + log: { + from: '2026-10-30T13:00:00Z', + to: '2026-10-30T14:00:00Z', + matched: 2, + entries: [ + { + time: '2026-10-30T13:59:00Z', + address: '2001:db8:1:2::7', + method: 'GET', + path: '/repository/npm-proxy/left-pad', + surface: 'plugin_npm', + repository: 'npm-proxy', + status: 404, + bytes: 21, + duration_ms: 3.5, + user_agent: 'npm/10.8.2 node/v22', + account: 'token', + user_id: 7, + username: 'ci-bot', + host: 'web-2', + }, + ], + paths: [{ value: '/repository/npm-proxy/left-pad', requests: 2 }], + user_agents: [{ value: 'npm/10.8.2 node/v22', requests: 2 }], + accounts: [{ account: 'token', user_id: 7, username: 'ci-bot', requests: 2 }], + statuses: [{ value: '404', requests: 2 }], + }, + recording, +} + +const settings: TrafficSettings = { + enabled: true, + client_addresses: true, + request_log: true, + statistics_retention_days: 400, + client_retention_days: 30, + request_log_retention_days: 7, +} + +function json(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }) +} + +const fetchMock = vi.fn() + +function mockApi(h: { overview?: TrafficOverview; onPut?: (body: unknown) => Response } = {}) { + fetchMock.mockImplementation((input: RequestInfo | URL, init?: RequestInit) => { + const url = new URL(String(input), 'http://localhost') + const method = (init?.method ?? 'GET').toUpperCase() + if (url.pathname.endsWith('/admin/me')) return Promise.resolve(json(adminMe)) + if (url.pathname.endsWith('/admin/traffic')) return Promise.resolve(json(h.overview ?? overview)) + if (url.pathname.endsWith('/admin/traffic/client')) return Promise.resolve(json(clientDetail)) + if (url.pathname.endsWith('/admin/settings/traffic')) { + if (method === 'PUT') { + const body: unknown = JSON.parse(String(init?.body)) + return Promise.resolve( + h.onPut ? h.onPut(body) : json({ ...settings, ...(body as object) }), + ) + } + return Promise.resolve(json(settings)) + } + return Promise.resolve(new Response('not found', { status: 404 })) + }) +} + +/** The query strings of the requests to `path` sent so far. */ +function sent(path: string, method = 'GET'): URLSearchParams[] { + return fetchMock.mock.calls + .filter(([, init]) => ((init as RequestInit | undefined)?.method ?? 'GET') === method) + .map(([input]) => new URL(String(input), 'http://localhost')) + .filter((u) => u.pathname === path) + .map((u) => u.searchParams) +} + +describe('admin traffic screens', () => { + beforeEach(() => { + vi.stubGlobal('fetch', fetchMock) + fetchMock.mockReset() + setSession('admin-csrf') + }) + afterEach(() => { + vi.unstubAllGlobals() + clearSession() + }) + + it('shows the totals, the top clients and the repositories', async () => { + mockApi() + await renderApp('/manage/traffic') + + expect(await screen.findByText('1,234')).toBeInTheDocument() + expect(screen.getByText('95 % answered within 250 ms, half within 10 ms')).toBeInTheDocument() + expect(sent('/admin/traffic')[0].get('range')).toBe('24h') + + const clients = screen.getByRole('table', { name: 'Top clients' }) + const row = within(clients).getAllByRole('row')[1] + expect(within(row).getByRole('link', { name: '2001:db8:1:2::/64' })).toBeInTheDocument() + expect(within(row).getByText('ci-bot (API token)')).toBeInTheDocument() + expect(within(row).getByText('web-1, web-2')).toBeInTheDocument() + + const repos = screen.getByRole('table', { name: 'Repositories by requests' }) + expect(within(repos).getByRole('link', { name: 'npm-proxy' })).toBeInTheDocument() + const surfaces = screen.getByRole('table', { name: 'Requests by surface' }) + expect(within(surfaces).getByText('Docker registry')).toBeInTheDocument() + expect(screen.getByRole('img', { name: /Requests over time: 1,234 in all/ })).toBeInTheDocument() + }) + + it('asks for the range, instance and repository chosen', async () => { + mockApi() + const user = userEvent.setup() + const router = await renderApp('/manage/traffic') + await screen.findByText('1,234') + + await user.click(screen.getByRole('radio', { name: '7 days' })) + await waitFor(() => expect(sent('/admin/traffic').some((q) => q.get('range') === '7d')).toBe(true)) + + await user.selectOptions(screen.getByRole('combobox', { name: 'Instance' }), 'web-2_20261030T110000Z_00b2') + await waitFor(() => + expect(sent('/admin/traffic').some((q) => q.get('instance') === 'web-2_20261030T110000Z_00b2')).toBe(true), + ) + + await user.click(screen.getByRole('link', { name: 'npm-proxy' })) + await waitFor(() => + expect(sent('/admin/traffic').some((q) => q.get('repository') === 'npm-proxy')).toBe(true), + ) + expect(router.state.location.search).toMatchObject({ + range: '7d', + instance: 'web-2_20261030T110000Z_00b2', + repository: 'npm-proxy', + }) + }) + + it('shows the points as a table on request', async () => { + mockApi() + const user = userEvent.setup() + await renderApp('/manage/traffic') + await screen.findByText('1,234') + + await user.click(screen.getByRole('radio', { name: 'Table' })) + const table = screen.getByRole('table', { name: 'Requests over time' }) + const rows = within(table).getAllByRole('row').slice(1) + expect(rows.map((r) => within(r).getAllByRole('cell')[1].textContent)).toEqual(['634', '600']) + }) + + it('says when nothing is recorded', async () => { + mockApi({ overview: { ...overview, recording: { ...recording, enabled: false } } }) + await renderApp('/manage/traffic') + expect(await screen.findByText('Traffic is not recorded')).toBeInTheDocument() + }) + + it("opens a client's view with its request log", async () => { + mockApi() + const user = userEvent.setup() + await renderApp('/manage/traffic?range=6h') + await user.click(await screen.findByRole('link', { name: '2001:db8:1:2::/64' })) + + const log = await screen.findByRole('table', { name: 'Request log' }) + const row = within(log).getAllByRole('row')[1] + expect(within(row).getByText('GET /repository/npm-proxy/left-pad')).toBeInTheDocument() + expect(within(row).getByText('404')).toBeInTheDocument() + expect(within(row).getByText('ci-bot (API token)')).toBeInTheDocument() + const accounts = screen.getByRole('table', { name: 'Accounts in the request log' }) + expect(within(accounts).getByText('ci-bot (API token)')).toBeInTheDocument() + + const asked = sent('/admin/traffic/client')[0] + expect(asked.get('network')).toBe('2001:db8:1:2::/64') + expect(asked.get('range')).toBe('6h') + }) + + it('saves each settings panel on its own', async () => { + mockApi() + const user = userEvent.setup() + await renderApp('/manage/traffic/settings') + + await user.click(await screen.findByRole('switch', { name: /Request log/ })) + await user.click(screen.getByRole('button', { name: 'Save recording' })) + await waitFor(() => expect(sent('/admin/settings/traffic', 'PUT')).toHaveLength(1)) + const puts = fetchMock.mock.calls.filter( + ([, init]) => (init as RequestInit | undefined)?.method === 'PUT', + ) + expect(JSON.parse(String((puts[0][1] as RequestInit).body))).toEqual({ + enabled: true, + client_addresses: true, + request_log: false, + }) + + const log = screen.getByRole('textbox', { name: 'Keep the request log for (days)' }) + await user.clear(log) + await user.type(log, '0') + await user.click(screen.getByRole('button', { name: 'Save retention' })) + expect(await screen.findByText(/Enter a whole number of days from 1/)).toBeInTheDocument() + expect(sent('/admin/settings/traffic', 'PUT')).toHaveLength(1) + + await user.clear(log) + await user.type(log, '14') + await user.click(screen.getByRole('button', { name: 'Save retention' })) + await waitFor(() => expect(sent('/admin/settings/traffic', 'PUT')).toHaveLength(2)) + const second = fetchMock.mock.calls.filter( + ([, init]) => (init as RequestInit | undefined)?.method === 'PUT', + )[1] + expect(JSON.parse(String((second[1] as RequestInit).body))).toEqual({ + statistics_retention_days: 400, + client_retention_days: 30, + request_log_retention_days: 14, + }) + }) +}) diff --git a/web/src/styles/forge-ui.css b/web/src/styles/forge-ui.css index 3f405268..0bae4df0 100644 --- a/web/src/styles/forge-ui.css +++ b/web/src/styles/forge-ui.css @@ -455,6 +455,25 @@ } .fg-dialog { max-height: calc(100dvh - 2 * var(--space-4)); overflow-y: auto; } + /* ---------- App additions: charts (Admin → Traffic) ---------- */ + /* One series in categorical slot 1, checked against both surfaces. */ + .fg-chart { --chart-series: #2a78d6; position: relative; min-width: 0; } + [data-theme='dark'] .fg-chart { --chart-series: #3987e5; } + .fg-chart svg { display: block; overflow: visible; } + .fg-chart svg:focus-visible { outline: 2px solid var(--focus); outline-offset: 2px; border-radius: var(--radius-xs); } + .fg-chart-grid { stroke: var(--line); stroke-width: 1; shape-rendering: crispEdges; } + .fg-chart-axis { fill: var(--ink-muted); font-size: 11px; font-variant-numeric: tabular-nums; } + .fg-chart-line { fill: none; stroke: var(--chart-series); stroke-width: 2; stroke-linejoin: round; stroke-linecap: round; } + .fg-chart-area { fill: var(--chart-series); opacity: 0.1; } + .fg-chart-cross { stroke: var(--line-strong); stroke-width: 1; } + .fg-chart-dot { fill: var(--chart-series); stroke: var(--surface); stroke-width: 2; } + .fg-chart-tip { position: absolute; top: 0; transform: translateX(-50%); pointer-events: none; padding: 6px 10px; border: 1px solid var(--line); border-radius: var(--radius-sm); background: var(--surface); box-shadow: var(--shadow-md); font-size: 12.5px; line-height: 18px; white-space: nowrap; } + .fg-chart-tip-value { color: var(--ink); font-weight: 600; font-variant-numeric: tabular-nums; } + .fg-chart-tip-label { color: var(--ink-muted); } + .fg-chart-key { display: inline-block; width: 12px; height: 2px; margin-right: 6px; border-radius: 1px; background: var(--chart-series); vertical-align: middle; } + .fg-filters { display: flex; flex-wrap: wrap; align-items: center; gap: var(--space-3); min-width: 0; } + .fg-filters > .fg-seg { max-width: 100%; overflow-x: auto; } + /* ---------- App additions: remembered accounts (login screen) ---------- */ .fg-accounts { display: grid; gap: var(--space-2); margin: 0; padding: 0; list-style: none; } .fg-account { display: flex; align-items: center; gap: var(--space-1); padding-right: var(--space-1); border: 1px solid var(--line); border-radius: var(--radius-md); background: var(--surface); transition: background-color 120ms ease-out, border-color 120ms ease-out; } From fb395402a12b42665749f1b76386bffbe9bec704 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 09:22:14 +0000 Subject: [PATCH 2/3] Traffic: count requests per day, repository and client; locate clients with MaxMind The statistics only need a count per day for each repository and client address, to compare where the requests come from, so the minute and hour files, the latency histograms, the top-client lists and the request log are replaced by one Parquet file per instance and day: a row per repository and client network with its requests and bytes. Clients are located with MaxMind databases: their country and autonomous system go on each row. The databases are kept in the bucket (traffic/v1/geoip/{country,asn}.mmdb) and loaded by every instance from there. Forge downloads GeoLite2-Country and GeoLite2-ASN every three days with a MaxMind account ID and license key set in the web UI (the key sealed like the other secret settings, the download URL configurable as traffic.maxmind_url), or an operator puts the files there. - forge-traffic: daily recorder and flusher, geo lookups (maxminddb), queries filtered by instance, repository, client, country and AS, and a small MaxMind database writer for tests (feature `testing`). - /admin/traffic takes the filters; /admin/traffic/geoip shows the databases and sets, downloads or removes the MaxMind account. - Admin -> Traffic: requests by day, repositories, countries, networks and top clients, each narrowing the view; the settings get a geolocation panel. Fixes #709 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012e6DJbpgZmQnhz5Jjjhug6 --- Cargo.lock | 20 + Cargo.toml | 2 + crates/db/src/secret_dao.rs | 13 +- crates/server/Cargo.toml | 5 + crates/server/src/admin/mod.rs | 11 +- crates/server/src/admin/traffic.rs | 511 ++++--- crates/server/src/config.rs | 49 + crates/server/src/images/dispatch.rs | 1 - crates/server/src/middleware/principal.rs | 1 - crates/server/src/plugins.rs | 1 - crates/server/src/telemetry.rs | 31 +- crates/server/src/traffic/maxmind.rs | 378 ++++++ .../server/src/{traffic.rs => traffic/mod.rs} | 116 +- crates/server/tests/traffic.rs | 369 +++-- crates/traffic/Cargo.toml | 7 + crates/traffic/src/geo.rs | 349 +++++ crates/traffic/src/layout.rs | 773 +++++------ crates/traffic/src/lib.rs | 72 +- crates/traffic/src/query.rs | 1196 ++++++----------- crates/traffic/src/record.rs | 374 ++---- crates/traffic/src/rows.rs | 277 +--- crates/traffic/src/settings.rs | 39 +- crates/traffic/src/slot.rs | 32 +- crates/traffic/src/testing.rs | 174 +++ crates/traffic/src/top.rs | 190 --- docs/changes/2.2.0/709.feature.md | 23 +- docs/deployment.md | 99 +- forge.toml.example | 7 + web/src/api/traffic.ts | 262 ++-- web/src/components/TrafficChart.tsx | 150 +-- web/src/lib/traffic.ts | 99 +- web/src/queries/useTraffic.ts | 61 +- web/src/routeTree.gen.ts | 21 - web/src/routes.ts | 9 +- web/src/routes/admin/traffic/Client.tsx | 271 ---- web/src/routes/admin/traffic/Settings.tsx | 291 ++-- web/src/routes/admin/traffic/Traffic.tsx | 376 ++++-- web/src/routes/admin/traffic/traffic.test.tsx | 345 +++-- 38 files changed, 3460 insertions(+), 3545 deletions(-) create mode 100644 crates/server/src/traffic/maxmind.rs rename crates/server/src/{traffic.rs => traffic/mod.rs} (69%) create mode 100644 crates/traffic/src/geo.rs create mode 100644 crates/traffic/src/testing.rs delete mode 100644 crates/traffic/src/top.rs delete mode 100644 web/src/routes/admin/traffic/Client.tsx diff --git a/Cargo.lock b/Cargo.lock index 97265e15..71d5529b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1919,6 +1919,7 @@ dependencies = [ "sha2 0.11.1", "sqlx", "subtle", + "tar", "tempfile", "testcontainers", "testcontainers-modules", @@ -1984,6 +1985,7 @@ dependencies = [ "chrono", "forge-storage", "futures-util", + "maxminddb", "moka", "parquet", "parquet_derive", @@ -2648,6 +2650,12 @@ version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" +[[package]] +name = "ipnetwork" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf370abdafd54d13e54a620e8c3e1145f28e46cc9d704bc6d94414559df41763" + [[package]] name = "itertools" version = "0.14.0" @@ -2921,6 +2929,18 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "maxminddb" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b43d2458d977b610b04a3858e6504f06133663d303a1a0606601b530bfc9bc6" +dependencies = [ + "ipnetwork", + "memchr", + "serde", + "thiserror", +] + [[package]] name = "md-5" version = "0.11.0" diff --git a/Cargo.toml b/Cargo.toml index 1f23f7ac..f9123884 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -198,6 +198,8 @@ aws-lc-rs = { version = "1.18" } # pure-Rust codec. parquet = { version = "60", default-features = false, features = ["snap"] } parquet_derive = { version = "60" } +# Country and ASN lookups in MaxMind databases (crates/traffic). +maxminddb = { version = "0.32" } # Temp files tempfile = { version = "3" } diff --git a/crates/db/src/secret_dao.rs b/crates/db/src/secret_dao.rs index 78be88eb..87931637 100644 --- a/crates/db/src/secret_dao.rs +++ b/crates/db/src/secret_dao.rs @@ -22,7 +22,8 @@ //! [`insert_realm_with_secrets`] / [`realm_config`]). //! //! Secrets kept in `forge_setting` ([`SECRET_SETTINGS`]: the generated -//! registry-token key, the OIDC issuer keys) are text values, +//! registry-token key, the OIDC issuer keys, the MaxMind license key) are +//! text values, //! `sealed:`, AAD-bound to `('forge_setting', 'value', 0)` //! plus the setting's key. They take part in the re-encrypt sweep and the DEK //! reference scan like the secret columns ([`store_secret_setting`], @@ -1072,8 +1073,16 @@ pub const REGISTRY_TOKEN_KEY_SETTING: &str = "docker_token_key"; /// assertions with. pub const OIDC_ISSUER_KEYS_SETTING: &str = "oidc_issuer_keys"; +/// `forge_setting` key of the MaxMind license key the traffic statistics' +/// geolocation databases are downloaded with. +pub const MAXMIND_LICENSE_KEY_SETTING: &str = "traffic_maxmind_license_key"; + /// The `forge_setting` keys whose values are secrets (module docs). -pub const SECRET_SETTINGS: &[&str] = &[REGISTRY_TOKEN_KEY_SETTING, OIDC_ISSUER_KEYS_SETTING]; +pub const SECRET_SETTINGS: &[&str] = &[ + REGISTRY_TOKEN_KEY_SETTING, + OIDC_ISSUER_KEYS_SETTING, + MAXMIND_LICENSE_KEY_SETTING, +]; /// Prefix of a sealed secret setting: `sealed:`. const SEALED_SETTING: &str = "sealed:"; diff --git a/crates/server/Cargo.toml b/crates/server/Cargo.toml index 5f9049af..608e876b 100644 --- a/crates/server/Cargo.toml +++ b/crates/server/Cargo.toml @@ -80,6 +80,8 @@ tokio-util = { workspace = true } multer = "3" # npm audits: a gzip-compressed request body. flate2 = { version = "1.1", default-features = false, features = ["zlib-rs"] } +# The MaxMind databases of the traffic statistics come as `.tar.gz`. +tar = { version = "0.4", default-features = false } # The file store: the MD5 of uploads (the `.md5` sidecar), file names in # headers, HTTP dates (`Last-Modified`, `If-Modified-Since`) and content # types guessed from file names. @@ -111,6 +113,9 @@ figment = { version = "0.10", features = ["toml", "env"] } tikv-jemallocator = { version = "0.7", features = ["background_threads"] } [dev-dependencies] +# Small MaxMind databases for the traffic statistics' tests. +forge-traffic = { path = "../traffic", features = ["testing"] } +tar = { version = "0.4", default-features = false } # Builds Helm chart archives for the registry tests. flate2 = { version = "1.1", default-features = false, features = ["zlib-rs"] } tokio = { workspace = true, features = ["rt-multi-thread", "macros", "fs", "io-util", "time", "test-util"] } diff --git a/crates/server/src/admin/mod.rs b/crates/server/src/admin/mod.rs index a5285087..770c1426 100644 --- a/crates/server/src/admin/mod.rs +++ b/crates/server/src/admin/mod.rs @@ -353,7 +353,16 @@ pub fn admin_router(state: AppState) -> Router { .route("/blob-cache", get(blob_cache::stats)) // --- traffic statistics -------------------------------------------- .route("/traffic", get(traffic::overview)) - .route("/traffic/client", get(traffic::client)) + .route( + "/traffic/geoip", + get(traffic::get_geoip) + .put(traffic::put_geoip) + .delete(traffic::delete_geoip), + ) + .route( + "/traffic/geoip/update", + axum::routing::post(traffic::update_geoip), + ) .route( "/settings/traffic", get(traffic::get_settings).put(traffic::put_settings), diff --git a/crates/server/src/admin/traffic.rs b/crates/server/src/admin/traffic.rs index 2abf83d7..c0dfb95b 100644 --- a/crates/server/src/admin/traffic.rs +++ b/crates/server/src/admin/traffic.rs @@ -1,16 +1,19 @@ //! The traffic statistics (Admin → Traffic). **Admin-only.** //! -//! * `GET /admin/traffic?range=&instance=&repository=` — the traffic of a -//! range ([`forge_traffic::query::Overview`]) over every instance or the -//! one `instance` names: counts by time bucket, surface, repository and -//! status, latency, and the top clients. `range` is one of [`RANGES`] -//! (default `24h`), or `from` and `to` (RFC 3339) give the range. -//! * `GET /admin/traffic/client?network=&range=&instance=` — one client -//! network's traffic and its requests in the request log -//! ([`forge_traffic::query::ClientDetail`]). +//! * `GET /admin/traffic?range=&instance=&repository=&network=&country=&asn=` +//! — the requests of a range of days ([`forge_traffic::query::Overview`]) +//! over every instance or the one `instance` names: by day, repository, +//! country, autonomous system and client network. `range` is one of +//! [`RANGES`] (default `30d`), or `from` and `to` (dates, both included) +//! give it. `repository=-` keeps the requests that addressed no +//! repository, `country=-` and `asn=0` the ones from nowhere known. //! * `GET /admin/settings/traffic` answers the [`TrafficSettings`]; `PUT` //! changes the fields a body names, for every instance (each follows with //! its next flush). +//! * `GET /admin/traffic/geoip` — the geolocation databases this instance +//! loaded and the MaxMind download's account and outcome; `PUT` sets the +//! account ID and license key (sealed), `DELETE` forgets them and deletes +//! the databases, `POST /admin/traffic/geoip/update` downloads them now. //! //! The files are read from the object store, so the answers cover every //! instance; a store that cannot be read answers 503. @@ -18,44 +21,44 @@ use axum::extract::{Query, State}; use axum::http::StatusCode; use axum::response::{IntoResponse, Json, Response}; -use chrono::{DateTime, Duration, Utc}; +use chrono::{Duration, NaiveDate, Utc}; use forge_core::auth::principal::Principal; -use forge_traffic::query::{self, Filter, Span}; -use forge_traffic::TrafficSettings; +use forge_traffic::query::{self, Filter}; +use forge_traffic::{DbInfo, GeoKind, TrafficSettings}; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; use crate::state::AppState; +use crate::traffic::maxmind::{self, MaxmindState}; use crate::traffic::{read_settings, write_settings}; -/// The ranges `range` names, and their length. -pub const RANGES: [(&str, i64); 6] = [ - ("1h", 3_600), - ("6h", 6 * 3_600), - ("24h", 24 * 3_600), - ("7d", 7 * 86_400), - ("30d", 30 * 86_400), - ("90d", 90 * 86_400), +/// The ranges `range` names, in days up to today (included). +pub const RANGES: [(&str, i64); 5] = [ + ("today", 1), + ("7d", 7), + ("30d", 30), + ("90d", 90), + ("365d", 365), ]; -/// The longest range `from` and `to` may give. -const MAX_RANGE_DAYS: i64 = 400; +/// What a filter value says for "none" (no repository, no known country). +const NONE: &str = "-"; -/// The most users whose names an answer looks up. -const MAX_USERNAMES: usize = 50; - -/// The query of both views. +/// The query of the overview. #[derive(Debug, Default, Deserialize)] -pub struct RangeQuery { +pub struct OverviewQuery { pub range: Option, - pub from: Option>, - pub to: Option>, + pub from: Option, + pub to: Option, /// Only this instance's files (an id of the answer's `instances`). pub instance: Option, - /// The overview: only this repository's counts. + /// Only this repository's requests; `-` for the ones to none. pub repository: Option, - /// The client view: the network ([`crate::traffic::network_label`]). + /// Only this client network's requests. pub network: Option, + /// Only this country's requests (ISO code); `-` for unknown. + pub country: Option, + /// Only this autonomous system's requests; `0` for unknown. + pub asn: Option, } fn bad_request(msg: impl Into) -> Response { @@ -71,81 +74,68 @@ fn unavailable(e: anyhow::Error) -> Response { .into_response() } -impl RangeQuery { - fn span(&self, now: DateTime) -> Result { - let (from, to) = match (self.from, self.to) { +fn some(value: &Option) -> Option { + value.as_deref().filter(|v| !v.is_empty()).map(|v| { + if v == NONE { + String::new() + } else { + v.to_string() + } + }) +} + +impl OverviewQuery { + /// The days the query covers, both included. + fn days(&self, today: NaiveDate) -> Result<(NaiveDate, NaiveDate), Response> { + match (self.from, self.to) { (Some(from), Some(to)) => { - if to <= from { - return Err(bad_request("to must be after from")); + if to < from { + return Err(bad_request("to must not be before from")); } - if to - from > Duration::days(MAX_RANGE_DAYS) { + if (to - from).num_days() >= query::MAX_DAYS { return Err(bad_request(format!( - "a range is at most {MAX_RANGE_DAYS} days" + "a range is at most {} days", + query::MAX_DAYS ))); } - (from, to) + Ok((from, to)) } (None, None) => { - let name = self.range.as_deref().unwrap_or("24h"); - let Some((_, secs)) = RANGES.iter().find(|(n, _)| *n == name) else { + let name = self.range.as_deref().unwrap_or("30d"); + let Some((_, days)) = RANGES.iter().find(|(n, _)| *n == name) else { let names: Vec<&str> = RANGES.iter().map(|(n, _)| *n).collect(); return Err(bad_request(format!( "range must be one of {}", names.join(", ") ))); }; - (now - Duration::seconds(*secs), now) + Ok((today - Duration::days(days - 1), today)) } - _ => return Err(bad_request("give both from and to, or neither")), - }; - Ok(Span::new(from, to, now)) - } - - fn instance(&self) -> Option { - self.instance.clone().filter(|i| !i.is_empty()) + _ => Err(bad_request("give both from and to, or neither")), + } } -} -/// The names of the users `ids` name, as far as they still exist. -async fn usernames(state: &AppState, ids: impl Iterator) -> HashMap { - let mut wanted: Vec = ids.collect(); - wanted.sort_unstable(); - wanted.dedup(); - wanted.truncate(MAX_USERNAMES); - let mut names = HashMap::new(); - for id in wanted { - match forge_db::user_dao::find_user_by_id(&state.pool, id).await { - Ok(Some(user)) => { - names.insert(id, user.username); - } - Ok(None) => {} - Err(e) => { - forge_db::observe_error(&e); - tracing::warn!(error = %e, "cannot look up the users of the traffic statistics"); - break; - } + fn filter(&self) -> Filter { + Filter { + instance: self.instance.clone().filter(|i| !i.is_empty()), + repository: some(&self.repository), + network: self.network.clone().filter(|n| !n.is_empty()), + country: some(&self.country), + asn: self.asn, } } - names } -/// What the views say of the settings, so the page can tell why something -/// is missing. +/// What is recorded now and what can be looked up, so the page can tell why +/// something is missing. #[derive(Debug, Serialize)] pub struct Recording { pub enabled: bool, pub client_addresses: bool, - pub request_log: bool, -} - -impl From<&TrafficSettings> for Recording { - fn from(s: &TrafficSettings) -> Self { - Recording { - enabled: s.enabled, - client_addresses: s.client_addresses, - request_log: s.request_log, - } - } + /// Whether this instance has a country database loaded. + pub countries: bool, + /// Whether this instance has an autonomous system database loaded. + pub asns: bool, } /// `GET /admin/traffic`. @@ -157,90 +147,29 @@ pub struct OverviewView { } /// `GET /admin/traffic` -pub async fn overview(State(state): State, Query(q): Query) -> Response { - let span = match q.span(Utc::now()) { - Ok(span) => span, +pub async fn overview(State(state): State, Query(q): Query) -> Response { + let (from, to) = match q.days(Utc::now().date_naive()) { + Ok(days) => days, Err(resp) => return resp, }; let settings = match stored(&state).await { Ok(settings) => settings, Err(resp) => return resp, }; - let filter = Filter { - instance: q.instance(), - repository: q.repository.clone().filter(|r| !r.is_empty()), - }; let traffic = &state.traffic; - let mut overview = match query::overview(&*traffic.objects, &traffic.cache, span, &filter).await - { - Ok(overview) => overview, - Err(e) => return unavailable(e), - }; - let names = usernames(&state, overview.clients.iter().filter_map(|c| c.user_id)).await; - for client in &mut overview.clients { - client.username = client.user_id.and_then(|id| names.get(&id).cloned()); - } + let overview = + match query::overview(&*traffic.objects, &traffic.cache, from, to, &q.filter()).await { + Ok(overview) => overview, + Err(e) => return unavailable(e), + }; Json(OverviewView { overview, - recording: Recording::from(&settings), - }) - .into_response() -} - -/// `GET /admin/traffic/client`. -#[derive(Debug, Serialize)] -pub struct ClientView { - #[serde(flatten)] - pub detail: query::ClientDetail, - pub recording: Recording, -} - -/// `GET /admin/traffic/client` -pub async fn client(State(state): State, Query(q): Query) -> Response { - let Some(network) = q.network.clone().filter(|n| !n.is_empty()) else { - return bad_request("network is required"); - }; - let span = match q.span(Utc::now()) { - Ok(span) => span, - Err(resp) => return resp, - }; - let settings = match stored(&state).await { - Ok(settings) => settings, - Err(resp) => return resp, - }; - let traffic = &state.traffic; - let instance = q.instance(); - let mut detail = match query::client( - &*traffic.objects, - &traffic.cache, - span, - instance.as_deref(), - &network, - ) - .await - { - Ok(detail) => detail, - Err(e) => return unavailable(e), - }; - let ids = detail - .totals - .iter() - .filter_map(|t| t.user_id) - .chain(detail.log.entries.iter().filter_map(|e| e.user_id)) - .chain(detail.log.accounts.iter().filter_map(|a| a.user_id)); - let names = usernames(&state, ids).await; - if let Some(totals) = &mut detail.totals { - totals.username = totals.user_id.and_then(|id| names.get(&id).cloned()); - } - for entry in &mut detail.log.entries { - entry.username = entry.user_id.and_then(|id| names.get(&id).cloned()); - } - for account in &mut detail.log.accounts { - account.username = account.user_id.and_then(|id| names.get(&id).cloned()); - } - Json(ClientView { - detail, - recording: Recording::from(&settings), + recording: Recording { + enabled: settings.enabled, + client_addresses: settings.client_addresses, + countries: traffic.geo.info(GeoKind::Country).is_some(), + asns: traffic.geo.info(GeoKind::Asn).is_some(), + }, }) .into_response() } @@ -264,10 +193,7 @@ async fn stored(state: &AppState) -> Result { pub struct TrafficSettingsUpdate { pub enabled: Option, pub client_addresses: Option, - pub request_log: Option, - pub statistics_retention_days: Option, - pub client_retention_days: Option, - pub request_log_retention_days: Option, + pub retention_days: Option, } /// `GET /admin/settings/traffic` @@ -289,23 +215,16 @@ pub async fn put_settings( Ok(settings) => settings, Err(resp) => return resp, }; - let mut next = current.clone(); let TrafficSettingsUpdate { enabled, client_addresses, - request_log, - statistics_retention_days, - client_retention_days, - request_log_retention_days, + retention_days, } = update; - next.enabled = enabled.unwrap_or(next.enabled); - next.client_addresses = client_addresses.unwrap_or(next.client_addresses); - next.request_log = request_log.unwrap_or(next.request_log); - next.statistics_retention_days = - statistics_retention_days.unwrap_or(next.statistics_retention_days); - next.client_retention_days = client_retention_days.unwrap_or(next.client_retention_days); - next.request_log_retention_days = - request_log_retention_days.unwrap_or(next.request_log_retention_days); + let next = TrafficSettings { + enabled: enabled.unwrap_or(current.enabled), + client_addresses: client_addresses.unwrap_or(current.client_addresses), + retention_days: retention_days.unwrap_or(current.retention_days), + }; if let Err(msg) = next.check() { return bad_request(msg); } @@ -313,7 +232,7 @@ pub async fn put_settings( if let Err(e) = write_settings(&state.pool, &next).await { return super::db_error(e); } - state.traffic.recorder.apply(&next); + state.traffic.recorder.set_enabled(next.enabled); tracing::info!( target: "forge::audit", settings = ?next, @@ -324,39 +243,239 @@ pub async fn put_settings( Json(next).into_response() } +/// `GET /admin/traffic/geoip`. +#[derive(Debug, Serialize)] +pub struct GeoipView { + /// The databases the answering instance loaded. + pub country: Option, + pub asn: Option, + /// The MaxMind download. + pub maxmind: MaxmindView, +} + +/// The MaxMind download, without the license key. +#[derive(Debug, Serialize)] +pub struct MaxmindView { + pub account_id: Option, + pub license_key_set: bool, + pub last_attempt: Option>, + pub last_success: Option>, + pub last_error: Option, + /// Where the databases are downloaded from (`traffic.maxmind_url`). + pub url: String, + /// The editions downloaded. + pub editions: Vec<&'static str>, +} + +async fn geoip_view(state: &AppState) -> Result { + let (stored, _) = maxmind::read_state(&state.pool) + .await + .map_err(super::db_error)?; + let license_key_set = maxmind::has_license_key(&state.pool) + .await + .map_err(super::db_error)?; + let MaxmindState { + account_id, + last_attempt, + last_success, + last_error, + } = stored; + Ok(GeoipView { + country: state.traffic.geo.info(GeoKind::Country), + asn: state.traffic.geo.info(GeoKind::Asn), + maxmind: MaxmindView { + account_id, + license_key_set, + last_attempt, + last_success, + last_error, + url: state.config.traffic.maxmind_base().to_string(), + editions: maxmind::EDITIONS.iter().map(|(_, e)| *e).collect(), + }, + }) +} + +/// `GET /admin/traffic/geoip` +pub async fn get_geoip(State(state): State) -> Response { + // Pick up databases another instance stored a moment ago. + crate::traffic::refresh_geo(&state.traffic).await; + match geoip_view(&state).await { + Ok(view) => Json(view).into_response(), + Err(resp) => resp, + } +} + +/// `PUT /admin/traffic/geoip` body. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct GeoipCredentials { + pub account_id: String, + /// Left out (or empty) keeps the stored one. + #[serde(default)] + pub license_key: Option, +} + +/// `PUT /admin/traffic/geoip` — set the MaxMind account ID and license key. +/// The databases are downloaded with the worker's next check, or at once +/// with `POST /admin/traffic/geoip/update`. +pub async fn put_geoip( + State(state): State, + axum::Extension(principal): axum::Extension, + Json(body): Json, +) -> Response { + let account_id = body.account_id.trim(); + if account_id.is_empty() + || account_id.len() > 32 + || !account_id.chars().all(|c| c.is_ascii_digit()) + { + return bad_request("account_id must be the MaxMind account ID, a number"); + } + let license_key = body + .license_key + .as_deref() + .map(str::trim) + .filter(|k| !k.is_empty()); + if let Some(key) = license_key { + if key.len() > 256 || !key.chars().all(|c| c.is_ascii_graphic()) { + return bad_request("license_key must be a MaxMind license key"); + } + } else { + match maxmind::has_license_key(&state.pool).await { + Ok(true) => {} + Ok(false) => return bad_request("license_key is required"), + Err(e) => return super::db_error(e), + } + } + if let Err(e) = maxmind::set_credentials(&state, account_id, license_key).await { + tracing::error!(error = %format!("{e:#}"), "cannot store the MaxMind credentials"); + return ( + StatusCode::INTERNAL_SERVER_ERROR, + "cannot store the MaxMind credentials", + ) + .into_response(); + } + tracing::info!( + target: "forge::audit", + account_id, + license_key_changed = license_key.is_some(), + principal = %principal.audit_label(), + "MaxMind credentials changed" + ); + match geoip_view(&state).await { + Ok(view) => Json(view).into_response(), + Err(resp) => resp, + } +} + +/// `DELETE /admin/traffic/geoip` — forget the MaxMind credentials and delete +/// the databases: requests are no longer located. +pub async fn delete_geoip( + State(state): State, + axum::Extension(principal): axum::Extension, +) -> Response { + if let Err(e) = maxmind::forget(&state).await { + tracing::error!(error = %format!("{e:#}"), "cannot remove the geolocation databases"); + return ( + StatusCode::SERVICE_UNAVAILABLE, + "cannot remove the geolocation databases", + ) + .into_response(); + } + tracing::info!( + target: "forge::audit", + principal = %principal.audit_label(), + "MaxMind credentials and geolocation databases removed" + ); + StatusCode::NO_CONTENT.into_response() +} + +/// `POST /admin/traffic/geoip/update` — download the databases now. Answers +/// the status, the download's error included; 409 without credentials. +pub async fn update_geoip(State(state): State) -> Response { + let (stored, raw) = match maxmind::read_state(&state.pool).await { + Ok(found) => found, + Err(e) => return super::db_error(e), + }; + match maxmind::has_license_key(&state.pool).await { + Ok(true) if stored.account_id.is_some() => {} + Ok(_) => { + return ( + StatusCode::CONFLICT, + "set the MaxMind account ID and license key first", + ) + .into_response() + } + Err(e) => return super::db_error(e), + } + match maxmind::claim(&state.pool, &stored, raw.as_deref(), Utc::now()).await { + Ok(true) => {} + Ok(false) => { + return ( + StatusCode::CONFLICT, + "another instance is downloading the databases; try again in a moment", + ) + .into_response() + } + Err(e) => return super::db_error(e), + } + // The outcome, a failure included, is in the answer's `last_error`. + let _ = maxmind::update(&state).await; + match geoip_view(&state).await { + Ok(view) => Json(view).into_response(), + Err(resp) => resp, + } +} + #[cfg(test)] mod tests { use super::*; - fn at(s: &str) -> DateTime { - DateTime::parse_from_rfc3339(s).unwrap().to_utc() + fn d(s: &str) -> NaiveDate { + NaiveDate::parse_from_str(s, "%Y-%m-%d").unwrap() } #[test] fn ranges_are_named_or_given() { - let now = at("2026-10-30T14:10:00Z"); - let span = RangeQuery::default().span(now).unwrap(); - assert_eq!(span.from, at("2026-10-29T14:10:00Z"), "24 hours by default"); - let q = RangeQuery { - range: Some("7d".into()), - ..RangeQuery::default() + let today = d("2026-10-30"); + assert_eq!( + OverviewQuery::default().days(today).unwrap(), + (d("2026-10-01"), today), + "30 days by default, today included" + ); + let q = OverviewQuery { + range: Some("today".into()), + ..OverviewQuery::default() }; - assert_eq!(q.span(now).unwrap().resolution, "hour"); - let q = RangeQuery { + assert_eq!(q.days(today).unwrap(), (today, today)); + let q = OverviewQuery { range: Some("2h".into()), - ..RangeQuery::default() + ..OverviewQuery::default() + }; + assert!(q.days(today).is_err()); + let q = OverviewQuery { + from: Some(d("2026-10-10")), + to: Some(d("2026-10-12")), + ..OverviewQuery::default() }; - assert!(q.span(now).is_err()); - let q = RangeQuery { - from: Some(at("2026-10-30T10:00:00Z")), - to: Some(at("2026-10-30T11:00:00Z")), - ..RangeQuery::default() + assert_eq!(q.days(today).unwrap(), (d("2026-10-10"), d("2026-10-12"))); + let q = OverviewQuery { + from: Some(d("2026-10-10")), + ..OverviewQuery::default() }; - assert_eq!(q.span(now).unwrap().to, at("2026-10-30T11:00:00Z")); - let q = RangeQuery { - from: Some(at("2026-10-30T10:00:00Z")), - ..RangeQuery::default() + assert!(q.days(today).is_err(), "from alone"); + } + + #[test] + fn a_dash_filters_for_none() { + let q = OverviewQuery { + repository: Some("-".into()), + country: Some("DE".into()), + network: Some(String::new()), + ..OverviewQuery::default() }; - assert!(q.span(now).is_err(), "from alone"); + let f = q.filter(); + assert_eq!(f.repository.as_deref(), Some("")); + assert_eq!(f.country.as_deref(), Some("DE")); + assert_eq!(f.network, None); } } diff --git a/crates/server/src/config.rs b/crates/server/src/config.rs index 4df17a6c..674170f6 100644 --- a/crates/server/src/config.rs +++ b/crates/server/src/config.rs @@ -96,6 +96,10 @@ pub struct Config { /// metrics report. #[serde(default)] pub telemetry: TelemetryConfig, + /// The traffic statistics (Admin → Traffic): where the geolocation + /// databases are downloaded from. What is recorded is set in the web UI. + #[serde(default)] + pub traffic: TrafficConfig, } /// `server.*` @@ -1458,6 +1462,50 @@ impl TelemetryConfig { } } +/// `[traffic]`: the traffic statistics ([`crate::traffic`]). What they +/// record, how long they keep it and the MaxMind account the geolocation +/// databases are downloaded with are set in the web UI (Admin → Traffic → +/// Settings). +#[derive(Debug, Clone, Deserialize)] +pub struct TrafficConfig { + /// Where the MaxMind databases are downloaded from: MaxMind's download + /// service, or a mirror that answers its API + /// (`{url}/geoip/databases/{edition}/download?suffix=tar.gz`, HTTP Basic + /// with the account ID and license key). Default + /// `https://download.maxmind.com`. + #[serde(default = "default_maxmind_url")] + pub maxmind_url: String, +} + +impl Default for TrafficConfig { + fn default() -> Self { + TrafficConfig { + maxmind_url: default_maxmind_url(), + } + } +} + +fn default_maxmind_url() -> String { + "https://download.maxmind.com".into() +} + +impl TrafficConfig { + fn validate(&self) -> Result<(), ConfigError> { + match url::Url::parse(&self.maxmind_url) { + Ok(url) if matches!(url.scheme(), "https" | "http") && url.host().is_some() => Ok(()), + _ => Err(ConfigError::Invalid(format!( + "traffic.maxmind_url must be an http(s) URL, not {:?}", + self.maxmind_url + ))), + } + } + + /// [`Self::maxmind_url`] without a trailing `/`. + pub fn maxmind_base(&self) -> &str { + self.maxmind_url.trim_end_matches('/') + } +} + /// `[docker]`: the Docker / OCI registry served under `/v2`. #[derive(Debug, Clone, Deserialize)] pub struct DockerConfig { @@ -2375,6 +2423,7 @@ impl Config { fn validate(&self) -> Result<(), ConfigError> { self.server.bind_addr()?; self.telemetry.validate()?; + self.traffic.validate()?; self.validate_public_base_url()?; // The DNS-rebinding guard's host list, checked before migrations run // rather than when the router is built. diff --git a/crates/server/src/images/dispatch.rs b/crates/server/src/images/dispatch.rs index 642a61bb..80c77607 100644 --- a/crates/server/src/images/dispatch.rs +++ b/crates/server/src/images/dispatch.rs @@ -290,7 +290,6 @@ impl Registry { .refresh_principal(principal) .await? .ok_or_else(invalid)?; - crate::traffic::note_principal(&principal); Ok(Caller { principal, claims: Some(claims), diff --git a/crates/server/src/middleware/principal.rs b/crates/server/src/middleware/principal.rs index 39b4e99b..7aee09e4 100644 --- a/crates/server/src/middleware/principal.rs +++ b/crates/server/src/middleware/principal.rs @@ -342,7 +342,6 @@ pub async fn attach_principal( .. } ); - crate::traffic::note_principal(&resolved.principal); req.extensions_mut().insert(resolved.principal); let mut resp = next.run(req).await; if named_by_headers { diff --git a/crates/server/src/plugins.rs b/crates/server/src/plugins.rs index ae893403..7bea6684 100644 --- a/crates/server/src/plugins.rs +++ b/crates/server/src/plugins.rs @@ -266,7 +266,6 @@ async fn dispatch_request(state: AppState, req: Request) -> Response { Ok(principal) => principal, Err(response) => return response, }; - crate::traffic::note_principal(&principal); let Some((repo_name, rest)) = split_path(uri.path()) else { return (StatusCode::NOT_FOUND, "repository not found").into_response(); }; diff --git a/crates/server/src/telemetry.rs b/crates/server/src/telemetry.rs index 7990ca13..e2fde0be 100644 --- a/crates/server/src/telemetry.rs +++ b/crates/server/src/telemetry.rs @@ -14,8 +14,8 @@ //! the HTTP metrics: request duration by surface (from the route, never //! the raw path), method and status, the requests in flight and the //! response body sizes. It also counts the request in the traffic -//! statistics ([`crate::traffic`]), which the metrics' bounded attributes -//! leave out: the repository, the client address, the path. +//! statistics ([`crate::traffic`]) by day, repository and client address, +//! which the metrics' bounded attributes leave out. //! * [`install_live_source`]: what the SDK reads from the process when it //! collects (the connection pool, the blob cache's counters, those of `GET //! /admin/blob-cache`, the circuit breaker, the staging space, the @@ -492,19 +492,8 @@ pub async fn track(State(recorder): State>, req: Request, next: Ne let span = request_span(&req, surface, route.as_deref()); // Health probes are no traffic. let counted = (surface != Surface::Health && recorder.enabled()).then(|| Counted { - path: if recorder.logs_requests() { - recorded_path(req.uri().path()).into_owned() - } else { - String::new() - }, - user_agent: req - .headers() - .get(header::USER_AGENT) - .and_then(|v| v.to_str().ok()) - .map(str::to_string), slot: forge_traffic::Slot::new(), recorder, - elapsed: Duration::ZERO, }); let started = Instant::now(); let active = forge_telemetry::http::Active::start(method); @@ -524,8 +513,7 @@ pub async fn track(State(recorder): State>, req: Request, next: Ne if status >= 500 { span.record("otel.status_code", "ERROR"); } - let elapsed = started.elapsed(); - forge_telemetry::http::request(surface, method, status, elapsed); + forge_telemetry::http::request(surface, method, status, started.elapsed()); response.map(|inner| { Body::new(CountingBody { inner, @@ -533,7 +521,7 @@ pub async fn track(State(recorder): State>, req: Request, next: Ne method, status, bytes: 0, - counted: counted.map(|c| Counted { elapsed, ..c }), + counted, _span: span, }) }) @@ -544,11 +532,6 @@ struct Counted { recorder: Arc, /// What the layers inside noted. slot: forge_traffic::Slot, - /// For the request log; empty when it is off. - path: String, - user_agent: Option, - /// Time to the response headers. - elapsed: Duration, } /// A response body that counts the bytes it yields: their total is @@ -597,13 +580,7 @@ impl Drop for CountingBody { if let Some(counted) = self.counted.take() { counted.recorder.record(forge_traffic::Event { at: chrono::Utc::now(), - surface: self.surface.as_str(), - method: self.method, - status: self.status, - duration: counted.elapsed, bytes: self.bytes, - path: counted.path, - user_agent: counted.user_agent, noted: counted.slot.take(), }); } diff --git a/crates/server/src/traffic/maxmind.rs b/crates/server/src/traffic/maxmind.rs new file mode 100644 index 00000000..9589c56c --- /dev/null +++ b/crates/server/src/traffic/maxmind.rs @@ -0,0 +1,378 @@ +//! The geolocation databases of the traffic statistics, downloaded from +//! MaxMind: GeoLite2-Country and GeoLite2-ASN, with the account ID and +//! license key an administrator sets (Admin → Traffic → Settings). +//! +//! * The license key is sealed in `forge_setting` +//! ([`forge_db::secret_dao::MAXMIND_LICENSE_KEY_SETTING`]); the account ID +//! and the downloads' outcome are kept next to it ([`STATE_SETTING`]). +//! * One instance downloads (the others see its claim on the state) and puts +//! the databases in the bucket ([`GeoKind::key`]); every instance loads them +//! from there ([`forge_traffic::geo::refresh`]). +//! * The databases are downloaded again every [`REFRESH_AFTER`] (MaxMind +//! publishes GeoLite2 twice a week, and its license asks for current +//! databases), at most every [`RETRY_AFTER`] after a failure. + +use std::io::Read; + +use chrono::{DateTime, Duration, Utc}; +use forge_crypto::{ExposeSecret, PlaintextSecret}; +use forge_db::authz_dao; +use forge_db::secret_dao::{self, MAXMIND_LICENSE_KEY_SETTING}; +use forge_traffic::GeoKind; +use serde::{Deserialize, Serialize}; + +use crate::state::AppState; + +/// The `forge_setting` key of the [`MaxmindState`]. +pub const STATE_SETTING: &str = "traffic_maxmind"; + +/// The editions downloaded, by the database they fill. +pub const EDITIONS: [(GeoKind, &str); 2] = [ + (GeoKind::Country, "GeoLite2-Country"), + (GeoKind::Asn, "GeoLite2-ASN"), +]; + +/// How old the databases may get before they are downloaded again. +pub const REFRESH_AFTER: Duration = Duration::days(3); +/// How long after an attempt the next one waits: a failure is not retried +/// every hour (MaxMind limits the downloads a day). +pub const RETRY_AFTER: Duration = Duration::hours(6); +/// The largest archive accepted, and the largest database in it. +const MAX_ARCHIVE_BYTES: usize = 256 << 20; +const MAX_DATABASE_BYTES: u64 = 512 << 20; +/// A download's deadline. +const DOWNLOAD_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(300); +/// The longest error kept. +const MAX_ERROR_BYTES: usize = 500; + +/// The MaxMind account and how its downloads went, kept in `forge_setting`. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct MaxmindState { + /// The MaxMind account ID; `None` without credentials. + pub account_id: Option, + /// When an instance last started a download. + pub last_attempt: Option>, + /// When the databases were last downloaded and stored. + pub last_success: Option>, + /// Why the last download failed, until one succeeds. + pub last_error: Option, +} + +/// The stored state and its raw value (for a compare-and-swap). +pub async fn read_state(pool: &sqlx::PgPool) -> sqlx::Result<(MaxmindState, Option)> { + let raw = authz_dao::get_setting(pool, STATE_SETTING).await?; + let state = match &raw { + None => MaxmindState::default(), + Some(raw) => serde_json::from_str(raw).map_err(|e| sqlx::Error::Decode(Box::new(e)))?, + }; + Ok((state, raw)) +} + +async fn write_state(pool: &sqlx::PgPool, state: &MaxmindState) -> sqlx::Result<()> { + let json = serde_json::to_string(state).map_err(|e| sqlx::Error::Encode(Box::new(e)))?; + authz_dao::set_setting(pool, STATE_SETTING, &json).await +} + +/// Store `state` while the stored value is still `expected` (`None`: unset). +/// Returns whether it was stored. +async fn swap_state( + pool: &sqlx::PgPool, + expected: Option<&str>, + state: &MaxmindState, +) -> sqlx::Result { + let json = serde_json::to_string(state).map_err(|e| sqlx::Error::Encode(Box::new(e)))?; + match expected { + Some(expected) => authz_dao::replace_setting(pool, STATE_SETTING, expected, &json).await, + None => authz_dao::insert_setting_if_absent(pool, STATE_SETTING, &json).await, + } +} + +/// The stored license key, opened; `None` when there is none. +pub async fn license_key(state: &AppState) -> anyhow::Result> { + let Some(value) = authz_dao::get_setting(&state.pool, MAXMIND_LICENSE_KEY_SETTING).await? + else { + return Ok(None); + }; + Ok(Some( + secret_dao::open_secret_setting(&state.envelope, MAXMIND_LICENSE_KEY_SETTING, &value) + .await?, + )) +} + +/// Whether a license key is stored. +pub async fn has_license_key(pool: &sqlx::PgPool) -> sqlx::Result { + Ok(authz_dao::get_setting(pool, MAXMIND_LICENSE_KEY_SETTING) + .await? + .is_some()) +} + +/// Store the account ID and, when given, the license key (sealed). The +/// downloads' outcome is cleared: the next check downloads with these. +pub async fn set_credentials( + state: &AppState, + account_id: &str, + license_key: Option<&str>, +) -> anyhow::Result<()> { + if let Some(key) = license_key { + let current = authz_dao::get_setting(&state.pool, MAXMIND_LICENSE_KEY_SETTING).await?; + let stored = secret_dao::store_secret_setting( + &state.pool, + &state.envelope, + MAXMIND_LICENSE_KEY_SETTING, + &PlaintextSecret::from(key.to_string()), + current.as_deref(), + ) + .await?; + if stored.is_none() { + anyhow::bail!("the license key changed while it was being saved; try again"); + } + } + write_state( + &state.pool, + &MaxmindState { + account_id: Some(account_id.to_string()), + ..MaxmindState::default() + }, + ) + .await?; + Ok(()) +} + +/// Forget the credentials and delete the databases from the bucket (every +/// instance unloads them with its next refresh). +pub async fn forget(state: &AppState) -> anyhow::Result<()> { + authz_dao::delete_setting(&state.pool, MAXMIND_LICENSE_KEY_SETTING).await?; + authz_dao::delete_setting(&state.pool, STATE_SETTING).await?; + for kind in GeoKind::ALL { + state.traffic.objects.delete(kind.key()).await?; + state.traffic.geo.unload(kind); + } + Ok(()) +} + +/// The HTTP client for MaxMind: bounded time, redirects followed (the +/// download answers with one to its storage; the credentials are not sent +/// along to another host). +fn http_client() -> reqwest::Client { + reqwest::Client::builder() + .timeout(DOWNLOAD_TIMEOUT) + .user_agent(concat!("forge/", env!("CARGO_PKG_VERSION"))) + .build() + .expect("the MaxMind HTTP client builds") +} + +fn clip(mut text: String) -> String { + if text.len() > MAX_ERROR_BYTES { + let mut end = MAX_ERROR_BYTES; + while !text.is_char_boundary(end) { + end -= 1; + } + text.truncate(end); + } + text +} + +/// The `.mmdb` file in the `.tar.gz` archive `archive`. +fn extract(archive: &[u8]) -> anyhow::Result> { + let mut tar = tar::Archive::new(flate2::read::GzDecoder::new(archive)); + for entry in tar.entries()? { + let entry = entry?; + let is_database = entry.path()?.extension().is_some_and(|ext| ext == "mmdb"); + if !is_database { + continue; + } + let size = entry.header().size()?; + if size > MAX_DATABASE_BYTES { + anyhow::bail!("the database in the archive is larger than {MAX_DATABASE_BYTES} bytes"); + } + let mut bytes = Vec::with_capacity(usize::try_from(size).unwrap_or(0)); + entry.take(MAX_DATABASE_BYTES).read_to_end(&mut bytes)?; + return Ok(bytes); + } + anyhow::bail!("the archive holds no .mmdb file") +} + +/// Download `edition` with the account `account_id` and `license_key`. +async fn download( + http: &reqwest::Client, + base: &str, + account_id: &str, + license_key: &PlaintextSecret, + edition: &str, +) -> anyhow::Result> { + let url = format!("{base}/geoip/databases/{edition}/download?suffix=tar.gz"); + let response = http + .get(&url) + .basic_auth(account_id, Some(license_key.expose_secret())) + .send() + .await + .map_err(|e| anyhow::anyhow!("cannot reach {base}: {e}"))?; + let status = response.status(); + if status == reqwest::StatusCode::UNAUTHORIZED { + anyhow::bail!("MaxMind refused the account ID or the license key (401)"); + } + if !status.is_success() { + let body = response.text().await.unwrap_or_default(); + anyhow::bail!("MaxMind answered {status} for {edition}: {}", clip(body)); + } + let mut archive = Vec::new(); + let mut response = response; + while let Some(chunk) = response.chunk().await? { + if archive.len() + chunk.len() > MAX_ARCHIVE_BYTES { + anyhow::bail!("the {edition} archive is larger than {MAX_ARCHIVE_BYTES} bytes"); + } + archive.extend_from_slice(&chunk); + } + tokio::task::spawn_blocking(move || extract(&archive)).await? +} + +/// Download both databases, check them, store them in the bucket and load +/// them here, and record the outcome. The caller has claimed the attempt. +pub async fn update(state: &AppState) -> anyhow::Result<()> { + let result = async { + let (stored, _) = read_state(&state.pool).await?; + let account_id = stored + .account_id + .ok_or_else(|| anyhow::anyhow!("no MaxMind account ID is set"))?; + let key = license_key(state) + .await? + .ok_or_else(|| anyhow::anyhow!("no MaxMind license key is set"))?; + let http = http_client(); + let base = state.config.traffic.maxmind_base(); + for (kind, edition) in EDITIONS { + let bytes = download(&http, base, &account_id, &key, edition).await?; + forge_traffic::geo::check(kind, bytes.clone()) + .map_err(|e| anyhow::anyhow!("the {edition} download is no database: {e}"))?; + state.traffic.objects.put(kind.key(), bytes).await?; + } + forge_traffic::geo::refresh(&*state.traffic.objects, &state.traffic.geo).await?; + anyhow::Ok(()) + } + .await; + let (mut recorded, _) = read_state(&state.pool).await.unwrap_or_default(); + match &result { + Ok(()) => { + recorded.last_success = Some(Utc::now()); + recorded.last_error = None; + tracing::info!("downloaded the MaxMind geolocation databases"); + } + Err(e) => { + recorded.last_error = Some(clip(format!("{e:#}"))); + tracing::warn!(error = %format!("{e:#}"), "cannot download the MaxMind databases"); + } + } + if let Err(e) = write_state(&state.pool, &recorded).await { + tracing::warn!(error = %e, "cannot record the MaxMind download"); + } + result +} + +/// Claim an attempt: record `now` as the last attempt, unless another +/// instance did since `raw` was read. Returns whether this instance won. +pub async fn claim( + pool: &sqlx::PgPool, + stored: &MaxmindState, + raw: Option<&str>, + now: DateTime, +) -> sqlx::Result { + let claimed = MaxmindState { + last_attempt: Some(now), + ..stored.clone() + }; + swap_state(pool, raw, &claimed).await +} + +/// Whether the databases are due for a download: credentials are set, the +/// last download is older than [`REFRESH_AFTER`] (or a database is missing), +/// and the last attempt is older than [`RETRY_AFTER`]. +pub fn due(stored: &MaxmindState, has_key: bool, missing: bool, now: DateTime) -> bool { + let stale = missing || stored.last_success.is_none_or(|t| now - t >= REFRESH_AFTER); + let rested = stored.last_attempt.is_none_or(|t| now - t >= RETRY_AFTER); + stored.account_id.is_some() && has_key && stale && rested +} + +/// The worker's check: download the databases if they are due and this +/// instance claims the download. +pub async fn update_if_due(state: &AppState) -> anyhow::Result<()> { + let (stored, raw) = read_state(&state.pool).await?; + if stored.account_id.is_none() { + return Ok(()); + } + let has_key = has_license_key(&state.pool).await?; + let listed = state.traffic.objects.list("traffic/v1/geoip/").await?; + let missing = GeoKind::ALL + .iter() + .any(|k| !listed.iter().any(|o| o.key == k.key())); + let now = Utc::now(); + if !due(&stored, has_key, missing, now) { + return Ok(()); + } + if !claim(&state.pool, &stored, raw.as_deref(), now).await? { + return Ok(()); + } + update(state).await +} + +#[cfg(test)] +mod tests { + use super::*; + + fn archive(name: &str, body: &[u8]) -> Vec { + let mut tar = tar::Builder::new(Vec::new()); + let mut header = tar::Header::new_gnu(); + header.set_size(body.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + tar.append_data(&mut header, name, body).unwrap(); + let tar = tar.into_inner().unwrap(); + let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + std::io::Write::write_all(&mut gz, &tar).unwrap(); + gz.finish().unwrap() + } + + #[test] + fn the_database_is_taken_from_its_archive() { + let a = archive("GeoLite2-ASN_20261030/GeoLite2-ASN.mmdb", b"database"); + assert_eq!(extract(&a).unwrap(), b"database"); + let none = archive("GeoLite2-ASN_20261030/LICENSE.txt", b"text"); + assert!(extract(&none).unwrap_err().to_string().contains("no .mmdb")); + assert!(extract(b"not gzip").is_err()); + } + + #[test] + fn downloads_are_due_when_old_and_rested() { + let now = Utc::now(); + let set = MaxmindState { + account_id: Some("1".into()), + ..MaxmindState::default() + }; + assert!(due(&set, true, false, now), "never downloaded"); + assert!(!due(&set, false, false, now), "no license key"); + assert!( + !due(&MaxmindState::default(), true, true, now), + "no account" + ); + let fresh = MaxmindState { + last_success: Some(now - Duration::hours(1)), + last_attempt: Some(now - Duration::hours(1)), + ..set.clone() + }; + assert!(!due(&fresh, true, false, now)); + let old = MaxmindState { + last_success: Some(now - Duration::days(4)), + last_attempt: Some(now - Duration::days(4)), + ..set.clone() + }; + assert!(due(&old, true, false, now)); + let failed_recently = MaxmindState { + last_attempt: Some(now - Duration::hours(1)), + last_error: Some("401".into()), + ..set + }; + assert!( + !due(&failed_recently, true, true, now), + "rests after a failure" + ); + } +} diff --git a/crates/server/src/traffic.rs b/crates/server/src/traffic/mod.rs similarity index 69% rename from crates/server/src/traffic.rs rename to crates/server/src/traffic/mod.rs index de2a000c..69f70c14 100644 --- a/crates/server/src/traffic.rs +++ b/crates/server/src/traffic/mod.rs @@ -1,6 +1,7 @@ //! The traffic statistics of Admin → Traffic ([`forge_traffic`]): what the //! server notes about a request for its count, the settings, and the worker -//! that writes the files and deletes the expired ones. +//! that writes the files, loads the geolocation databases and deletes the +//! expired files. //! //! * The client address is the one the principal layer resolves (the socket //! peer, or what a trusted proxy forwarded), counted under its network as @@ -9,15 +10,19 @@ //! * The settings ([`TrafficSettings`]) are kept in `forge_setting` under //! [`SETTINGS_KEY`]; every instance follows a change with its next flush. //! * The worker flushes once a minute, a couple of seconds after the minute -//! turns, and prunes about once an hour, at a random offset per instance. +//! turns; it loads the geolocation databases from the bucket when they +//! change ([`GEO_REFRESH_INTERVAL`]), and about once an hour, at a random +//! offset per instance, prunes and downloads the databases from MaxMind +//! when they are due ([`maxmind`]). + +pub mod maxmind; use std::net::IpAddr; use std::time::Duration; use chrono::{DateTime, Utc}; use forge_core::auth::login_throttle::client_network; -use forge_core::auth::principal::{AuthMethod, Principal}; -use forge_traffic::{Account, Traffic, TrafficSettings}; +use forge_traffic::{Traffic, TrafficSettings}; use sqlx::PgPool; use tokio::task::JoinHandle; use tracing::Instrument; @@ -30,8 +35,11 @@ pub const SETTINGS_KEY: &str = "traffic"; /// The worker's name (`forge.worker`). pub const WORKER: &str = "traffic"; -/// How often the worker prunes. -const PRUNE_INTERVAL: Duration = Duration::from_secs(3_600); +/// How often the worker prunes and checks the MaxMind download. +const HOURLY: Duration = Duration::from_secs(3_600); + +/// How often the worker looks for new geolocation databases in the bucket. +pub const GEO_REFRESH_INTERVAL: Duration = Duration::from_secs(600); /// How long the last flush on shutdown may take. const SHUTDOWN_FLUSH: Duration = Duration::from_secs(10); @@ -50,38 +58,6 @@ pub fn note_client(ip: IpAddr) { forge_traffic::note_client(ip.to_canonical(), network_label(ip)); } -/// The account `principal` is, as the statistics name it. -pub fn account_of(principal: &Principal) -> Account { - match principal { - Principal::Unauthenticated | Principal::Anonymous { .. } => Account { - kind: "anonymous", - user_id: None, - }, - Principal::User { - user_id, - token_id, - method, - } => Account { - kind: match (token_id, method) { - (Some(_), _) | (None, AuthMethod::Token) => "token", - (None, AuthMethod::Session) => "session", - (None, AuthMethod::TrustedHeader) => "trusted_header", - (None, AuthMethod::Password) => "password", - }, - user_id: Some(*user_id), - }, - Principal::Workload { .. } => Account { - kind: "workload", - user_id: None, - }, - } -} - -/// Note the account the request runs as. -pub fn note_principal(principal: &Principal) { - forge_traffic::note_account(account_of(principal)); -} - /// The stored [`TrafficSettings`], or the defaults when never saved. pub async fn read_settings(pool: &PgPool) -> sqlx::Result { match forge_db::authz_dao::get_setting(pool, SETTINGS_KEY).await? { @@ -105,7 +81,7 @@ async fn follow_settings( ) -> TrafficSettings { match read_settings(pool).await { Ok(settings) => { - traffic.recorder.apply(&settings); + traffic.recorder.set_enabled(settings.enabled); settings } Err(e) => { @@ -116,6 +92,17 @@ async fn follow_settings( } } +/// Load the geolocation databases from the bucket when they changed. +pub async fn refresh_geo(traffic: &Traffic) -> bool { + match forge_traffic::geo::refresh(&*traffic.objects, &traffic.geo).await { + Ok(()) => true, + Err(e) => { + tracing::warn!(error = %format!("{e:#}"), "cannot look for geolocation databases"); + false + } + } +} + /// How long until `now`'s next minute is two seconds old. fn until_next_flush(now: DateTime) -> Duration { let into_minute = @@ -123,24 +110,31 @@ fn until_next_flush(now: DateTime) -> Duration { Duration::from_millis(u64::try_from(62_000 - into_minute).unwrap_or(0)) } -/// The worker: flush every minute, prune every hour. +/// The worker: flush every minute, look for new geolocation databases every +/// ten, prune and check the MaxMind download every hour. pub fn spawn_worker(state: AppState) -> JoinHandle<()> { tokio::spawn(async move { let traffic = state.traffic.clone(); let mut settings = follow_settings(&state.pool, &traffic, &TrafficSettings::default()).await; - // Instances started together prune at different times. - let mut next_prune = tokio::time::Instant::now() - + Duration::from_secs(rand::random_range(60..PRUNE_INTERVAL.as_secs())); + refresh_geo(&traffic).await; + let mut next_geo = tokio::time::Instant::now() + GEO_REFRESH_INTERVAL; + // Instances started together prune and download at different times. + let mut next_hourly = tokio::time::Instant::now() + + Duration::from_secs(rand::random_range(60..HOURLY.as_secs())); loop { tokio::time::sleep(until_next_flush(Utc::now())).await; let run = forge_telemetry::workers::Run::start(WORKER); let ok = async { settings = follow_settings(&state.pool, &traffic, &settings).await; - let report = traffic.flush(&settings, Utc::now()).await; - let mut ok = report.failures == 0; - if tokio::time::Instant::now() >= next_prune { - next_prune = tokio::time::Instant::now() + PRUNE_INTERVAL; + let mut ok = true; + if tokio::time::Instant::now() >= next_geo { + next_geo = tokio::time::Instant::now() + GEO_REFRESH_INTERVAL; + ok &= refresh_geo(&traffic).await; + } + ok &= traffic.flush(&settings, Utc::now()).await.failures == 0; + if tokio::time::Instant::now() >= next_hourly { + next_hourly = tokio::time::Instant::now() + HOURLY; match forge_traffic::layout::prune(&*traffic.objects, &settings, Utc::now()) .await { @@ -153,6 +147,13 @@ pub fn spawn_worker(state: AppState) -> JoinHandle<()> { tracing::warn!(error = %e, "cannot delete expired traffic statistics"); } } + if let Err(e) = maxmind::update_if_due(&state).await { + ok = false; + tracing::warn!( + error = %format!("{e:#}"), + "the MaxMind geolocation download failed" + ); + } } ok } @@ -168,7 +169,7 @@ pub fn spawn_worker(state: AppState) -> JoinHandle<()> { pub async fn flush_on_shutdown(traffic: &Traffic, pool: &PgPool) { let flush = async { let settings = read_settings(pool).await.unwrap_or_default(); - traffic.flush_all(&settings).await + traffic.flush(&settings, Utc::now()).await }; match tokio::time::timeout(SHUTDOWN_FLUSH, flush).await { Ok(report) if report.failures > 0 => { @@ -211,23 +212,4 @@ mod tests { Duration::from_millis(2_500) ); } - - #[test] - fn accounts_name_how_the_caller_signed_in() { - let user = |token_id, method| Principal::User { - user_id: 9, - token_id, - method, - }; - assert_eq!( - account_of(&user(Some(1), AuthMethod::Password)).kind, - "token" - ); - assert_eq!(account_of(&user(None, AuthMethod::Session)).kind, "session"); - assert_eq!( - account_of(&user(None, AuthMethod::Session)).user_id, - Some(9) - ); - assert_eq!(account_of(&Principal::Unauthenticated).kind, "anonymous"); - } } diff --git a/crates/server/tests/traffic.rs b/crates/server/tests/traffic.rs index e63a953d..020f2c44 100644 --- a/crates/server/tests/traffic.rs +++ b/crates/server/tests/traffic.rs @@ -2,38 +2,54 @@ //! (Postgres + versitygw via the shared harness), or //! `FORGE_TEST_DATABASE_URL` / `FORGE_TEST_S3_ENDPOINT`. //! -//! Requests are counted with what the layers know of them (the repository, -//! the client address, the account), written to the object store by a flush -//! and read back by the admin API: the overview, a client's view with its -//! request log, and the settings. +//! Requests are counted by day, repository and client network, written to +//! the object store by a flush and read back by the admin API; the +//! geolocation databases are downloaded from a mock of MaxMind's download +//! service, stored in the bucket and used to locate the clients. mod common; use common::{unique, TestServer}; +use forge_traffic::testing::{asn_db, country_db}; use forge_traffic::TrafficSettings; use reqwest::StatusCode; use serde_json::{json, Value}; +use wiremock::matchers::{header, method, path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; -async fn server() -> TestServer { +async fn server(extra_toml: Option) -> TestServer { common::start_with( common::fresh_pool().await, common::ServerOpts { bootstrap_admin: Some((format!("admin{}@example.com", unique()), "adminpw".into())), + extra_toml, ..Default::default() }, ) .await } -async fn admin_get(srv: &TestServer, path: &str) -> (StatusCode, Value) { +async fn admin( + srv: &TestServer, + verb: &str, + path: &str, + body: Option, +) -> (StatusCode, Value) { let (u, p) = srv.admin_basic(); - let resp = srv - .client() - .get(format!("{}{path}", srv.base_url)) - .basic_auth(&u, Some(&p)) - .send() - .await - .unwrap(); + let url = format!("{}{path}", srv.base_url); + let client = srv.client(); + let request = match verb { + "GET" => client.get(url), + "PUT" => client.put(url), + "POST" => client.post(url), + "DELETE" => client.delete(url), + _ => unreachable!(), + }; + let request = match body { + Some(body) => request.json(&body), + None => request, + }; + let resp = request.basic_auth(&u, Some(&p)).send().await.unwrap(); let status = resp.status(); let text = resp.text().await.unwrap(); ( @@ -42,53 +58,51 @@ async fn admin_get(srv: &TestServer, path: &str) -> (StatusCode, Value) { ) } -async fn put_settings(srv: &TestServer, body: Value) -> (StatusCode, Value) { - let (u, p) = srv.admin_basic(); - let resp = srv - .client() - .put(format!("{}/admin/settings/traffic", srv.base_url)) - .basic_auth(&u, Some(&p)) - .json(&body) - .send() - .await - .unwrap(); - let status = resp.status(); - let text = resp.text().await.unwrap(); - ( - status, - serde_json::from_str(&text).unwrap_or(Value::String(text)), +async fn create_raw(srv: &TestServer) -> String { + let name = format!("traffic{}", unique()); + let (status, body) = admin( + srv, + "POST", + "/admin/repositories", + Some(json!({"name": name, "kind": "hosted", "format": "raw"})), ) + .await; + assert_eq!(status, StatusCode::CREATED, "{body}"); + name +} + +/// A `.tar.gz` holding `database` as MaxMind ships it. +fn archive(edition: &str, database: &[u8]) -> Vec { + let mut tar = tar::Builder::new(Vec::new()); + let mut header = tar::Header::new_gnu(); + header.set_size(database.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + tar.append_data( + &mut header, + format!("{edition}_20261030/{edition}.mmdb"), + database, + ) + .unwrap(); + let tar = tar.into_inner().unwrap(); + let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + std::io::Write::write_all(&mut gz, &tar).unwrap(); + gz.finish().unwrap() } #[tokio::test] -async fn requests_are_counted_written_and_read_back() { - let srv = server().await; +async fn requests_are_counted_by_day_repository_and_client() { + let srv = server(None).await; let (u, p) = srv.admin_basic(); let client = srv.client(); + let name = create_raw(&srv).await; - let name = format!("traffic{}", unique()); - let resp = client - .post(format!("{}/admin/repositories", srv.base_url)) - .basic_auth(&u, Some(&p)) - .json(&json!({"name": name, "kind": "hosted", "format": "raw"})) - .send() - .await - .unwrap(); - assert_eq!( - resp.status(), - StatusCode::CREATED, - "{}", - resp.text().await.unwrap() - ); - - // Two answers from the repository, one refused, a health probe that is - // not counted, and a repository that does not exist (its name is not - // recorded). + // Three requests to the repository, a health probe that is not counted, + // and a repository that does not exist (its name is not recorded). let file = format!("{}/repository/{name}/dir/hello.txt", srv.base_url); let resp = client .put(&file) .basic_auth(&u, Some(&p)) - .header("user-agent", "curl/8.10") .body("hello") .send() .await @@ -97,7 +111,6 @@ async fn requests_are_counted_written_and_read_back() { let resp = client .get(&file) .basic_auth(&u, Some(&p)) - .header("user-agent", "curl/8.10") .send() .await .unwrap(); @@ -121,18 +134,20 @@ async fn requests_are_counted_written_and_read_back() { // What a minute's flush writes, written now. let traffic = &srv.state.traffic; - let report = traffic.flush_all(&TrafficSettings::default()).await; + let report = traffic + .flush(&TrafficSettings::default(), chrono::Utc::now()) + .await; assert_eq!(report.failures, 0, "{report:?}"); - assert!(report.log_rows >= 5, "{report:?}"); let instance = traffic.instance.to_string(); - let (status, overview) = admin_get( + let (status, overview) = admin( &srv, - &format!("/admin/traffic?range=1h&instance={instance}"), + "GET", + &format!("/admin/traffic?range=today&instance={instance}"), + None, ) .await; assert_eq!(status, StatusCode::OK, "{overview}"); - assert_eq!(overview["resolution"], "minute"); assert!( overview["instances"] .as_array() @@ -141,60 +156,54 @@ async fn requests_are_counted_written_and_read_back() { .any(|i| i["id"] == instance.as_str()), "{overview}" ); + assert_eq!(overview["days"].as_array().unwrap().len(), 1); let repositories = overview["repositories"].as_array().unwrap(); - assert_eq!( - repositories.len(), - 1, - "only the repository that exists: {overview}" - ); - assert_eq!(repositories[0]["name"], name.as_str()); - assert_eq!(repositories[0]["requests"], 3); - assert_eq!(repositories[0]["client_errors"], 1); - let surfaces: Vec<&str> = overview["surfaces"] - .as_array() - .unwrap() + let repo = repositories .iter() - .map(|s| s["surface"].as_str().unwrap()) - .collect(); - assert!(surfaces.contains(&"plugin_raw"), "{surfaces:?}"); - assert!(!surfaces.contains(&"health"), "{surfaces:?}"); - assert!(overview["totals"]["denied"].as_u64().unwrap() >= 1); - let client_row = &overview["clients"][0]; - assert_eq!(client_row["network"], "127.0.0.1"); - assert_eq!(overview["recording"]["enabled"], true); - - // Only this repository's counts. - let (_, only) = admin_get( + .find(|r| r["name"] == name.as_str()) + .expect("the repository is listed"); + assert_eq!(repo["requests"], 3); + assert_eq!(repo["clients"], 1); + assert!( + repositories + .iter() + .all(|r| !r["name"].as_str().unwrap().starts_with("no-such-")), + "only repositories that exist: {overview}" + ); + let top = &overview["clients"][0]; + assert_eq!(top["network"], "127.0.0.1"); + assert_eq!(top["repositories"][0], name.as_str()); + assert_eq!(overview["recording"]["countries"], false, "no database yet"); + + // One repository's requests, then the requests to none. + let (_, only) = admin( &srv, - &format!("/admin/traffic?range=1h&instance={instance}&repository={name}"), + "GET", + &format!("/admin/traffic?range=today&instance={instance}&repository={name}"), + None, ) .await; assert_eq!(only["totals"]["requests"], 3); - - // The client's view: its requests in the request log, newest first. - let (status, detail) = admin_get( + assert!( + only["totals"]["bytes"].as_u64().unwrap() >= 5, + "the response bodies, the download's included: {only}" + ); + let (_, none) = admin( &srv, - &format!("/admin/traffic/client?network=127.0.0.1&range=1h&instance={instance}"), + "GET", + &format!("/admin/traffic?range=today&instance={instance}&repository=-"), + None, ) .await; - assert_eq!(status, StatusCode::OK, "{detail}"); - assert!(detail["log"]["matched"].as_u64().unwrap() >= 5, "{detail}"); - let entries = detail["log"]["entries"].as_array().unwrap(); - let get = entries + assert!(none["totals"]["requests"].as_u64().unwrap() >= 2, "{none}"); + assert!(none["repositories"] + .as_array() + .unwrap() .iter() - .find(|e| e["method"] == "GET" && e["status"] == 200 && e["repository"] == name.as_str()) - .expect("the download is in the log"); - assert_eq!(get["path"], format!("/repository/{name}/dir/hello.txt")); - assert_eq!(get["user_agent"], "curl/8.10"); - assert_eq!(get["bytes"], 5); - assert_eq!(get["account"], "token"); - assert_eq!(get["username"], u.as_str()); - - let (status, _) = admin_get(&srv, "/admin/traffic/client?range=1h").await; - assert_eq!(status, StatusCode::BAD_REQUEST, "network is required"); - let (status, _) = admin_get(&srv, "/admin/traffic?range=2h").await; - assert_eq!(status, StatusCode::BAD_REQUEST); + .all(|r| r["name"] == "")); + let (status, _) = admin(&srv, "GET", "/admin/traffic?range=2h", None).await; + assert_eq!(status, StatusCode::BAD_REQUEST); // Admins only. let resp = client .get(format!("{}/admin/traffic", srv.base_url)) @@ -205,34 +214,168 @@ async fn requests_are_counted_written_and_read_back() { } #[tokio::test] -async fn settings_switch_parts_off_and_are_checked() { - let srv = server().await; - let (status, settings) = admin_get(&srv, "/admin/settings/traffic").await; +async fn settings_are_checked_and_applied() { + let srv = server(None).await; + let (status, settings) = admin(&srv, "GET", "/admin/settings/traffic", None).await; assert_eq!(status, StatusCode::OK); assert_eq!( serde_json::from_value::(settings).unwrap(), TrafficSettings::default() ); - let (status, settings) = put_settings( + let (status, settings) = admin( &srv, - json!({"request_log": false, "client_retention_days": 7}), + "PUT", + "/admin/settings/traffic", + Some(json!({"enabled": false, "retention_days": 30})), ) .await; assert_eq!(status, StatusCode::OK, "{settings}"); - assert_eq!(settings["request_log"], false); - assert_eq!(settings["client_retention_days"], 7); + assert_eq!(settings["retention_days"], 30); assert_eq!(settings["client_addresses"], true, "untouched"); - assert!( - !srv.state.traffic.recorder.logs_requests(), - "applied at once" - ); + assert!(!srv.state.traffic.recorder.enabled(), "applied at once"); - let (status, _) = put_settings(&srv, json!({"request_log_retention_days": 0})).await; + let (status, _) = admin( + &srv, + "PUT", + "/admin/settings/traffic", + Some(json!({"retention_days": 0})), + ) + .await; assert_eq!(status, StatusCode::BAD_REQUEST); - let (status, _) = put_settings(&srv, json!({"unknown": true})).await; + let (status, _) = admin( + &srv, + "PUT", + "/admin/settings/traffic", + Some(json!({"unknown": true})), + ) + .await; assert!(status.is_client_error()); + let (_, settings) = admin(&srv, "GET", "/admin/settings/traffic", None).await; + assert_eq!(settings["enabled"], false, "stored"); +} + +#[tokio::test] +async fn maxmind_databases_are_downloaded_and_locate_clients() { + let maxmind = MockServer::start().await; + // "1234:secret", as MaxMind's download expects it. + let auth = "Basic MTIzNDpzZWNyZXQ="; + for (edition, database) in [ + ("GeoLite2-Country", country_db(&[("127.0.0.0/8", "DE")])), + ( + "GeoLite2-ASN", + asn_db(&[("127.0.0.0/8", 64_500, "Example AS")]), + ), + ] { + Mock::given(method("GET")) + .and(path(format!("/geoip/databases/{edition}/download"))) + .and(header("authorization", auth)) + .respond_with(ResponseTemplate::new(200).set_body_bytes(archive(edition, &database))) + .mount(&maxmind) + .await; + } + Mock::given(method("GET")) + .respond_with(ResponseTemplate::new(401).set_body_string("Invalid license key")) + .mount(&maxmind) + .await; + let srv = server(Some(format!( + "[traffic]\nmaxmind_url = \"{}\"\n", + maxmind.uri() + ))) + .await; + + let (status, view) = admin(&srv, "GET", "/admin/traffic/geoip", None).await; + assert_eq!(status, StatusCode::OK, "{view}"); + assert_eq!(view["maxmind"]["license_key_set"], false); + let (status, _) = admin(&srv, "POST", "/admin/traffic/geoip/update", None).await; + assert_eq!(status, StatusCode::CONFLICT, "no credentials yet"); + let (status, _) = admin( + &srv, + "PUT", + "/admin/traffic/geoip", + Some(json!({"account_id": "1234"})), + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST, "a license key is needed"); + + // A wrong key: the download fails and says why. + let (status, view) = admin( + &srv, + "PUT", + "/admin/traffic/geoip", + Some(json!({"account_id": "1234", "license_key": "wrong"})), + ) + .await; + assert_eq!(status, StatusCode::OK, "{view}"); + assert_eq!(view["maxmind"]["license_key_set"], true); + assert!( + view["maxmind"].get("license_key").is_none(), + "never answered" + ); + let (_, view) = admin(&srv, "POST", "/admin/traffic/geoip/update", None).await; + assert!( + view["maxmind"]["last_error"] + .as_str() + .unwrap() + .contains("refused the account ID or the license key"), + "{view}" + ); + let sealed = forge_db::authz_dao::get_setting(&srv.pool, "traffic_maxmind_license_key") + .await + .unwrap() + .unwrap(); + assert!( + sealed.starts_with("sealed:") && !sealed.contains("wrong"), + "sealed at rest" + ); + + // The right key (the account ID alone keeps it otherwise). + admin( + &srv, + "PUT", + "/admin/traffic/geoip", + Some(json!({"account_id": "1234", "license_key": "secret"})), + ) + .await; + let (status, view) = admin(&srv, "POST", "/admin/traffic/geoip/update", None).await; + assert_eq!(status, StatusCode::OK, "{view}"); + assert_eq!(view["maxmind"]["last_error"], Value::Null, "{view}"); + assert_eq!(view["country"]["database_type"], "GeoLite2-Country"); + assert_eq!(view["asn"]["database_type"], "GeoLite2-ASN"); + + // A request is now located. + let name = create_raw(&srv).await; + srv.client() + .get(format!("{}/repository/{name}/x", srv.base_url)) + .send() + .await + .unwrap(); + let traffic = &srv.state.traffic; + traffic + .flush(&TrafficSettings::default(), chrono::Utc::now()) + .await; + let instance = traffic.instance.to_string(); + let (_, overview) = admin( + &srv, + "GET", + &format!("/admin/traffic?range=today&instance={instance}&country=DE"), + None, + ) + .await; + assert!( + overview["totals"]["requests"].as_u64().unwrap() >= 1, + "{overview}" + ); + assert_eq!(overview["countries"][0]["country"], "DE"); + assert_eq!(overview["asns"][0]["asn"], 64_500); + assert_eq!(overview["asns"][0]["org"], "Example AS"); + assert_eq!(overview["clients"][0]["country"], "DE"); + assert_eq!(overview["recording"]["countries"], true); - let (_, settings) = admin_get(&srv, "/admin/settings/traffic").await; - assert_eq!(settings["request_log"], false, "stored"); + // Turned off: the credentials and the databases are gone. + let (status, _) = admin(&srv, "DELETE", "/admin/traffic/geoip", None).await; + assert_eq!(status, StatusCode::NO_CONTENT); + let (_, view) = admin(&srv, "GET", "/admin/traffic/geoip", None).await; + assert_eq!(view["country"], Value::Null); + assert_eq!(view["maxmind"]["license_key_set"], false); } diff --git a/crates/traffic/Cargo.toml b/crates/traffic/Cargo.toml index a86bc92b..4b897f4a 100644 --- a/crates/traffic/Cargo.toml +++ b/crates/traffic/Cargo.toml @@ -14,6 +14,8 @@ forge-storage = { path = "../storage" } tokio = { workspace = true, features = ["sync", "time", "rt"] } parquet = { workspace = true } parquet_derive = { workspace = true } +# Country and ASN lookups in MaxMind databases (GeoLite2, GeoIP2). +maxminddb = { workspace = true } bytes = "1" chrono = { workspace = true, features = ["serde"] } serde = { workspace = true } @@ -27,3 +29,8 @@ rand = { workspace = true } [dev-dependencies] tokio = { workspace = true, features = ["rt-multi-thread", "macros", "sync", "time"] } serde_json = { workspace = true } + +[features] +# `testing`: a writer of small MaxMind databases, for the tests of the crates +# that use this one. +testing = [] diff --git a/crates/traffic/src/geo.rs b/crates/traffic/src/geo.rs new file mode 100644 index 00000000..43c8efb2 --- /dev/null +++ b/crates/traffic/src/geo.rs @@ -0,0 +1,349 @@ +//! The country and autonomous system of a client address, from MaxMind +//! databases (`.mmdb`: GeoLite2 or GeoIP2). +//! +//! Two databases, each optional: one with countries (a Country or City +//! edition) and one with autonomous systems (an ASN or ISP edition). They +//! are kept in Forge's bucket ([`GeoKind::key`]), where Forge's download from +//! MaxMind puts them or an operator does, and every instance loads them from +//! there ([`refresh`]): a database replaced in the bucket is loaded again, +//! one removed is unloaded. + +use std::net::IpAddr; +use std::sync::{Arc, RwLock}; + +use chrono::{DateTime, TimeZone, Utc}; +use maxminddb::{geoip2, Reader}; +use serde::Serialize; + +use crate::objects::Objects; + +/// The two databases. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum GeoKind { + Country, + Asn, +} + +impl GeoKind { + pub const ALL: [GeoKind; 2] = [GeoKind::Country, GeoKind::Asn]; + + /// Where the database is kept in the bucket. + pub fn key(self) -> &'static str { + match self { + GeoKind::Country => "traffic/v1/geoip/country.mmdb", + GeoKind::Asn => "traffic/v1/geoip/asn.mmdb", + } + } + + /// Whether a database of `database_type` (its metadata's) answers this + /// kind: a Country or City edition for countries, an ASN or ISP edition + /// for autonomous systems. + fn accepts(self, database_type: &str) -> bool { + let t = database_type.to_ascii_lowercase(); + match self { + GeoKind::Country => t.contains("country") || t.contains("city"), + GeoKind::Asn => t.contains("asn") || t.contains("isp"), + } + } +} + +/// What a loaded database is. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct DbInfo { + /// Its edition, as its metadata names it (`GeoLite2-Country`). + pub database_type: String, + /// When MaxMind built it. + pub built_at: Option>, + pub size: u64, + /// The bucket object's ETag it was loaded from. + #[serde(skip)] + pub etag: String, +} + +struct Db { + reader: Reader>, + info: DbInfo, +} + +/// Where an address is. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Location { + /// ISO 3166-1 alpha-2; `""` unknown. + pub country: String, + /// `0` unknown. + pub asn: u32, + /// `""` unknown. + pub as_org: String, +} + +/// The loaded databases. +#[derive(Default)] +pub struct Geo { + country: RwLock>>, + asn: RwLock>>, +} + +impl std::fmt::Debug for Geo { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Geo") + .field("country", &self.info(GeoKind::Country)) + .field("asn", &self.info(GeoKind::Asn)) + .finish() + } +} + +/// `bytes` as a database of `kind`, checked: it parses, and its edition +/// answers `kind`. +fn open(kind: GeoKind, bytes: Vec, etag: String) -> anyhow::Result { + let size = bytes.len() as u64; + let reader = Reader::from_source(bytes)?; + let meta = reader.metadata(); + let database_type = meta.database_type.clone(); + if !kind.accepts(&database_type) { + anyhow::bail!( + "{database_type} is not a {} database", + match kind { + GeoKind::Country => "country (Country or City)", + GeoKind::Asn => "autonomous system (ASN or ISP)", + } + ); + } + let built_at = i64::try_from(meta.build_epoch) + .ok() + .and_then(|secs| Utc.timestamp_opt(secs, 0).single()); + Ok(Db { + reader, + info: DbInfo { + database_type, + built_at, + size, + etag, + }, + }) +} + +/// Check that `bytes` is a database of `kind`; what it is. +pub fn check(kind: GeoKind, bytes: Vec) -> anyhow::Result { + Ok(open(kind, bytes, String::new())?.info) +} + +impl Geo { + fn slot(&self, kind: GeoKind) -> &RwLock>> { + match kind { + GeoKind::Country => &self.country, + GeoKind::Asn => &self.asn, + } + } + + fn get(&self, kind: GeoKind) -> Option> { + self.slot(kind) + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() + } + + fn set(&self, kind: GeoKind, db: Option>) { + *self + .slot(kind) + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) = db; + } + + /// Load `bytes`, the bucket object of ETag `etag`, as the database of + /// `kind`. + pub fn load(&self, kind: GeoKind, bytes: Vec, etag: String) -> anyhow::Result { + let db = open(kind, bytes, etag)?; + let info = db.info.clone(); + self.set(kind, Some(Arc::new(db))); + Ok(info) + } + + /// Unload the database of `kind`. + pub fn unload(&self, kind: GeoKind) { + self.set(kind, None); + } + + /// What is loaded as the database of `kind`. + pub fn info(&self, kind: GeoKind) -> Option { + self.get(kind).map(|db| db.info.clone()) + } + + /// Whether any database is loaded. + pub fn any(&self) -> bool { + GeoKind::ALL.iter().any(|k| self.get(*k).is_some()) + } + + /// Where `ip` is, as far as the loaded databases know. + pub fn locate(&self, ip: IpAddr) -> Location { + let ip = ip.to_canonical(); + let mut location = Location::default(); + if let Some(db) = self.get(GeoKind::Country) { + if let Ok(Some(c)) = db + .reader + .lookup(ip) + .and_then(|r| r.decode::()) + { + location.country = c + .country + .iso_code + .or(c.registered_country.iso_code) + .unwrap_or_default() + .to_string(); + } + } + if let Some(db) = self.get(GeoKind::Asn) { + if let Ok(Some(a)) = db.reader.lookup(ip).and_then(|r| r.decode::()) { + location.asn = a.autonomous_system_number.unwrap_or(0); + location.as_org = a + .autonomous_system_organization + .unwrap_or_default() + .to_string(); + } + } + location + } +} + +/// Load the databases kept in the bucket that changed since they were +/// loaded, and unload the ones removed. A database that cannot be read is +/// logged and left as it was. +pub async fn refresh(objects: &dyn Objects, geo: &Geo) -> anyhow::Result<()> { + let listed = objects.list("traffic/v1/geoip/").await?; + for kind in GeoKind::ALL { + let Some(object) = listed.iter().find(|o| o.key == kind.key()) else { + if geo.info(kind).is_some() { + tracing::info!(?kind, "the geolocation database was removed; unloading it"); + geo.unload(kind); + } + continue; + }; + let etag = object.etag.clone().unwrap_or_default(); + if geo + .info(kind) + .is_some_and(|i| i.etag == etag && !etag.is_empty()) + { + continue; + } + let Some(bytes) = objects.get(kind.key()).await? else { + continue; + }; + match geo.load(kind, bytes.to_vec(), etag) { + Ok(info) => tracing::info!( + database = %info.database_type, + built_at = ?info.built_at, + "loaded the geolocation database" + ), + Err(e) => tracing::warn!( + error = %e, + key = kind.key(), + "cannot load the geolocation database" + ), + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::objects::MemoryObjects; + use crate::testing::{asn_db, country_db}; + + #[test] + fn databases_locate_addresses() { + let geo = Geo::default(); + assert_eq!( + geo.locate("192.0.2.1".parse().unwrap()), + Location::default() + ); + geo.load( + GeoKind::Country, + country_db(&[("192.0.2.0/24", "DE")]), + "1".into(), + ) + .unwrap(); + geo.load( + GeoKind::Asn, + asn_db(&[("192.0.2.0/24", 64_500, "Example AS")]), + "1".into(), + ) + .unwrap(); + assert_eq!( + geo.locate("192.0.2.77".parse().unwrap()), + Location { + country: "DE".into(), + asn: 64_500, + as_org: "Example AS".into() + } + ); + assert_eq!( + geo.locate("::ffff:192.0.2.77".parse().unwrap()).country, + "DE", + "a mapped address is its IPv4 address" + ); + assert_eq!( + geo.locate("198.51.100.1".parse().unwrap()), + Location::default() + ); + assert_eq!( + geo.locate("2001:db8::1".parse().unwrap()), + Location::default(), + "an IPv6 address in an IPv4 database is unknown" + ); + assert_eq!( + geo.info(GeoKind::Asn).unwrap().database_type, + "GeoLite2-ASN" + ); + } + + #[test] + fn a_database_of_the_other_kind_is_refused() { + let geo = Geo::default(); + let err = geo + .load( + GeoKind::Country, + asn_db(&[("192.0.2.0/24", 1, "x")]), + "1".into(), + ) + .unwrap_err(); + assert!(err.to_string().contains("not a country"), "{err}"); + assert!(geo + .load(GeoKind::Asn, b"no database".to_vec(), "1".into()) + .is_err()); + assert!(!geo.any()); + } + + #[tokio::test] + async fn refresh_follows_the_bucket() { + let objects = MemoryObjects::default(); + let geo = Geo::default(); + objects + .put( + GeoKind::Country.key(), + country_db(&[("192.0.2.0/24", "DE")]), + ) + .await + .unwrap(); + refresh(&objects, &geo).await.unwrap(); + assert_eq!(geo.locate("192.0.2.1".parse().unwrap()).country, "DE"); + + objects + .put( + GeoKind::Country.key(), + country_db(&[("192.0.2.0/24", "FR")]), + ) + .await + .unwrap(); + refresh(&objects, &geo).await.unwrap(); + assert_eq!( + geo.locate("192.0.2.1".parse().unwrap()).country, + "FR", + "replaced" + ); + + objects.delete(GeoKind::Country.key()).await.unwrap(); + refresh(&objects, &geo).await.unwrap(); + assert!(!geo.any(), "removed"); + } +} diff --git a/crates/traffic/src/layout.rs b/crates/traffic/src/layout.rs index b1383fc8..6c431621 100644 --- a/crates/traffic/src/layout.rs +++ b/crates/traffic/src/layout.rs @@ -1,194 +1,84 @@ //! Where the files are, what writes them and what deletes them. //! -//! Every key is under `traffic/v1/` in Forge's bucket, by area, then date -//! and hour (UTC), then the writing instance ([`instance_id`]): +//! Every key is under `traffic/v1/` in Forge's bucket: //! -//! * `minutes/{date}/{hour}/{instance}.buckets.parquet` and -//! `….clients.parquet`: one instance's counts and top clients of an hour, -//! by minute; rewritten every minute while the hour lasts. Kept -//! [`MINUTE_RETENTION_DAYS`] (the clients no longer than the clients' -//! retention). -//! * `hours/{date}/{instance}.buckets.parquet` and `….clients.parquet`: -//! the same of a day, by hour; rewritten every minute while the day lasts. -//! Kept as long as the settings say ([`TrafficSettings`]). -//! * `requests/{date}/{hour}/{instance}.{unix ms}.parquet`: the request -//! log's rows of one flush. +//! * `days/{date}/{instance}.parquet`: one instance's requests of a day +//! (UTC), a row per repository and client network with their country and +//! autonomous system; rewritten every minute while the day lasts. Kept as +//! long as the settings say ([`TrafficSettings::retention_days`]). +//! * `geoip/{country,asn}.mmdb`: the geolocation databases +//! ([`crate::geo`]). //! -//! Each instance writes only files named after itself, so two instances -//! never write the same object and a restarted process (a new name) never -//! overwrites what its predecessor wrote. +//! Each instance writes only files named after itself ([`instance_id`]), so +//! two instances never write the same object and a restarted process (a new +//! name) never overwrites what its predecessor wrote. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; use chrono::{DateTime, NaiveDate, TimeZone, Utc}; use futures_util::{stream, StreamExt}; +use crate::geo::{Geo, Location}; use crate::objects::Objects; -use crate::record::Drained; -use crate::rows::{encode, BucketKey, BucketRow, ClientRow, Counts}; -use crate::top::{ClientCounts, TopClients}; +use crate::record::{Drained, OTHER}; +use crate::rows::{encode, DayRow}; use crate::TrafficSettings; -/// The prefix of every key. -pub const ROOT: &str = "traffic/v1/"; -/// How long the counts by minute are kept: long enough for the views of the -/// last 48 hours. -pub const MINUTE_RETENTION_DAYS: u32 = 3; -/// The top clients a file keeps per time bucket. -pub const TOP_PER_BUCKET: usize = 100; -/// The networks an hour of a day file keeps track of before its top list -/// is cut to [`TOP_PER_BUCKET`]. -const HOUR_CLIENTS: usize = 4_096; +/// The prefix of the day files. +pub const DAYS: &str = "traffic/v1/days/"; +/// The rows a day file holds before new client networks are counted as +/// [`OTHER`]: a bound on the memory and the file whatever the number of +/// addresses. +const MAX_DAY_ROWS: usize = 500_000; /// Deletes and reads in flight at once. pub(crate) const CONCURRENCY: usize = 16; -const HOUR: i64 = 3_600; const DAY: i64 = 86_400; -/// The three areas of files. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Area { - /// By minute, a file per instance and hour. - Minutes, - /// By hour, a file per instance and day. - Hours, - /// The request log. - Requests, -} - -impl Area { - /// The prefix of the area's keys. - pub fn prefix(self) -> &'static str { - match self { - Area::Minutes => "traffic/v1/minutes/", - Area::Hours => "traffic/v1/hours/", - Area::Requests => "traffic/v1/requests/", - } - } - - /// How long one of its files covers, in seconds. - pub fn span(self) -> i64 { - match self { - Area::Minutes | Area::Requests => HOUR, - Area::Hours => DAY, - } - } - - /// The prefix of its files of the day starting at `day` (Unix seconds). - pub fn day_prefix(self, day: i64) -> String { - format!("{}{}/", self.prefix(), date(day)) - } -} - -/// What a file holds. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum FileKind { - Buckets, - Clients, - Requests, -} - -/// A file, as its key names it. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct FileKey { - pub area: Area, - pub kind: FileKind, - /// The start of the time it covers ([`Area::span`]), Unix seconds. - pub start: i64, - /// The instance that wrote it ([`instance_id`]). - pub instance: String, -} - -fn date(start: i64) -> String { - Utc.timestamp_opt(start, 0) +/// The UTC date of `day` (Unix seconds), as the keys write it. +pub(crate) fn date(day: i64) -> String { + Utc.timestamp_opt(day, 0) .single() .map(|t| t.format("%Y-%m-%d").to_string()) .unwrap_or_default() } -fn hour_of_day(start: i64) -> String { - format!("{:02}", start.rem_euclid(DAY) / HOUR) -} - -fn kind_suffix(kind: FileKind) -> &'static str { - match kind { - FileKind::Buckets => "buckets", - FileKind::Clients => "clients", - FileKind::Requests => "requests", - } +/// The start of the day `date` (`YYYY-MM-DD`) names, in Unix seconds. +pub(crate) fn day_of(date: &str) -> Option { + Some( + NaiveDate::parse_from_str(date, "%Y-%m-%d") + .ok()? + .and_hms_opt(0, 0, 0)? + .and_utc() + .timestamp(), + ) } -/// The key of the counts file (`kind` buckets or clients) of `instance` -/// covering the hour (minutes area) or day (hours area) starting at -/// `start`. -pub fn counts_key(area: Area, start: i64, instance: &str, kind: FileKind) -> String { - let suffix = kind_suffix(kind); - match area { - Area::Minutes => format!( - "{}{}/{}/{instance}.{suffix}.parquet", - area.prefix(), - date(start), - hour_of_day(start) - ), - Area::Hours | Area::Requests => { - format!( - "{}{}/{instance}.{suffix}.parquet", - area.prefix(), - date(start) - ) - } - } +/// The prefix of the files of `day`. +pub fn day_prefix(day: i64) -> String { + format!("{DAYS}{}/", date(day)) } -/// The key of the request log file `instance` flushes at `at`. -pub fn log_key(at: DateTime, instance: &str) -> String { - let start = at.timestamp(); - format!( - "{}{}/{}/{instance}.{}.parquet", - Area::Requests.prefix(), - date(start), - hour_of_day(start), - at.timestamp_millis() - ) +/// The key of `instance`'s file of `day`. +pub fn day_key(day: i64, instance: &str) -> String { + format!("{}{instance}.parquet", day_prefix(day)) } -/// What `key`, a key of `area`, names; `None` for a key Forge did not -/// write. -pub fn parse_key(area: Area, key: &str) -> Option { - let rest = key.strip_prefix(area.prefix())?; - let mut parts = rest.split('/'); - let day = NaiveDate::parse_from_str(parts.next()?, "%Y-%m-%d").ok()?; - let mut start = day.and_hms_opt(0, 0, 0)?.and_utc().timestamp(); - if area != Area::Hours { - let hour: i64 = parts.next()?.parse().ok()?; - if !(0..24).contains(&hour) { - return None; - } - start += hour * HOUR; - } - let name = parts.next()?.strip_suffix(".parquet")?; - if parts.next().is_some() { +/// The day and the instance `key`, a day file's key, names; `None` for a +/// key Forge did not write. +pub fn parse_day_key(key: &str) -> Option<(i64, String)> { + let (date, name) = key.strip_prefix(DAYS)?.split_once('/')?; + let instance = name.strip_suffix(".parquet")?; + if instance.is_empty() || instance.contains('/') { return None; } - let (instance, suffix) = name.split_once('.')?; - let kind = match (area, suffix) { - (Area::Requests, millis) if millis.parse::().is_ok() => FileKind::Requests, - (Area::Minutes | Area::Hours, "buckets") => FileKind::Buckets, - (Area::Minutes | Area::Hours, "clients") => FileKind::Clients, - _ => return None, - }; - Some(FileKey { - area, - kind, - start, - instance: instance.to_string(), - }) + Some((day_of(date)?, instance.to_string())) } /// This process's name in the files: its host name (`-` for anything a /// host name does not hold), when it started and a random suffix, so that -/// two processes never share one. No `.` or `_` but the separators. +/// two processes never share one. No `.`, `_` or `/` but the separators. pub fn instance_id(host: Option<&str>, started_at: DateTime) -> String { let host: String = host .unwrap_or("forge") @@ -226,51 +116,35 @@ pub fn instance_parts(id: &str) -> (String, Option>) { (host, started) } -/// One instance's counts of an hour (by minute) or a day (by hour), as -/// written to its files. -#[derive(Debug, Default)] -struct Period { - buckets: BTreeMap<(i64, BucketKey), Counts>, - clients: BTreeMap, - dirty: bool, +/// What a day's row is counted under. With client addresses, the network +/// (its country and autonomous system are looked up when the file is +/// written); without, the country and autonomous system, looked up before +/// the address is dropped. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +struct RowKey { + repository: String, + network: String, + country: String, + asn: u32, } -impl Period { - fn bucket_rows(&self) -> Vec { - self.buckets - .iter() - .map(|((start, key), counts)| BucketRow::new(*start, key, counts)) - .collect() - } - - fn client_rows(&self) -> Vec { - let mut rows = Vec::new(); - for (start, top) in &self.clients { - for (network, c) in top.top(TOP_PER_BUCKET) { - rows.push(client_row(*start, network, c)); - } - } - rows - } +#[derive(Debug, Clone, Default)] +struct RowCounts { + requests: u64, + bytes: u64, + /// The last address seen; `""` without addresses. + address: String, + /// The autonomous system's organisation, for a row without a network. + as_org: String, } -fn client_row(start: i64, network: &str, c: &ClientCounts) -> ClientRow { - let signed = |n: u64| i64::try_from(n).unwrap_or(i64::MAX); - ClientRow { - start, - network: network.to_string(), - address: c.address.clone(), - requests: signed(c.requests), - bytes: signed(c.bytes), - client_errors: signed(c.client_errors), - denied: signed(c.denied), - not_found: signed(c.not_found), - throttled: signed(c.throttled), - server_errors: signed(c.server_errors), - user_agent: c.user_agent.clone(), - account: c.account.clone(), - user_id: c.user_id.unwrap_or(0), - } +/// One instance's counts of a day. +#[derive(Debug, Default)] +struct Day { + rows: BTreeMap, + /// Where each network is, looked up once a day (again while unknown). + located: HashMap, + dirty: bool, } /// What a flush did. @@ -278,19 +152,19 @@ fn client_row(start: i64, network: &str, c: &ClientCounts) -> ClientRow { pub struct FlushReport { /// Files written. pub files: usize, - /// Request log rows written. - pub log_rows: usize, - /// Writes that failed (retried with the next flush, but for the log). + /// Files that could not be written (written with the next flush). pub failures: usize, } -/// Writes one instance's files: the counts of the hours and days not over -/// yet, and the request log. +/// Writes one instance's day files. pub struct Flusher { objects: Arc, instance: Arc, - hours: BTreeMap, - days: BTreeMap, + days: BTreeMap, +} + +fn signed(n: u64) -> i64 { + i64::try_from(n).unwrap_or(i64::MAX) } impl Flusher { @@ -299,216 +173,161 @@ impl Flusher { Flusher { objects, instance, - hours: BTreeMap::new(), days: BTreeMap::new(), } } - /// Add `drained` to the hours and days it falls in and write the files - /// that changed, and the request log rows. A file that cannot be written - /// is written with the next flush; log rows that cannot be are dropped. + /// Add `drained` to its days, locating the client networks with `geo`, + /// and write the files of the days that changed. A file that cannot be + /// written is written with the next flush. pub async fn flush( &mut self, drained: Drained, settings: &TrafficSettings, + geo: &Geo, now: DateTime, ) -> FlushReport { - let mut report = FlushReport::default(); - for minute in drained.minutes { - let hour = minute.start - minute.start.rem_euclid(HOUR); - let day = minute.start - minute.start.rem_euclid(DAY); - let in_hour = self.hours.entry(hour).or_default(); - let in_day = self.days.entry(day).or_default(); - for (key, counts) in &minute.buckets { - in_hour - .buckets - .entry((minute.start, key.clone())) - .or_default() - .add(counts); - in_day - .buckets - .entry((hour, key.clone())) - .or_default() - .add(counts); - } - if settings.client_addresses && !minute.clients.is_empty() { - let top = in_hour - .clients - .entry(minute.start) - .or_insert_with(|| TopClients::new(TOP_PER_BUCKET)); - top.merge(&minute.clients); - top.truncate(TOP_PER_BUCKET); - in_day - .clients - .entry(hour) - .or_insert_with(|| TopClients::new(HOUR_CLIENTS)) - .merge(&minute.clients); - } - in_hour.dirty = true; - in_day.dirty = true; + if drained.overflowed > 0 { + tracing::warn!( + requests = drained.overflowed, + "more client networks than the traffic statistics hold between two flushes; \ + counted as `other`" + ); } - if !settings.client_addresses { - for period in self.hours.values_mut().chain(self.days.values_mut()) { - period.clients.clear(); + for (key, counts) in drained.counts { + let day = self.days.entry(key.day).or_default(); + day.dirty = true; + let address = counts.address.map(|a| a.to_string()).unwrap_or_default(); + let mut row = if !settings.client_addresses || key.network.is_empty() { + // No network to keep: locate the address now, then drop it. + let at = counts.address.map(|a| geo.locate(a)).unwrap_or_default(); + ( + RowKey { + repository: key.repository, + network: String::new(), + country: at.country, + asn: at.asn, + }, + String::new(), + at.as_org, + ) + } else { + ( + RowKey { + repository: key.repository, + network: key.network, + country: String::new(), + asn: 0, + }, + address, + String::new(), + ) + }; + if day.rows.len() >= MAX_DAY_ROWS && !day.rows.contains_key(&row.0) { + row.0.network = OTHER.to_string(); + row.1.clear(); } - } - - let instance = self.instance.clone(); - for (area, periods) in [ - (Area::Minutes, &mut self.hours), - (Area::Hours, &mut self.days), - ] { - for (start, period) in periods.iter_mut().filter(|(_, p)| p.dirty) { - let mut ok = true; - let mut files = vec![(FileKind::Buckets, encode(&period.bucket_rows()))]; - if settings.client_addresses { - files.push((FileKind::Clients, encode(&period.client_rows()))); - } - for (kind, body) in files { - let key = counts_key(area, *start, &instance, kind); - match body { - Ok(body) => match self.objects.put(&key, body).await { - Ok(()) => report.files += 1, - Err(e) => { - tracing::warn!(error = %e, key, "cannot write the traffic statistics"); - ok = false; - } - }, - Err(e) => { - tracing::error!(error = %e, key, "cannot encode the traffic statistics"); - ok = false; - } - } - } - if ok { - period.dirty = false; - } else { - report.failures += 1; - } + let (row_key, address, as_org) = row; + let entry = day.rows.entry(row_key).or_default(); + entry.requests += counts.requests; + entry.bytes += counts.bytes; + if !address.is_empty() { + entry.address = address; + } + if !as_org.is_empty() { + entry.as_org = as_org; } } - if drained.log_dropped > 0 { - tracing::warn!( - dropped = drained.log_dropped, - "the request log dropped rows: more requests than it holds between two flushes" - ); - } - if !drained.log.is_empty() { - let key = log_key(now, &instance); - let written = match encode(&drained.log) { + let mut report = FlushReport::default(); + for (start, day) in self.days.iter_mut().filter(|(_, d)| d.dirty) { + let rows = day_rows(*start, day, geo); + let key = day_key(*start, &self.instance); + let written = match encode(&rows) { Ok(body) => self.objects.put(&key, body).await, Err(e) => Err(e), }; match written { Ok(()) => { + day.dirty = false; report.files += 1; - report.log_rows = drained.log.len(); } Err(e) => { report.failures += 1; - tracing::warn!( - error = %e, - key, - rows = drained.log.len(), - "cannot write the request log; its rows are lost" - ); + tracing::warn!(error = %e, key, "cannot write the traffic statistics"); } } } - - self.forget(now); - report - } - - /// Drop the periods that are over and written, and the ones that could - /// not be written for too long; cut the top lists of the hours that are - /// over. - fn forget(&mut self, now: DateTime) { - let now = now.timestamp(); - let this_hour = now - now.rem_euclid(HOUR); - let today = now - now.rem_euclid(DAY); - self.hours.retain(|start, period| { - *start >= this_hour || (period.dirty && *start >= this_hour - 3 * HOUR) - }); + // Keep today and yesterday (a request that ended after midnight + // counts in its own day), and any day not written yet but for a + // week at most. + let today = now.timestamp().div_euclid(DAY) * DAY; self.days - .retain(|start, period| *start >= today || (period.dirty && *start >= today - DAY)); - for day in self.days.values_mut() { - for (hour, top) in day.clients.iter_mut() { - if *hour < this_hour { - top.truncate(TOP_PER_BUCKET); - } - } - } + .retain(|start, day| *start >= today - DAY || (day.dirty && *start >= today - 7 * DAY)); + report } } -/// How long the files of `area` holding `kind` are kept, in seconds; -/// `None` to keep them. -pub fn retention(settings: &TrafficSettings, area: Area, kind: FileKind) -> Option { - let days = |d: u32| i64::from(d) * DAY; - match (area, kind) { - (Area::Minutes, FileKind::Clients) => Some(days( - MINUTE_RETENTION_DAYS.min(settings.client_retention_days), - )), - (Area::Minutes, _) => Some(days(MINUTE_RETENTION_DAYS)), - (Area::Hours, FileKind::Clients) => Some(days(settings.client_retention_days)), - (Area::Hours, _) => match settings.statistics_retention_days { - 0 => None, - d => Some(days(d)), - }, - (Area::Requests, _) => Some(days(settings.request_log_retention_days)), +/// The rows of `day`, its networks located with `geo`. +fn day_rows(start: i64, day: &mut Day, geo: &Geo) -> Vec { + let mut rows = Vec::with_capacity(day.rows.len()); + for (key, counts) in &day.rows { + let at = if key.network.is_empty() || key.network == OTHER || counts.address.is_empty() { + Location { + country: key.country.clone(), + asn: key.asn, + as_org: counts.as_org.clone(), + } + } else { + let known = day.located.get(&key.network); + if known.is_some_and(|l| !l.country.is_empty() || l.asn != 0) { + known.cloned().unwrap_or_default() + } else { + let at = counts + .address + .parse() + .map(|ip| geo.locate(ip)) + .unwrap_or_default(); + day.located.insert(key.network.clone(), at.clone()); + at + } + }; + rows.push(DayRow { + day: start, + repository: key.repository.clone(), + network: key.network.clone(), + address: counts.address.clone(), + country: at.country, + asn: i64::from(at.asn), + as_org: at.as_org, + requests: signed(counts.requests), + bytes: signed(counts.bytes), + }); } + rows } -/// Delete the files past their retention ([`retention`]), whoever wrote -/// them. Idempotent and safe on every instance at once. Returns how many -/// files it deleted. +/// Delete the day files past `settings`' retention, whoever wrote them. +/// Idempotent and safe on every instance at once. Returns how many files it +/// deleted. pub async fn prune( objects: &dyn Objects, settings: &TrafficSettings, now: DateTime, ) -> anyhow::Result { - let now = now.timestamp(); + let today = now.timestamp().div_euclid(DAY) * DAY; + let oldest_kept = today - i64::from(settings.retention_days.max(1) - 1) * DAY; let mut expired = Vec::new(); - for area in [Area::Minutes, Area::Hours, Area::Requests] { - let kinds: &[FileKind] = match area { - Area::Requests => &[FileKind::Requests], - _ => &[FileKind::Buckets, FileKind::Clients], - }; - // A day can hold expired files once the shortest retention of the - // area has passed since it ended. - let Some(shortest) = kinds - .iter() - .filter_map(|kind| retention(settings, area, *kind)) - .min() + for dir in objects.list_dirs(DAYS).await? { + let Some(day) = dir + .strip_prefix(DAYS) + .and_then(|d| d.strip_suffix('/')) + .and_then(day_of) else { continue; }; - for dir in objects.list_dirs(area.prefix()).await? { - let Some(day) = dir - .strip_prefix(area.prefix()) - .and_then(|d| d.strip_suffix('/')) - .and_then(|d| NaiveDate::parse_from_str(d, "%Y-%m-%d").ok()) - .and_then(|d| d.and_hms_opt(0, 0, 0)) - .map(|d| d.and_utc().timestamp()) - else { - continue; - }; - if day + area.span() + shortest > now { - // Even its earliest file is within the shortest retention. - continue; - } - for listed in objects.list(&dir).await? { - let Some(file) = parse_key(area, &listed.key) else { - continue; - }; - if retention(settings, area, file.kind) - .is_some_and(|keep| file.start + area.span() + keep <= now) - { - expired.push(listed.key); - } - } + if day < oldest_kept { + expired.extend(objects.list(&dir).await?.into_iter().map(|o| o.key)); } } let deleted = expired.len(); @@ -527,11 +346,12 @@ pub async fn prune( #[cfg(test)] mod tests { use super::*; + use crate::geo::GeoKind; use crate::objects::MemoryObjects; use crate::record::{Event, Recorder}; use crate::rows::decode; use crate::slot::Noted; - use std::time::Duration; + use crate::testing::{asn_db, country_db}; fn t(rfc3339: &str) -> DateTime { DateTime::parse_from_rfc3339(rfc3339).unwrap().to_utc() @@ -539,36 +359,17 @@ mod tests { #[test] fn keys_name_what_they_hold() { - let start = t("2026-10-30T14:00:00Z").timestamp(); - let key = counts_key(Area::Minutes, start, "web-1_x_0001", FileKind::Clients); - assert_eq!( - key, - "traffic/v1/minutes/2026-10-30/14/web-1_x_0001.clients.parquet" - ); - let parsed = parse_key(Area::Minutes, &key).unwrap(); - assert_eq!( - (parsed.start, parsed.kind, parsed.instance.as_str()), - (start, FileKind::Clients, "web-1_x_0001") - ); - let day = t("2026-10-30T00:00:00Z").timestamp(); - let key = counts_key(Area::Hours, day, "web-1", FileKind::Buckets); - assert_eq!(key, "traffic/v1/hours/2026-10-30/web-1.buckets.parquet"); - assert_eq!(parse_key(Area::Hours, &key).unwrap().start, day); - - let key = log_key(t("2026-10-30T14:05:01.250Z"), "web-1"); - assert_eq!( - key, - "traffic/v1/requests/2026-10-30/14/web-1.1793369101250.parquet" - ); - assert_eq!(parse_key(Area::Requests, &key).unwrap().start, start); + let key = day_key(day, "web-1_x_0001"); + assert_eq!(key, "traffic/v1/days/2026-10-30/web-1_x_0001.parquet"); + assert_eq!(parse_key_ok(&key), (day, "web-1_x_0001".to_string())); + assert!(parse_day_key("traffic/v1/days/2026-13-30/a.parquet").is_none()); + assert!(parse_day_key("traffic/v1/days/2026-10-30/a.csv").is_none()); + assert!(parse_day_key("traffic/v1/geoip/country.mmdb").is_none()); + } - assert!(parse_key( - Area::Minutes, - "traffic/v1/minutes/2026-10-30/24/a.buckets.parquet" - ) - .is_none()); - assert!(parse_key(Area::Hours, "traffic/v1/hours/2026-10-30/a.other.parquet").is_none()); + fn parse_key_ok(key: &str) -> (i64, String) { + parse_day_key(key).unwrap() } #[test] @@ -583,110 +384,154 @@ mod tests { assert!(instance_id(None, started).starts_with("forge_")); } - fn event(at: DateTime, network: &str) -> Event { + fn event(at: &str, client: &str, repository: Option<&str>) -> Event { Event { - at, - surface: "oci", - method: "GET", - status: 200, - duration: Duration::from_millis(3), + at: t(at), bytes: 10, - path: "/v2/".into(), - user_agent: None, noted: Noted { - client: network.parse().ok(), - network: Some(network.into()), - account: None, - repository: Some("docker".into()), + client: client.parse().ok(), + network: Some(client.into()), + repository: repository.map(str::to_string), }, } } + fn geo() -> Geo { + let geo = Geo::default(); + geo.load( + GeoKind::Country, + country_db(&[("192.0.2.0/24", "DE")]), + "1".into(), + ) + .unwrap(); + geo.load( + GeoKind::Asn, + asn_db(&[("192.0.2.0/24", 64_500, "Example AS")]), + "1".into(), + ) + .unwrap(); + geo + } + + async fn rows(objects: &MemoryObjects, key: &str) -> Vec { + decode(objects.get(key).await.unwrap().unwrap()).unwrap() + } + #[tokio::test] - async fn a_flush_writes_the_hour_the_day_and_the_log() { + async fn a_flush_writes_the_day_with_countries_and_networks() { let objects = Arc::new(MemoryObjects::default()); let mut flusher = Flusher::new(objects.clone(), "a".into()); let settings = TrafficSettings::default(); + let geo = geo(); let recorder = Recorder::new(); - recorder.record(event(t("2026-10-30T14:00:10Z"), "192.0.2.1")); - recorder.record(event(t("2026-10-30T14:00:20Z"), "192.0.2.1")); - recorder.record(event(t("2026-10-30T14:01:20Z"), "192.0.2.2")); + recorder.record(event("2026-10-30T14:00:10Z", "192.0.2.1", Some("npm"))); + recorder.record(event("2026-10-30T14:00:20Z", "192.0.2.1", Some("npm"))); + recorder.record(event("2026-10-30T14:01:20Z", "198.51.100.2", None)); let now = t("2026-10-30T14:02:00Z"); - let report = flusher.flush(recorder.drain(now), &settings, now).await; + let report = flusher.flush(recorder.drain(), &settings, &geo, now).await; assert_eq!( report, FlushReport { - files: 5, - log_rows: 3, + files: 1, failures: 0 } ); - let keys = objects.keys(); - assert!(keys.contains(&"traffic/v1/hours/2026-10-30/a.buckets.parquet".to_string())); - - let rows: Vec = decode( - objects - .get("traffic/v1/minutes/2026-10-30/14/a.buckets.parquet") - .await - .unwrap() - .unwrap(), - ) - .unwrap(); - assert_eq!(rows.len(), 2, "one row per minute and key"); - assert_eq!(rows[0].requests, 2); - // The next hour: the previous hour's files are not rewritten. - recorder.record(event(t("2026-10-30T15:00:05Z"), "192.0.2.1")); - let now = t("2026-10-30T15:01:00Z"); - let report = flusher.flush(recorder.drain(now), &settings, now).await; + recorder.record(event("2026-10-30T14:03:00Z", "192.0.2.1", Some("npm"))); + flusher.flush(recorder.drain(), &settings, &geo, now).await; + let day = rows(&objects, "traffic/v1/days/2026-10-30/a.parquet").await; + assert_eq!(day.len(), 2); + let npm = day.iter().find(|r| r.repository == "npm").unwrap(); + assert_eq!((npm.requests, npm.bytes), (3, 30), "added up over flushes"); assert_eq!( - report.files, 5, - "the new hour's two, the day's two, the log" + ( + npm.network.as_str(), + npm.country.as_str(), + npm.asn, + npm.as_org.as_str() + ), + ("192.0.2.1", "DE", 64_500, "Example AS") ); - assert_eq!(flusher.hours.len(), 1, "the hour that is over is forgotten"); - let day: Vec = decode( - objects - .get("traffic/v1/hours/2026-10-30/a.buckets.parquet") - .await - .unwrap() - .unwrap(), - ) - .unwrap(); - assert_eq!(day.len(), 2, "one row per hour"); - assert_eq!(day.iter().map(|r| r.requests).sum::(), 4); + let other = day.iter().find(|r| r.repository.is_empty()).unwrap(); + assert_eq!( + (other.country.as_str(), other.asn), + ("", 0), + "not in the databases" + ); + + // Nothing new: nothing written. + let report = flusher.flush(recorder.drain(), &settings, &geo, now).await; + assert_eq!(report.files, 0); + // The next day: the day before is still kept, the one before that is not. + recorder.record(event("2026-10-31T00:00:05Z", "192.0.2.1", Some("npm"))); + flusher + .flush(recorder.drain(), &settings, &geo, t("2026-11-01T00:00:10Z")) + .await; + assert!(objects + .keys() + .contains(&"traffic/v1/days/2026-10-31/a.parquet".to_string())); + assert_eq!(flusher.days.len(), 1); + } + + #[tokio::test] + async fn without_addresses_rows_keep_the_country_alone() { + let objects = Arc::new(MemoryObjects::default()); + let mut flusher = Flusher::new(objects.clone(), "a".into()); + let settings = TrafficSettings { + client_addresses: false, + ..TrafficSettings::default() + }; + let recorder = Recorder::new(); + recorder.record(event("2026-10-30T14:00:10Z", "192.0.2.1", Some("npm"))); + recorder.record(event("2026-10-30T14:00:20Z", "192.0.2.2", Some("npm"))); + let now = t("2026-10-30T14:02:00Z"); + flusher + .flush(recorder.drain(), &settings, &geo(), now) + .await; + let day = rows(&objects, "traffic/v1/days/2026-10-30/a.parquet").await; + assert_eq!(day.len(), 1, "both addresses in one row"); + assert_eq!( + ( + day[0].network.as_str(), + day[0].address.as_str(), + day[0].country.as_str() + ), + ("", "", "DE") + ); + assert_eq!((day[0].requests, day[0].asn), (2, 64_500)); } #[tokio::test] - async fn pruning_deletes_by_area_and_kind() { + async fn pruning_deletes_the_days_past_the_retention() { let objects = MemoryObjects::default(); let day = |d: &str| t(&format!("{d}T00:00:00Z")).timestamp(); for key in [ - counts_key(Area::Minutes, day("2026-10-25"), "a", FileKind::Buckets), - counts_key(Area::Minutes, day("2026-10-29"), "a", FileKind::Buckets), - counts_key(Area::Hours, day("2025-01-01"), "a", FileKind::Buckets), - counts_key(Area::Hours, day("2026-09-01"), "a", FileKind::Buckets), - counts_key(Area::Hours, day("2026-09-01"), "a", FileKind::Clients), - log_key(t("2026-10-20T10:00:00Z"), "a"), - log_key(t("2026-10-29T10:00:00Z"), "a"), - "traffic/v1/hours/not-a-date/x.buckets.parquet".to_string(), + day_key(day("2026-07-31"), "a"), + day_key(day("2026-08-01"), "a"), + day_key(day("2026-08-01"), "b"), + day_key(day("2026-10-30"), "a"), + "traffic/v1/days/not-a-date/x.parquet".to_string(), + "traffic/v1/geoip/country.mmdb".to_string(), ] { objects.put(&key, vec![1]).await.unwrap(); } + // 90 days up to 2026-10-29: from 2026-08-01 on. let deleted = prune( &objects, &TrafficSettings::default(), - t("2026-10-30T12:00:00Z"), + t("2026-10-29T12:00:00Z"), ) .await .unwrap(); - assert_eq!(deleted, 4); + assert_eq!(deleted, 1); assert_eq!( objects.keys(), vec![ - "traffic/v1/hours/2026-09-01/a.buckets.parquet", - "traffic/v1/hours/not-a-date/x.buckets.parquet", - "traffic/v1/minutes/2026-10-29/00/a.buckets.parquet", - "traffic/v1/requests/2026-10-29/10/a.1793268000000.parquet", + "traffic/v1/days/2026-08-01/a.parquet", + "traffic/v1/days/2026-08-01/b.parquet", + "traffic/v1/days/2026-10-30/a.parquet", + "traffic/v1/days/not-a-date/x.parquet", + "traffic/v1/geoip/country.mmdb", ] ); } diff --git a/crates/traffic/src/lib.rs b/crates/traffic/src/lib.rs index 6e7d547d..cc09af48 100644 --- a/crates/traffic/src/lib.rs +++ b/crates/traffic/src/lib.rs @@ -1,34 +1,30 @@ -//! Forge's traffic statistics: what every instance answered, counted in time -//! buckets and kept as Parquet files in the object store, so that any -//! instance can show the traffic of all of them (Admin → Traffic). +//! Forge's traffic statistics: how many requests each client network made to +//! each repository on each day, from which country and autonomous system, +//! kept as Parquet files in the object store, so that any instance can show +//! the traffic of all of them (Admin → Traffic). //! //! * **Recording** ([`Recorder`]) is a counter update in memory, behind one -//! short lock: no I/O on the request path. A request is counted in its -//! minute by surface, repository, method and status (requests, response -//! bytes, a latency histogram), its client address in that minute's top -//! list ([`TopClients`], bounded whatever the number of addresses), and, -//! with the request log on, as a row of the log. +//! short lock: no I/O on the request path. A request is counted under its +//! day (UTC), its repository and its client network, with the bytes sent. //! * **What a request is** comes from the layers that know it: the server's //! outermost middleware opens a [`Slot`] around the request, and the -//! principal layer ([`note_client`], [`note_account`]) and the handlers -//! that resolve a repository ([`note_repository`]) fill it in. -//! * **Flushing** ([`Flusher`]) runs once a minute on every instance. Each -//! instance writes files of its own ([`layout`]), named after the process -//! ([`instance_id`]), so instances never write the same object: -//! * per hour, its buckets and top clients by minute, rewritten every -//! minute while the hour lasts; -//! * per day, the same by hour; -//! * every minute, the rows of the request log since the last flush. -//! * **Reading** ([`query`]) lists the files of a range in the object store, -//! reads them (closed files are cached) and sums them: the view over all -//! instances needs no call between them. -//! * **Pruning** ([`layout::prune`]) deletes the files past their retention +//! principal layer ([`note_client`]) and the handlers that resolve a +//! repository ([`note_repository`]) fill it in. +//! * **Flushing** ([`Flusher`]) runs once a minute on every instance: it +//! locates the client networks in the geolocation databases ([`geo`]) and +//! rewrites the instance's file of the day ([`layout`]), named after the +//! process ([`instance_id`]), so instances never write the same object. +//! * **Reading** ([`query`]) lists the files of a range of days in the +//! object store, reads them (cached by ETag) and sums them: the view over +//! all instances needs no call between them. +//! * **Pruning** ([`layout::prune`]) deletes the days past their retention //! ([`TrafficSettings`]); it is idempotent and runs on every instance. //! -//! Client addresses are personal data: they are kept apart from the counts -//! (the `clients` files and the request log), with a retention of their own, -//! and can be switched off. +//! Client addresses are personal data: they can be switched off, and then +//! requests are still counted by repository, country and autonomous system, +//! the address only looked up in memory before it is dropped. +pub mod geo; pub mod layout; pub mod objects; pub mod query; @@ -36,25 +32,27 @@ mod record; mod rows; mod settings; mod slot; -mod top; +#[cfg(any(test, feature = "testing"))] +pub mod testing; use std::sync::Arc; +pub use geo::{DbInfo, Geo, GeoKind, Location}; pub use layout::{instance_id, FlushReport, Flusher}; pub use objects::{Listed, MemoryObjects, Objects}; -pub use record::{Drained, Event, Minute, Recorder}; -pub use rows::{BucketKey, Counts, LATENCY_BOUNDS_MS}; +pub use record::{Drained, Event, PendingCounts, PendingKey, Recorder, OTHER}; pub use settings::{TrafficSettings, MAX_RETENTION_DAYS}; -pub use slot::{note_account, note_client, note_repository, Account, Noted, Slot}; -pub use top::{ClientCounts, TopClients}; +pub use slot::{note_client, note_repository, Noted, Slot}; /// What the server holds of the traffic statistics: this instance's -/// [`Recorder`] and [`Flusher`], the object store the files go to, and the -/// cache of the files read back. +/// [`Recorder`] and [`Flusher`], the geolocation databases, the object store +/// the files go to, and the cache of the files read back. #[derive(Clone)] pub struct Traffic { /// Counts this instance's requests. pub recorder: Arc, + /// The geolocation databases this instance loaded. + pub geo: Arc, /// Where the files are written and read. pub objects: Arc, /// This process's name in the files ([`instance_id`]). @@ -73,6 +71,7 @@ impl Traffic { let instance: Arc = instance.into(); Traffic { recorder: Arc::new(Recorder::new()), + geo: Arc::new(Geo::default()), flusher: Arc::new(tokio::sync::Mutex::new(Flusher::new( objects.clone(), instance.clone(), @@ -83,22 +82,15 @@ impl Traffic { } } - /// Write the minutes that ended before `now`, and the request log. + /// Write what was counted since the last flush. pub async fn flush( &self, settings: &TrafficSettings, now: chrono::DateTime, ) -> FlushReport { - let mut flusher = self.flusher.lock().await; - flusher.flush(self.recorder.drain(now), settings, now).await - } - - /// Write everything counted, the minute still being counted included - /// (on shutdown). - pub async fn flush_all(&self, settings: &TrafficSettings) -> FlushReport { let mut flusher = self.flusher.lock().await; flusher - .flush(self.recorder.drain_all(), settings, chrono::Utc::now()) + .flush(self.recorder.drain(), settings, &self.geo, now) .await } } diff --git a/crates/traffic/src/query.rs b/crates/traffic/src/query.rs index 0e2b7044..ad3ea771 100644 --- a/crates/traffic/src/query.rs +++ b/crates/traffic/src/query.rs @@ -1,68 +1,43 @@ -//! Reading the files back: the traffic of a time range, over all instances -//! or one, and one client's. +//! Reading the day files back: the requests of a range of days, over all +//! instances or one, and where they came from. //! -//! A range of up to [`MINUTE_RANGE_HOURS`] hours within the minute files' -//! retention is read from them (by minute); a longer one from the files by -//! hour. The files are found by listing the days of the range, read in -//! parallel and summed. Files of counts are cached decoded, keyed by their -//! ETag, so a file rewritten since is read again. +//! The files are found by listing the days of the range, read in parallel +//! and summed. They are cached decoded, keyed by their ETag, so a file +//! rewritten since (today's) is read again. -use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::sync::Arc; -use chrono::{DateTime, TimeZone, Utc}; +use chrono::{NaiveDate, TimeZone, Utc}; use futures_util::{stream, StreamExt, TryStreamExt}; use serde::Serialize; -use crate::layout::{instance_parts, parse_key, Area, FileKey, FileKind, CONCURRENCY}; +use crate::layout::{day_prefix, instance_parts, parse_day_key, CONCURRENCY}; use crate::objects::{Listed, Objects}; -use crate::rows::{decode, BucketRow, ClientRow, Counts, LogRow, LATENCY_BOUNDS_MS}; - -/// The longest range read by minute, in hours. -pub const MINUTE_RANGE_HOURS: i64 = 48; -/// The longest stretch of the request log a client's view reads, in hours. -pub const LOG_RANGE_HOURS: i64 = 24; -/// The most request log rows a client's view returns. -pub const LOG_LIMIT: usize = 500; -/// The lists' lengths: top repositories, top clients. +use crate::record::OTHER; +use crate::rows::{decode, DayRow}; + +/// The lists' lengths: repositories, autonomous systems, clients. pub const TOP: usize = 20; -/// The most points of a series. -const MAX_POINTS: i64 = 180; -/// The steps a series may take, in seconds. -const STEPS: [i64; 13] = [ - 60, 300, 600, 900, 1_800, 3_600, 7_200, 10_800, 21_600, 43_200, 86_400, 172_800, 604_800, -]; /// The rows the cache of decoded files holds. -const CACHE_ROWS: u64 = 100_000; +const CACHE_ROWS: u64 = 500_000; +/// The longest range, in days. +pub const MAX_DAYS: i64 = 400; -const HOUR: i64 = 3_600; const DAY: i64 = 86_400; -#[derive(Debug)] -enum Decoded { - Buckets(Vec), - Clients(Vec), -} - -impl Decoded { - fn rows(&self) -> usize { - match self { - Decoded::Buckets(rows) => rows.len(), - Decoded::Clients(rows) => rows.len(), - } - } -} - -/// Files of counts read back, decoded. +/// Day files read back, decoded. #[derive(Clone)] -pub struct FileCache(moka::future::Cache>); +pub struct FileCache(moka::future::Cache>>); impl FileCache { pub fn new() -> FileCache { FileCache( moka::future::Cache::builder() .max_capacity(CACHE_ROWS) - .weigher(|_, d: &Arc| u32::try_from(d.rows().max(1)).unwrap_or(u32::MAX)) + .weigher(|_, rows: &Arc>| { + u32::try_from(rows.len().max(1)).unwrap_or(u32::MAX) + }) .build(), ) } @@ -80,7 +55,7 @@ pub struct InstanceView { /// Its name in the files ([`crate::instance_id`]). pub id: String, pub host: String, - pub started_at: Option>, + pub started_at: Option>, } impl InstanceView { @@ -94,765 +69,456 @@ impl InstanceView { } } -/// The time range a view covers, and its points. -#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] -pub struct Span { - pub from: DateTime, - pub to: DateTime, - /// The length of a point of the series. - pub step_seconds: i64, - /// What the files read count by: `minute` or `hour`. - pub resolution: &'static str, -} - -impl Span { - /// The span of `[from, to)` read at `now`: by minute when it is short - /// and recent enough, else by hour; `from` aligned down to the step. - pub fn new(from: DateTime, to: DateTime, now: DateTime) -> Span { - // The end rounded up to the second: a request of the second the - // range ends in (the request log's times are in milliseconds) is in. - let to_s = to.timestamp() + i64::from(to.timestamp_subsec_nanos() > 0); - let (from_s, to_s) = (from.timestamp(), to_s.max(from.timestamp() + 60)); - let minute_files_from = - now.timestamp() - i64::from(crate::layout::MINUTE_RETENTION_DAYS) * DAY + HOUR; - let by_minute = to_s - from_s <= MINUTE_RANGE_HOURS * HOUR && from_s >= minute_files_from; - let resolution = if by_minute { 60 } else { HOUR }; - let step = STEPS - .into_iter() - .filter(|step| *step >= resolution) - .find(|step| (to_s - from_s).div_euclid(*step) <= MAX_POINTS) - .unwrap_or(STEPS[STEPS.len() - 1]); - let from_s = from_s - from_s.rem_euclid(step); - Span { - from: at(from_s), - to: at(to_s), - step_seconds: step, - resolution: if by_minute { "minute" } else { "hour" }, - } - } - - fn area(&self) -> Area { - if self.resolution == "minute" { - Area::Minutes - } else { - Area::Hours - } - } - - fn contains(&self, start: i64) -> bool { - (self.from.timestamp()..self.to.timestamp()).contains(&start) - } - - fn point(&self, start: i64) -> usize { - usize::try_from((start - self.from.timestamp()).div_euclid(self.step_seconds)).unwrap_or(0) - } - - fn points(&self) -> Vec { - let mut points = Vec::new(); - let mut start = self.from.timestamp(); - while start < self.to.timestamp() { - points.push(Point { - start: at(start), - ..Point::default() - }); - start += self.step_seconds; - } - points +/// What to read: the days `from` to `to`, both included, and what to keep. +#[derive(Debug, Clone, Default)] +pub struct Filter { + /// Only the files of this instance ([`crate::instance_id`]). + pub instance: Option, + /// Only the requests to this repository; `""` for the requests that + /// addressed none. + pub repository: Option, + /// Only the requests of this client network. + pub network: Option, + /// Only the requests from this country (ISO code; `""` unknown). + pub country: Option, + /// Only the requests from this autonomous system (`0` unknown). + pub asn: Option, +} + +impl Filter { + fn keeps(&self, row: &DayRow) -> bool { + self.repository + .as_ref() + .is_none_or(|r| *r == row.repository) + && self.network.as_ref().is_none_or(|n| *n == row.network) + && self.country.as_ref().is_none_or(|c| *c == row.country) + && self.asn.is_none_or(|a| i64::from(a) == row.asn) } } -fn at(secs: i64) -> DateTime { - Utc.timestamp_opt(secs, 0).single().unwrap_or_default() +/// A day of the range. +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct DayPoint { + pub day: NaiveDate, + pub requests: u64, + pub bytes: u64, + /// The client networks that made requests that day. + pub clients: u64, } -/// A point of a series. +/// The counts of the range. #[derive(Debug, Clone, Default, Serialize, PartialEq, Eq)] -pub struct Point { - pub start: DateTime, +pub struct Totals { pub requests: u64, pub bytes: u64, - pub client_errors: u64, - pub server_errors: u64, + /// The client networks that made requests. + pub clients: u64, + /// The countries they came from (known ones). + pub countries: u64, + /// The autonomous systems they came from (known ones). + pub asns: u64, } -impl Point { - fn add(&mut self, status: u16, requests: u64, bytes: u64) { - self.requests += requests; - self.bytes += bytes; - match status { - 400..=499 => self.client_errors += requests, - 500..=599 => self.server_errors += requests, - _ => {} - } - } -} - -/// The counts of a range. -#[derive(Debug, Clone, Default, Serialize, PartialEq)] -pub struct Totals { +/// A share of the range: a repository's, a country's, an autonomous +/// system's. +#[derive(Debug, Clone, Default, Serialize, PartialEq, Eq)] +pub struct Share { pub requests: u64, pub bytes: u64, - /// 4xx answers (the next three included). - pub client_errors: u64, - /// 401 and 403 answers. - pub denied: u64, - pub not_found: u64, - /// 429 answers. - pub throttled: u64, - /// 5xx answers. - pub server_errors: u64, - /// The mean time to the response headers. - pub mean_ms: Option, - /// The latency histogram's bound below which half, 95 % and 99 % of - /// the requests were answered ([`LATENCY_BOUNDS_MS`]); `None` without - /// requests, and above the last bound. - pub p50_ms: Option, - pub p95_ms: Option, - pub p99_ms: Option, + /// The client networks among them. + pub clients: u64, } -impl Totals { - fn add_status(&mut self, status: u16, requests: u64) { - match status { - 401 | 403 => self.denied += requests, - 404 => self.not_found += requests, - 429 => self.throttled += requests, - _ => {} - } - match status { - 400..=499 => self.client_errors += requests, - 500..=599 => self.server_errors += requests, - _ => {} +#[derive(Default)] +struct Tally { + share: Share, + networks: HashSet, +} + +impl Tally { + fn add(&mut self, row: &DayRow) { + self.share.requests += u64::try_from(row.requests).unwrap_or(0); + self.share.bytes += u64::try_from(row.bytes).unwrap_or(0); + if counts_as_client(&row.network) { + self.networks.insert(row.network.clone()); } } - fn finish(&mut self, counts: &Counts) { - if counts.requests == 0 { - return; - } - self.mean_ms = Some(counts.duration_us as f64 / counts.requests as f64 / 1_000.0); - let quantile = |q: f64| { - let wanted = (counts.requests as f64 * q).ceil() as u64; - let mut seen = 0; - for (i, n) in counts.latency.iter().enumerate() { - seen += n; - if seen >= wanted { - return LATENCY_BOUNDS_MS.get(i).copied(); - } - } - None - }; - self.p50_ms = quantile(0.5); - self.p95_ms = quantile(0.95); - self.p99_ms = quantile(0.99); + fn finish(mut self) -> Share { + self.share.clients = self.networks.len() as u64; + self.share } } -/// A surface's share of a range. -#[derive(Debug, Clone, Serialize, PartialEq, Eq)] -pub struct SurfaceShare { - pub surface: String, - pub requests: u64, - pub bytes: u64, +/// A client network being summed: its share, its requests by repository, +/// the days it was seen and the latest of them. +#[derive(Default)] +struct ClientTally<'a> { + client: ClientShare, + repositories: HashMap<&'a str, u64>, + days: BTreeSet, + latest: Option, } -/// A repository's share of a range. +fn counts_as_client(network: &str) -> bool { + !network.is_empty() && network != OTHER +} + +/// A repository's requests. #[derive(Debug, Clone, Serialize, PartialEq, Eq)] pub struct RepositoryShare { + /// `""` for the requests that addressed no repository. pub name: String, - pub requests: u64, - pub bytes: u64, - pub client_errors: u64, - pub server_errors: u64, + #[serde(flatten)] + pub share: Share, } -/// A status's share of a range. +/// A country's requests. #[derive(Debug, Clone, Serialize, PartialEq, Eq)] -pub struct StatusShare { - pub status: u16, - pub requests: u64, +pub struct CountryShare { + /// ISO 3166-1 alpha-2; `""` unknown. + pub country: String, + #[serde(flatten)] + pub share: Share, } -/// A client network's counts in a range. +/// An autonomous system's requests. +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct AsnShare { + /// `0` unknown. + pub asn: u32, + pub org: String, + #[serde(flatten)] + pub share: Share, +} + +/// A client network's requests. #[derive(Debug, Clone, Default, Serialize, PartialEq, Eq)] -pub struct ClientView { +pub struct ClientShare { pub network: String, /// The last address of the network seen. pub address: String, + pub country: String, + pub asn: u32, + pub as_org: String, pub requests: u64, pub bytes: u64, - pub client_errors: u64, - pub denied: u64, - pub not_found: u64, - pub throttled: u64, - pub server_errors: u64, - pub user_agent: Option, - /// The last account kind seen. - pub account: Option, - pub user_id: Option, - /// The user's name, filled in by the server. - pub username: Option, - /// The hosts of the instances it reached. - pub hosts: Vec, - pub first_seen: Option>, - pub last_seen: Option>, + /// The repositories it made the most requests to, most first. + pub repositories: Vec, + /// The days it made requests on. + pub days: u64, + pub first_day: Option, + pub last_day: Option, } -/// The traffic of a range: `GET /admin/traffic`. +/// The requests of a range of days: `GET /admin/traffic`. #[derive(Debug, Clone, Serialize)] pub struct Overview { - #[serde(flatten)] - pub span: Span, + pub from: NaiveDate, + /// The last day, included. + pub to: NaiveDate, /// The instances whose files the range holds (whatever the instance /// asked for). pub instances: Vec, pub totals: Totals, - pub series: Vec, - pub surfaces: Vec, + pub days: Vec, /// The [`TOP`] repositories with the most requests. pub repositories: Vec, - pub statuses: Vec, - /// The [`TOP`] client networks with the most requests (by the top - /// lists of the time buckets, whatever the repository asked for). - pub clients: Vec, + /// Every country, most requests first. + pub countries: Vec, + /// The [`TOP`] autonomous systems with the most requests. + pub asns: Vec, + /// The [`TOP`] client networks with the most requests. + pub clients: Vec, } -/// What to read. -#[derive(Debug, Clone, Default)] -pub struct Filter { - /// Only the files of this instance ([`crate::instance_id`]). - pub instance: Option, - /// Only the counts of this repository. - pub repository: Option, +fn naive(day: i64) -> NaiveDate { + Utc.timestamp_opt(day, 0) + .single() + .map(|t| t.date_naive()) + .unwrap_or_default() } -/// The files of `area` covering any of `[from, to)`. +/// The day files of the days `from` to `to` (Unix seconds, both included). async fn files( objects: &dyn Objects, - area: Area, from: i64, to: i64, -) -> anyhow::Result> { - let first_day = (from - area.span() + 1).div_euclid(DAY) * DAY; +) -> anyhow::Result> { let days: Vec = (0..) - .map(|i| first_day + i * DAY) - .take_while(|day| *day < to) + .map(|i| from + i * DAY) + .take_while(|day| *day <= to) .collect(); let listings: Vec> = stream::iter(days) - .map(|day| async move { objects.list(&area.day_prefix(day)).await }) + .map(|day| async move { objects.list(&day_prefix(day)).await }) .buffered(CONCURRENCY) .try_collect() .await?; Ok(listings .into_iter() .flatten() - .filter_map(|listed| Some((parse_key(area, &listed.key)?, listed))) - .filter(|(file, _)| file.start < to && file.start + area.span() > from) + .filter_map(|listed| { + let (day, instance) = parse_day_key(&listed.key)?; + Some((day, instance, listed)) + }) .collect()) } -/// The decoded file `listed`, from the cache when it holds this version. -async fn read_counts( +/// The rows of the day file `listed`, from the cache when it holds this +/// version. +async fn read( objects: &dyn Objects, cache: &FileCache, - file: &FileKey, listed: &Listed, -) -> anyhow::Result>> { +) -> anyhow::Result>> { let cache_key = format!("{}#{}", listed.key, listed.etag.as_deref().unwrap_or("")); if let Some(hit) = cache.0.get(&cache_key).await { - return Ok(Some(hit)); + return Ok(hit); } let Some(body) = objects.get(&listed.key).await? else { // Deleted since the listing (pruned). - return Ok(None); - }; - let decoded = match file.kind { - FileKind::Buckets => Decoded::Buckets(decode(body)?), - FileKind::Clients => Decoded::Clients(decode(body)?), - FileKind::Requests => return Ok(None), + return Ok(Arc::new(Vec::new())); }; - let decoded = Arc::new(decoded); + let rows = Arc::new(decode::(body)?); if listed.etag.is_some() { - cache.0.insert(cache_key, decoded.clone()).await; + cache.0.insert(cache_key, rows.clone()).await; } - Ok(Some(decoded)) + Ok(rows) } -/// The files of counts of `span` (of `filter`'s instance, if any), read, -/// with the instance that wrote each; and every instance found. -async fn read_span( +/// The requests of the days `from` to `to` (both included) over +/// `objects`' files, as `filter` narrows them. +pub async fn overview( objects: &dyn Objects, cache: &FileCache, - span: &Span, - instance: Option<&str>, - kinds: &[FileKind], -) -> anyhow::Result<(Vec<(String, Arc)>, Vec)> { - let found = files( - objects, - span.area(), - span.from.timestamp(), - span.to.timestamp(), - ) - .await?; - let instances: BTreeSet = found.iter().map(|(f, _)| f.instance.clone()).collect(); - let wanted: Vec<(FileKey, Listed)> = found + from: NaiveDate, + to: NaiveDate, + filter: &Filter, +) -> anyhow::Result { + let start = |d: NaiveDate| { + d.and_hms_opt(0, 0, 0) + .map(|t| t.and_utc().timestamp()) + .unwrap_or_default() + }; + let (from_s, to_s) = (start(from), start(to)); + let found = files(objects, from_s, to_s).await?; + let instances: BTreeSet = found.iter().map(|(_, i, _)| i.clone()).collect(); + let wanted: Vec = found .into_iter() - .filter(|(f, _)| kinds.contains(&f.kind)) - .filter(|(f, _)| instance.is_none_or(|i| i == f.instance)) + .filter(|(_, i, _)| filter.instance.as_ref().is_none_or(|w| w == i)) + .map(|(_, _, listed)| listed) .collect(); - let read: Vec)>> = stream::iter(wanted) - .map(|(file, listed)| async move { - let decoded = read_counts(objects, cache, &file, &listed).await?; - anyhow::Ok(decoded.map(|d| (file.instance, d))) - }) + let read: Vec>> = stream::iter(wanted) + .map(|listed| async move { read(objects, cache, &listed).await }) .buffer_unordered(CONCURRENCY) .try_collect() .await?; - Ok(( - read.into_iter().flatten().collect(), - instances.iter().map(|id| InstanceView::of(id)).collect(), - )) -} -/// Sum client rows into views by network. -fn sum_clients<'a>(rows: impl Iterator) -> Vec { - let mut by_network: HashMap<&str, (ClientView, i64, BTreeSet)> = HashMap::new(); - for (instance, row) in rows { - let (view, latest, hosts) = by_network.entry(&row.network).or_insert_with(|| { - ( - ClientView { - network: row.network.clone(), - ..ClientView::default() - }, - i64::MIN, - BTreeSet::new(), - ) - }); - let n = |v: i64| u64::try_from(v).unwrap_or(0); - view.requests += n(row.requests); - view.bytes += n(row.bytes); - view.client_errors += n(row.client_errors); - view.denied += n(row.denied); - view.not_found += n(row.not_found); - view.throttled += n(row.throttled); - view.server_errors += n(row.server_errors); - hosts.insert(instance_parts(instance).0); - let start = at(row.start); - view.first_seen = Some(view.first_seen.map_or(start, |s| s.min(start))); - view.last_seen = Some(view.last_seen.map_or(start, |s| s.max(start))); - if row.start >= *latest { - *latest = row.start; - let some = |s: &str| (!s.is_empty()).then(|| s.to_string()); - if !row.address.is_empty() { - view.address.clone_from(&row.address); - } - if let Some(ua) = some(&row.user_agent) { - view.user_agent = Some(ua); - } - if let Some(account) = some(&row.account) { - view.account = Some(account); - view.user_id = (row.user_id != 0).then_some(row.user_id); - } - } - } - let mut views: Vec = by_network - .into_values() - .map(|(mut view, _, hosts)| { - view.hosts = hosts.into_iter().collect(); - view - }) + let mut days: BTreeMap = (0..) + .map(|i| from_s + i * DAY) + .take_while(|d| *d <= to_s) + .map(|d| (d, Tally::default())) .collect(); - views.sort_by(|a, b| b.requests.cmp(&a.requests).then(a.network.cmp(&b.network))); - views -} - -/// The traffic of `span` over `objects`' files. -pub async fn overview( - objects: &dyn Objects, - cache: &FileCache, - span: Span, - filter: &Filter, -) -> anyhow::Result { - let (files, instances) = read_span( - objects, - cache, - &span, - filter.instance.as_deref(), - &[FileKind::Buckets, FileKind::Clients], - ) - .await?; - - let mut series = span.points(); - let mut totals = Totals::default(); - let mut all = Counts::default(); - let mut surfaces: BTreeMap<&str, (u64, u64)> = BTreeMap::new(); - let mut repositories: HashMap<&str, RepositoryShare> = HashMap::new(); - let mut statuses: BTreeMap = BTreeMap::new(); - let mut clients = Vec::new(); - for (instance, decoded) in &files { - match &**decoded { - Decoded::Buckets(rows) => { - for row in rows { - if !span.contains(row.start) - || filter - .repository - .as_deref() - .is_some_and(|r| r != row.repository) - { - continue; - } - let counts = row.counts(); - let status = u16::try_from(row.status).unwrap_or(0); - all.add(&counts); - totals.add_status(status, counts.requests); - if let Some(point) = series.get_mut(span.point(row.start)) { - point.add(status, counts.requests, counts.bytes); - } - let surface = surfaces.entry(&row.surface).or_default(); - surface.0 += counts.requests; - surface.1 += counts.bytes; - *statuses.entry(status).or_default() += counts.requests; - if !row.repository.is_empty() { - let share = repositories.entry(&row.repository).or_insert_with(|| { - RepositoryShare { - name: row.repository.clone(), - requests: 0, - bytes: 0, - client_errors: 0, - server_errors: 0, - } - }); - share.requests += counts.requests; - share.bytes += counts.bytes; - match status { - 400..=499 => share.client_errors += counts.requests, - 500..=599 => share.server_errors += counts.requests, - _ => {} - } - } + let mut all = Tally::default(); + let mut repositories: HashMap<&str, Tally> = HashMap::new(); + let mut countries: HashMap<&str, Tally> = HashMap::new(); + let mut asns: HashMap = HashMap::new(); + let mut clients: HashMap<&str, ClientTally> = HashMap::new(); + for row in read.iter().flat_map(|rows| rows.iter()) { + if !(from_s..=to_s).contains(&row.day) || !filter.keeps(row) { + continue; + } + all.add(row); + if let Some(day) = days.get_mut(&row.day) { + day.add(row); + } + repositories.entry(&row.repository).or_default().add(row); + countries.entry(&row.country).or_default().add(row); + let asn = asns + .entry(row.asn) + .or_insert_with(|| (Tally::default(), "")); + asn.0.add(row); + if asn.1.is_empty() { + asn.1 = &row.as_org; + } + if counts_as_client(&row.network) { + let tally = clients.entry(&row.network).or_default(); + let client = &mut tally.client; + if client.network.is_empty() { + client.network.clone_from(&row.network); + } + let requests = u64::try_from(row.requests).unwrap_or(0); + client.requests += requests; + client.bytes += u64::try_from(row.bytes).unwrap_or(0); + *tally.repositories.entry(&row.repository).or_default() += requests; + tally.days.insert(row.day); + if tally.latest.is_none_or(|latest| row.day >= latest) { + tally.latest = Some(row.day); + if !row.address.is_empty() { + client.address.clone_from(&row.address); + } + if !row.country.is_empty() || row.asn != 0 { + client.country.clone_from(&row.country); + client.asn = u32::try_from(row.asn).unwrap_or(0); + client.as_org.clone_from(&row.as_org); } } - Decoded::Clients(rows) => clients.extend( - rows.iter() - .filter(|row| span.contains(row.start)) - .map(|row| (instance.as_str(), row)), - ), } } - totals.requests = all.requests; - totals.bytes = all.bytes; - totals.finish(&all); - let mut surfaces: Vec = surfaces + let by_requests = |a: &Share, b: &Share| b.requests.cmp(&a.requests); + let mut repositories: Vec = repositories .into_iter() - .map(|(surface, (requests, bytes))| SurfaceShare { - surface: surface.to_string(), - requests, - bytes, + .map(|(name, t)| RepositoryShare { + name: name.to_string(), + share: t.finish(), }) .collect(); - surfaces.sort_by(|a, b| b.requests.cmp(&a.requests).then(a.surface.cmp(&b.surface))); - let mut repositories: Vec = repositories.into_values().collect(); - repositories.sort_by(|a, b| b.requests.cmp(&a.requests).then(a.name.cmp(&b.name))); + repositories.sort_by(|a, b| by_requests(&a.share, &b.share).then(a.name.cmp(&b.name))); repositories.truncate(TOP); - let mut clients = sum_clients(clients.into_iter()); - clients.truncate(TOP); - - Ok(Overview { - span, - instances, - totals, - series, - surfaces, - repositories, - statuses: statuses - .into_iter() - .map(|(status, requests)| StatusShare { status, requests }) - .collect(), - clients, - }) -} - -/// A request of the request log. -#[derive(Debug, Clone, Serialize, PartialEq)] -pub struct LogEntry { - pub time: DateTime, - pub address: String, - pub method: String, - pub path: String, - pub surface: String, - pub repository: Option, - pub status: u16, - pub bytes: u64, - /// Time to the response headers. - pub duration_ms: f64, - pub user_agent: Option, - pub account: Option, - pub user_id: Option, - /// The user's name, filled in by the server. - pub username: Option, - /// The host of the instance that answered. - pub host: String, -} - -/// How often a value occurs in a client's log. -#[derive(Debug, Clone, Serialize, PartialEq, Eq)] -pub struct Tally { - pub value: String, - pub requests: u64, -} - -/// How often an account occurs in a client's log. -#[derive(Debug, Clone, Serialize, PartialEq, Eq)] -pub struct AccountTally { - /// The account kind ([`crate::Account::kind`]); `None` when unknown. - pub account: Option, - pub user_id: Option, - /// The user's name, filled in by the server. - pub username: Option, - pub requests: u64, -} - -/// The request log's part of a client's view. -#[derive(Debug, Clone, Serialize)] -pub struct ClientLog { - /// The stretch read: the last [`LOG_RANGE_HOURS`] of the range at most. - pub from: DateTime, - pub to: DateTime, - /// The client's requests in it. - pub matched: u64, - /// The newest [`LOG_LIMIT`] of them, newest first. - pub entries: Vec, - /// The paths, user agents, accounts and statuses of all of them, most - /// frequent first ([`TOP`] each). - pub paths: Vec, - pub user_agents: Vec, - pub accounts: Vec, - pub statuses: Vec, -} - -/// One client network's traffic: `GET /admin/traffic/client`. -#[derive(Debug, Clone, Serialize)] -pub struct ClientDetail { - pub network: String, - #[serde(flatten)] - pub span: Span, - /// The instances whose files the range holds. - pub instances: Vec, - /// Its counts, from the time buckets it was among the top clients of. - pub totals: Option, - pub series: Vec, - pub log: ClientLog, -} - -/// `network`'s traffic in `span`: its counts in the top lists, and its -/// requests in the request log. -pub async fn client( - objects: &dyn Objects, - cache: &FileCache, - span: Span, - instance: Option<&str>, - network: &str, -) -> anyhow::Result { - let (files, instances) = - read_span(objects, cache, &span, instance, &[FileKind::Clients]).await?; - let mut series = span.points(); - let mut rows = Vec::new(); - for (instance, decoded) in &files { - if let Decoded::Clients(found) = &**decoded { - for row in found { - if row.network == network && span.contains(row.start) { - if let Some(point) = series.get_mut(span.point(row.start)) { - let n = |v: i64| u64::try_from(v).unwrap_or(0); - point.requests += n(row.requests); - point.bytes += n(row.bytes); - point.client_errors += n(row.client_errors); - point.server_errors += n(row.server_errors); - } - rows.push((instance.as_str(), row)); - } - } - } - } - let totals = sum_clients(rows.into_iter()).into_iter().next(); - let log = client_log(objects, &span, instance, network).await?; - Ok(ClientDetail { - network: network.to_string(), - span, - instances, - totals, - series, - log, - }) -} - -fn tallies(counts: HashMap) -> Vec { - let mut tallies: Vec = counts + let mut countries: Vec = countries .into_iter() - .map(|(value, requests)| Tally { value, requests }) + .map(|(country, t)| CountryShare { + country: country.to_string(), + share: t.finish(), + }) .collect(); - tallies.sort_by(|a, b| b.requests.cmp(&a.requests).then(a.value.cmp(&b.value))); - tallies.truncate(TOP); - tallies -} - -async fn client_log( - objects: &dyn Objects, - span: &Span, - instance: Option<&str>, - network: &str, -) -> anyhow::Result { - let to = span.to.timestamp(); - let from = span.from.timestamp().max(to - LOG_RANGE_HOURS * HOUR); - // A file is named after the hour of its flush, which comes up to a - // minute after its rows. - let wanted: Vec<(FileKey, Listed)> = files(objects, Area::Requests, from, to + 120) - .await? + countries.sort_by(|a, b| by_requests(&a.share, &b.share).then(a.country.cmp(&b.country))); + let known_countries = countries.iter().filter(|c| !c.country.is_empty()).count(); + let mut asns: Vec = asns .into_iter() - .filter(|(f, _)| instance.is_none_or(|i| i == f.instance)) - .collect(); - let matched: Vec> = stream::iter(wanted) - .map(|(file, listed)| async move { - let Some(body) = objects.get(&listed.key).await? else { - return anyhow::Ok(Vec::new()); - }; - let rows: Vec = decode(body)?; - Ok(rows - .into_iter() - .filter(|row| row.network == network) - .filter(|row| (from * 1_000..to * 1_000).contains(&row.time_ms)) - .map(|row| (file.instance.clone(), row)) - .collect()) + .map(|(asn, (t, org))| AsnShare { + asn: u32::try_from(asn).unwrap_or(0), + org: org.to_string(), + share: t.finish(), }) - .buffer_unordered(CONCURRENCY) - .try_collect() - .await?; - let mut matched: Vec<(String, LogRow)> = matched.into_iter().flatten().collect(); - matched.sort_by_key(|(_, row)| std::cmp::Reverse(row.time_ms)); - - let (mut paths, mut user_agents, mut accounts, mut statuses) = ( - HashMap::new(), - HashMap::new(), - HashMap::new(), - HashMap::new(), - ); - for (_, row) in &matched { - *paths.entry(row.path.clone()).or_default() += 1; - if !row.user_agent.is_empty() { - *user_agents.entry(row.user_agent.clone()).or_default() += 1; - } - *accounts - .entry((row.account.clone(), row.user_id)) - .or_insert(0u64) += 1; - *statuses.entry(row.status.to_string()).or_default() += 1; - } - let some = |s: String| (!s.is_empty()).then_some(s); - Ok(ClientLog { - from: at(from), - to: at(to), - matched: matched.len() as u64, - paths: tallies(paths), - user_agents: tallies(user_agents), - accounts: { - let mut accounts: Vec = accounts + .collect(); + asns.sort_by(|a, b| by_requests(&a.share, &b.share).then(a.asn.cmp(&b.asn))); + let known_asns = asns.iter().filter(|a| a.asn != 0).count(); + asns.truncate(TOP); + let mut clients: Vec = clients + .into_values() + .map(|tally| { + let ClientTally { + mut client, + repositories: repos, + days: seen, + .. + } = tally; + let mut repos: Vec<(&str, u64)> = repos.into_iter().collect(); + repos.sort_by(|a, b| b.1.cmp(&a.1).then(a.0.cmp(b.0))); + client.repositories = repos .into_iter() - .map(|((account, user_id), requests)| AccountTally { - account: (!account.is_empty()).then_some(account), - user_id: (user_id != 0).then_some(user_id), - username: None, - requests, - }) + .take(5) + .map(|(r, _)| r.to_string()) .collect(); - accounts.sort_by(|a, b| { - b.requests - .cmp(&a.requests) - .then_with(|| (&a.account, a.user_id).cmp(&(&b.account, b.user_id))) - }); - accounts.truncate(TOP); - accounts + client.days = seen.len() as u64; + client.first_day = seen.first().map(|d| naive(*d)); + client.last_day = seen.last().map(|d| naive(*d)); + client + }) + .collect(); + clients.sort_by(|a, b| b.requests.cmp(&a.requests).then(a.network.cmp(&b.network))); + clients.truncate(TOP); + + let total = all.finish(); + Ok(Overview { + from, + to, + instances: instances.iter().map(|id| InstanceView::of(id)).collect(), + totals: Totals { + requests: total.requests, + bytes: total.bytes, + clients: total.clients, + countries: known_countries as u64, + asns: known_asns as u64, }, - statuses: tallies(statuses), - entries: matched + days: days .into_iter() - .take(LOG_LIMIT) - .map(|(instance, row)| LogEntry { - time: Utc - .timestamp_millis_opt(row.time_ms) - .single() - .unwrap_or_default(), - address: row.address, - method: row.method, - path: row.path, - surface: row.surface, - repository: some(row.repository), - status: u16::try_from(row.status).unwrap_or(0), - bytes: u64::try_from(row.bytes).unwrap_or(0), - duration_ms: row.duration_us as f64 / 1_000.0, - user_agent: some(row.user_agent), - account: some(row.account), - user_id: (row.user_id != 0).then_some(row.user_id), - username: None, - host: instance_parts(&instance).0, + .map(|(day, t)| { + let share = t.finish(); + DayPoint { + day: naive(day), + requests: share.requests, + bytes: share.bytes, + clients: share.clients, + } }) .collect(), + repositories, + countries, + asns, + clients, }) } #[cfg(test)] mod tests { use super::*; + use crate::geo::{Geo, GeoKind}; use crate::layout::Flusher; use crate::objects::MemoryObjects; use crate::record::{Event, Recorder}; - use crate::slot::{Account, Noted}; + use crate::slot::Noted; + use crate::testing::{asn_db, country_db}; use crate::TrafficSettings; - use std::time::Duration; + use chrono::DateTime; fn t(rfc3339: &str) -> DateTime { DateTime::parse_from_rfc3339(rfc3339).unwrap().to_utc() } - fn event(at: &str, network: &str, repository: &str, status: u16, ms: u64) -> Event { + fn d(date: &str) -> NaiveDate { + NaiveDate::parse_from_str(date, "%Y-%m-%d").unwrap() + } + + fn event(at: &str, client: &str, repository: &str) -> Event { Event { at: t(at), - surface: "plugin_npm", - method: "GET", - status, - duration: Duration::from_millis(ms), bytes: 1_000, - path: format!("/repository/{repository}/left-pad"), - user_agent: Some("npm/10".into()), noted: Noted { - client: network.parse().ok(), - network: Some(network.into()), - account: Some(Account { - kind: "token", - user_id: Some(3), - }), - repository: Some(repository.into()), + client: client.parse().ok(), + network: Some(client.into()), + repository: (!repository.is_empty()).then(|| repository.to_string()), }, } } - /// Two instances, each with its own files. + /// Two instances, each with its own files, over two days. async fn two_instances() -> Arc { let objects = Arc::new(MemoryObjects::default()); let settings = TrafficSettings::default(); - let now = t("2026-10-30T14:10:00Z"); + let geo = Geo::default(); + geo.load( + GeoKind::Country, + country_db(&[("192.0.2.0/24", "DE"), ("198.51.100.0/24", "US")]), + "1".into(), + ) + .unwrap(); + geo.load( + GeoKind::Asn, + asn_db(&[ + ("192.0.2.0/24", 64_500, "Example DE"), + ("198.51.100.0/24", 64_501, "Example US"), + ]), + "1".into(), + ) + .unwrap(); for (instance, events) in [ ( - "a_20261030T100000Z_0001", + "a_20261029T100000Z_0001", vec![ - event("2026-10-30T14:00:01Z", "192.0.2.1", "npm", 200, 20), - event("2026-10-30T14:00:02Z", "192.0.2.1", "npm", 404, 2), - event("2026-10-30T14:05:00Z", "192.0.2.2", "npm-proxy", 500, 900), + event("2026-10-29T10:00:00Z", "192.0.2.1", "npm"), + event("2026-10-30T10:00:00Z", "192.0.2.1", "npm"), + event("2026-10-30T10:00:01Z", "198.51.100.9", "docker"), ], ), ( - "b_20261030T110000Z_0002", + "b_20261029T110000Z_0002", vec![ - event("2026-10-30T14:00:30Z", "192.0.2.1", "npm", 200, 40), - event("2026-10-30T13:59:59Z", "192.0.2.3", "npm", 401, 1), + event("2026-10-30T11:00:00Z", "192.0.2.1", "npm"), + event("2026-10-30T11:00:00Z", "192.0.2.1", "npm"), + event("2026-10-30T11:00:01Z", "203.0.113.5", ""), ], ), ] { @@ -861,7 +527,9 @@ mod tests { recorder.record(e); } let mut flusher = Flusher::new(objects.clone(), instance.into()); - flusher.flush(recorder.drain(now), &settings, now).await; + flusher + .flush(recorder.drain(), &settings, &geo, t("2026-10-30T12:00:00Z")) + .await; } objects } @@ -870,116 +538,114 @@ mod tests { async fn the_overview_sums_every_instance() { let objects = two_instances().await; let cache = FileCache::new(); - let now = t("2026-10-30T14:10:00Z"); - let span = Span::new(t("2026-10-30T13:10:00Z"), now, now); - assert_eq!((span.resolution, span.step_seconds), ("minute", 60)); - - let all = overview(&*objects, &cache, span, &Filter::default()) - .await - .unwrap(); - assert_eq!(all.instances.len(), 2); - assert_eq!(all.instances[0].host, "a"); - assert_eq!(all.totals.requests, 5); - assert_eq!( - ( - all.totals.client_errors, - all.totals.denied, - all.totals.not_found - ), - (2, 1, 1) - ); - assert_eq!(all.totals.server_errors, 1); - assert_eq!(all.totals.p50_ms, Some(25)); - assert_eq!(all.totals.p99_ms, Some(1_000)); - assert_eq!(all.series.len(), 60); - assert_eq!(all.series.iter().map(|p| p.requests).sum::(), 5); - assert_eq!(all.repositories[0].name, "npm"); - assert_eq!(all.repositories[0].requests, 4); - let top = &all.clients[0]; - assert_eq!((top.network.as_str(), top.requests), ("192.0.2.1", 3)); - assert_eq!(top.hosts, vec!["a", "b"]); - assert_eq!(top.user_id, Some(3)); - - let one = overview( + let all = overview( &*objects, &cache, - span, - &Filter { - instance: Some("b_20261030T110000Z_0002".into()), - repository: None, - }, + d("2026-10-24"), + d("2026-10-30"), + &Filter::default(), ) .await .unwrap(); - assert_eq!(one.totals.requests, 2); - assert_eq!(one.instances.len(), 2, "the selector still lists both"); - - let repo = overview( - &*objects, - &cache, - span, - &Filter { - instance: None, - repository: Some("npm-proxy".into()), - }, - ) - .await - .unwrap(); - assert_eq!(repo.totals.requests, 1); - - // A week reads the files by hour. - let week = Span::new(t("2026-10-23T14:10:00Z"), now, now); - assert_eq!((week.resolution, week.step_seconds), ("hour", 3_600)); - let week = overview(&*objects, &cache, week, &Filter::default()) - .await - .unwrap(); - assert_eq!(week.totals.requests, 5); - } - - #[tokio::test] - async fn a_client_view_reads_its_log() { - let objects = two_instances().await; - let now = t("2026-10-30T14:10:00Z"); - let span = Span::new(t("2026-10-30T13:10:00Z"), now, now); - let detail = client(&*objects, &FileCache::new(), span, None, "192.0.2.1") - .await - .unwrap(); - assert_eq!(detail.totals.as_ref().unwrap().requests, 3); - assert_eq!(detail.log.matched, 3); + assert_eq!(all.instances.len(), 2); + assert_eq!(all.instances[0].host, "a"); + assert_eq!(all.days.len(), 7); + assert_eq!(all.days[5].requests, 1, "the 29th"); assert_eq!( - detail.log.entries[0].time, - t("2026-10-30T14:00:30Z"), - "newest first" + (all.days[6].requests, all.days[6].clients), + (5, 3), + "the 30th" ); - assert_eq!(detail.log.entries[0].host, "b"); - assert_eq!(detail.log.entries[0].repository.as_deref(), Some("npm")); - assert_eq!(detail.log.paths[0].value, "/repository/npm/left-pad"); assert_eq!( - detail.log.statuses[0], - Tally { - value: "200".into(), - requests: 2 + all.totals, + Totals { + requests: 6, + bytes: 6_000, + clients: 3, + countries: 2, + asns: 2 } ); + assert_eq!(all.repositories[0].name, "npm"); + assert_eq!( + ( + all.repositories[0].share.requests, + all.repositories[0].share.clients + ), + (4, 1) + ); + assert_eq!(all.countries[0].country, "DE"); + let unknown = all.countries.iter().find(|c| c.country.is_empty()).unwrap(); + assert_eq!(unknown.share.requests, 1, "203.0.113.5 is in no database"); + assert_eq!( + (all.asns[0].asn, all.asns[0].org.as_str()), + (64_500, "Example DE") + ); + let top = &all.clients[0]; + assert_eq!( + ( + top.network.as_str(), + top.requests, + top.days, + top.country.as_str() + ), + ("192.0.2.1", 4, 2, "DE") + ); + assert_eq!(top.repositories, vec!["npm"]); + assert_eq!(top.first_day, Some(d("2026-10-29"))); } - #[test] - fn a_span_ends_after_the_second_it_ends_in() { - let now = t("2026-10-30T14:10:00.250Z"); - let span = Span::new(t("2026-10-30T13:10:00Z"), now, now); - assert_eq!(span.to, t("2026-10-30T14:10:01Z")); - } - - #[test] - fn spans_pick_their_resolution_and_step() { - let now = t("2026-10-30T14:10:00Z"); - let day = Span::new(t("2026-10-29T14:10:00Z"), now, now); - assert_eq!((day.resolution, day.step_seconds), ("minute", 600)); - assert_eq!(day.from, t("2026-10-29T14:10:00Z")); - let month = Span::new(t("2026-09-30T14:10:00Z"), now, now); - assert_eq!((month.resolution, month.step_seconds), ("hour", 21_600)); - // Older than the minute files: by hour. - let old = Span::new(t("2026-10-20T00:00:00Z"), t("2026-10-20T01:00:00Z"), now); - assert_eq!(old.resolution, "hour"); + #[tokio::test] + async fn filters_narrow_every_list() { + let objects = two_instances().await; + let cache = FileCache::new(); + let range = (d("2026-10-24"), d("2026-10-30")); + let only = |f: Filter| { + let objects = objects.clone(); + let cache = cache.clone(); + async move { + overview(&*objects, &cache, range.0, range.1, &f) + .await + .unwrap() + } + }; + let b = only(Filter { + instance: Some("b_20261029T110000Z_0002".into()), + ..Filter::default() + }) + .await; + assert_eq!(b.totals.requests, 3); + assert_eq!(b.instances.len(), 2, "the selector still lists both"); + let docker = only(Filter { + repository: Some("docker".into()), + ..Filter::default() + }) + .await; + assert_eq!(docker.totals.requests, 1); + assert_eq!(docker.countries[0].country, "US"); + let none = only(Filter { + repository: Some(String::new()), + ..Filter::default() + }) + .await; + assert_eq!(none.totals.requests, 1, "the requests to no repository"); + let de = only(Filter { + country: Some("DE".into()), + ..Filter::default() + }) + .await; + assert_eq!(de.totals.requests, 4); + let asn = only(Filter { + asn: Some(64_501), + ..Filter::default() + }) + .await; + assert_eq!(asn.clients[0].network, "198.51.100.9"); + let client = only(Filter { + network: Some("192.0.2.1".into()), + ..Filter::default() + }) + .await; + assert_eq!((client.totals.requests, client.totals.clients), (4, 1)); } } diff --git a/crates/traffic/src/record.rs b/crates/traffic/src/record.rs index c69e73ed..7cc421a6 100644 --- a/crates/traffic/src/record.rs +++ b/crates/traffic/src/record.rs @@ -1,100 +1,70 @@ -//! Counting requests in memory, minute by minute. +//! Counting requests in memory until the next flush. use std::collections::HashMap; +use std::net::IpAddr; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Mutex, PoisonError}; -use std::time::Duration; use chrono::{DateTime, Utc}; -use crate::rows::{BucketKey, Counts, LogRow}; use crate::slot::Noted; -use crate::top::TopClients; -use crate::TrafficSettings; -/// The networks a minute's top list keeps track of ([`TopClients`]). -const MINUTE_CLIENTS: usize = 1_024; -/// The keys a minute counts under before new repositories are counted -/// without their name: a bound whatever the handlers note. -const MAX_KEYS_PER_MINUTE: usize = 10_000; -/// The request log rows kept between two flushes; more are dropped (and -/// counted in the log of the flush). -const MAX_LOG_ROWS: usize = 250_000; -/// The closed minutes kept while flushes fail; older ones are dropped. -const MAX_CLOSED_MINUTES: usize = 180; -/// The longest path and user agent the request log keeps, in bytes. -const MAX_PATH_BYTES: usize = 1_024; -const MAX_USER_AGENT_BYTES: usize = 256; +/// The keys counted between two flushes before new client networks are +/// counted as [`OTHER`]: a bound whatever the number of addresses. +const MAX_PENDING: usize = 100_000; + +/// The network requests are counted under past a bound. +pub const OTHER: &str = "other"; + +const DAY: i64 = 86_400; /// One answered request. #[derive(Debug, Clone)] pub struct Event { /// When the response was sent (its body done or cut off). pub at: DateTime, - /// The surface (`forge_telemetry::http::Surface`). - pub surface: &'static str, - pub method: &'static str, - pub status: u16, - /// Time to the response headers. - pub duration: Duration, /// Response body bytes sent. pub bytes: u64, - /// The path as the log keeps it (secrets replaced); empty when the log - /// is off ([`Recorder::logs_requests`]). - pub path: String, - pub user_agent: Option, /// What the layers of the request noted ([`crate::Slot`]). pub noted: Noted, } -/// The counts of one minute. -#[derive(Debug, Clone)] -pub struct Minute { - /// Its start, in Unix seconds. - pub start: i64, - pub buckets: HashMap, - pub clients: TopClients, +/// What requests are counted under until the next flush. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct PendingKey { + /// The start of the day (UTC), in Unix seconds. + pub day: i64, + /// `""` for none. + pub repository: String, + /// `""` for none. + pub network: String, } -impl Minute { - fn new(start: i64) -> Minute { - Minute { - start, - buckets: HashMap::new(), - clients: TopClients::new(MINUTE_CLIENTS), - } - } - - fn is_empty(&self) -> bool { - self.buckets.is_empty() - } +/// The counts of a [`PendingKey`]. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct PendingCounts { + pub requests: u64, + pub bytes: u64, + /// The last address of the network seen. Kept in memory until the + /// flush looks up its country and autonomous system, even with client + /// addresses switched off. + pub address: Option, } /// What a flush takes from the [`Recorder`]. #[derive(Debug, Default)] pub struct Drained { - /// The minutes counted to the end. - pub minutes: Vec, - pub(crate) log: Vec, - /// Log rows dropped for want of room. - pub log_dropped: u64, -} - -#[derive(Debug)] -struct Pending { - current: Minute, - closed: Vec, - log: Vec, + pub counts: Vec<(PendingKey, PendingCounts)>, + /// Requests counted under [`OTHER`] for want of room. + pub overflowed: u64, } /// Counts this instance's requests until the next flush. #[derive(Debug)] pub struct Recorder { enabled: AtomicBool, - addresses: AtomicBool, - log: AtomicBool, - log_dropped: AtomicU64, - pending: Mutex, + overflowed: AtomicU64, + pending: Mutex>, } impl Default for Recorder { @@ -103,176 +73,59 @@ impl Default for Recorder { } } -/// `s` cut to at most `max` bytes, on a character boundary. -fn clip(mut s: String, max: usize) -> String { - if s.len() > max { - let mut end = max; - while !s.is_char_boundary(end) { - end -= 1; - } - s.truncate(end); - } - s -} - impl Recorder { - /// A recorder with the default settings ([`TrafficSettings::default`]), - /// until [`Self::apply`] says otherwise. + /// A recorder that counts until [`Self::set_enabled`] says otherwise. pub fn new() -> Recorder { - let defaults = TrafficSettings::default(); Recorder { - enabled: AtomicBool::new(defaults.enabled), - addresses: AtomicBool::new(defaults.client_addresses), - log: AtomicBool::new(defaults.request_log), - log_dropped: AtomicU64::new(0), - pending: Mutex::new(Pending { - current: Minute::new(0), - closed: Vec::new(), - log: Vec::new(), - }), + enabled: AtomicBool::new(true), + overflowed: AtomicU64::new(0), + pending: Mutex::new(HashMap::new()), } } - /// Follow `settings` from the next request on. - pub fn apply(&self, settings: &TrafficSettings) { - self.enabled.store(settings.enabled, Ordering::Relaxed); - self.addresses - .store(settings.client_addresses, Ordering::Relaxed); - self.log.store(settings.request_log, Ordering::Relaxed); + /// Count requests from the next one on, or not. + pub fn set_enabled(&self, enabled: bool) { + self.enabled.store(enabled, Ordering::Relaxed); } - /// Whether requests are counted at all. + /// Whether requests are counted. pub fn enabled(&self) -> bool { self.enabled.load(Ordering::Relaxed) } - /// Whether requests go to the request log: only then does an - /// [`Event`] need its path. - pub fn logs_requests(&self) -> bool { - self.enabled() && self.log.load(Ordering::Relaxed) - } - - fn lock(&self) -> std::sync::MutexGuard<'_, Pending> { - self.pending.lock().unwrap_or_else(PoisonError::into_inner) - } - /// Count `event`. pub fn record(&self, event: Event) { if !self.enabled() { return; } - let addresses = self.addresses.load(Ordering::Relaxed); - let Event { - at, - surface, - method, - status, - duration, - bytes, - path, - user_agent, - noted, - } = event; - let user_agent = user_agent.map(|ua| clip(ua, MAX_USER_AGENT_BYTES)); - let (network, address) = match (addresses, noted.network, noted.client) { - (true, Some(network), client) => { - (network, client.map(|c| c.to_string()).unwrap_or_default()) - } - _ => (String::new(), String::new()), - }; - let (account, user_id) = noted - .account - .map(|a| (a.kind, a.user_id)) - .unwrap_or(("", None)); - let repository = noted.repository.unwrap_or_default(); - let row = self.log.load(Ordering::Relaxed).then(|| LogRow { - time_ms: at.timestamp_millis(), - network: network.clone(), - address: address.clone(), - method: method.to_string(), - path: clip(path, MAX_PATH_BYTES), - surface: surface.to_string(), - repository: repository.clone(), - status: i32::from(status), - bytes: i64::try_from(bytes).unwrap_or(i64::MAX), - duration_us: i64::try_from(duration.as_micros()).unwrap_or(i64::MAX), - user_agent: user_agent.clone().unwrap_or_default(), - account: account.to_string(), - user_id: user_id.unwrap_or(0), - }); - let mut key = BucketKey { - surface: surface.to_string(), - repository, - method: method.to_string(), - status, + let Event { at, bytes, noted } = event; + let mut key = PendingKey { + day: at.timestamp().div_euclid(DAY) * DAY, + repository: noted.repository.unwrap_or_default(), + network: noted.network.unwrap_or_default(), }; - - let minute = at.timestamp().div_euclid(60) * 60; - let mut pending = self.lock(); - if minute > pending.current.start { - let done = std::mem::replace(&mut pending.current, Minute::new(minute)); - if !done.is_empty() { - pending.closed.push(done); - if pending.closed.len() > MAX_CLOSED_MINUTES { - pending.closed.remove(0); - } - } - } - // A request that ended in an earlier minute than one already counted - // (the clock stepped back) counts in the current one. - let current = &mut pending.current; - if current.buckets.len() >= MAX_KEYS_PER_MINUTE && !current.buckets.contains_key(&key) { - key.repository.clear(); - } - current - .buckets - .entry(key) - .or_default() - .observe(duration, bytes); - if !network.is_empty() { - let client = current.clients.entry(&network); - client.observe(status, bytes); - client.address = address; - if let Some(ua) = user_agent { - client.user_agent = ua; - } - if !account.is_empty() { - client.account = account.to_string(); - client.user_id = user_id; - } + let mut address = noted.client; + let mut pending = self.pending.lock().unwrap_or_else(PoisonError::into_inner); + if pending.len() >= MAX_PENDING && !pending.contains_key(&key) { + key.network = OTHER.to_string(); + address = None; + self.overflowed.fetch_add(1, Ordering::Relaxed); } - if let Some(row) = row { - if pending.log.len() < MAX_LOG_ROWS { - pending.log.push(row); - } else { - self.log_dropped.fetch_add(1, Ordering::Relaxed); - } + let counts = pending.entry(key).or_default(); + counts.requests += 1; + counts.bytes += bytes; + if address.is_some() { + counts.address = address; } } - /// Take the minutes that ended before `now`, and the request log. - pub fn drain(&self, now: DateTime) -> Drained { - self.take(Some(now.timestamp().div_euclid(60) * 60)) - } - - /// Take everything, the minute still being counted included (on - /// shutdown). - pub fn drain_all(&self) -> Drained { - self.take(None) - } - - fn take(&self, before: Option) -> Drained { - let mut pending = self.lock(); - let close = before.is_none_or(|start| pending.current.start < start); - if close && !pending.current.is_empty() { - let next = Minute::new(before.unwrap_or(pending.current.start)); - let done = std::mem::replace(&mut pending.current, next); - pending.closed.push(done); - } + /// Take what was counted since the last call. + pub fn drain(&self) -> Drained { + let pending = + std::mem::take(&mut *self.pending.lock().unwrap_or_else(PoisonError::into_inner)); Drained { - minutes: std::mem::take(&mut pending.closed), - log: std::mem::take(&mut pending.log), - log_dropped: self.log_dropped.swap(0, Ordering::Relaxed), + counts: pending.into_iter().collect(), + overflowed: self.overflowed.swap(0, Ordering::Relaxed), } } } @@ -280,98 +133,43 @@ impl Recorder { #[cfg(test)] mod tests { use super::*; - use crate::slot::Account; use chrono::TimeZone; - fn at(secs: i64) -> DateTime { - Utc.timestamp_opt(1_700_000_000 + secs, 0).unwrap() - } - - fn event(secs: i64, status: u16) -> Event { + fn event(secs: i64, network: &str, repository: Option<&str>) -> Event { Event { - at: at(secs), - surface: "nuget", - method: "GET", - status, - duration: Duration::from_millis(12), + at: Utc.timestamp_opt(1_793_318_400 + secs, 0).unwrap(), bytes: 100, - path: "/repository/nuget/v3/index.json".into(), - user_agent: Some("NuGet Command Line/6.0".into()), noted: Noted { - client: Some("192.0.2.10".parse().unwrap()), - network: Some("192.0.2.10".into()), - account: Some(Account { - kind: "token", - user_id: Some(7), - }), - repository: Some("nuget".into()), + client: network.parse().ok(), + network: Some(network.into()), + repository: repository.map(str::to_string), }, } } #[test] - fn minutes_close_when_the_next_one_starts() { + fn requests_are_counted_by_day_repository_and_network() { let r = Recorder::new(); - // 1_700_000_000 is 20 s into its minute: the first two requests - // share it, the third starts the next one. - r.record(event(0, 200)); - r.record(event(30, 404)); - r.record(event(45, 200)); - let early = r.drain(at(41)); - assert_eq!(early.minutes.len(), 1, "the first minute ended"); - assert_eq!(early.minutes[0].start, 1_699_999_980); - assert_eq!(early.log.len(), 3, "the log goes with every flush"); - let minute = &early.minutes[0]; - let ok = BucketKey { - surface: "nuget".into(), - repository: "nuget".into(), - method: "GET".into(), - status: 200, - }; - assert_eq!(minute.buckets[&ok].requests, 1); - let top = minute.clients.top(1); - assert_eq!(top[0].0, "192.0.2.10"); - assert_eq!(top[0].1.user_id, Some(7)); - - let rest = r.drain_all(); - assert_eq!(rest.minutes.len(), 1); - assert_eq!(rest.minutes[0].start, 1_700_000_040); - assert_eq!( - rest.minutes[0] - .buckets - .values() - .map(|c| c.requests) - .sum::(), - 1 - ); - assert!(r.drain_all().minutes.is_empty()); + r.record(event(10, "192.0.2.1", Some("npm"))); + r.record(event(20, "192.0.2.1", Some("npm"))); + r.record(event(30, "192.0.2.1", None)); + r.record(event(DAY + 1, "192.0.2.1", Some("npm"))); + let mut counts = r.drain().counts; + counts.sort_by_key(|(k, _)| (k.day, k.repository.clone())); + assert_eq!(counts.len(), 3); + assert_eq!(counts[0].0.repository, ""); + assert_eq!(counts[1].1.requests, 2); + assert_eq!(counts[1].1.bytes, 200); + assert_eq!(counts[1].1.address, "192.0.2.1".parse().ok()); + assert_eq!(counts[2].0.day, 1_793_318_400 + DAY); + assert!(r.drain().counts.is_empty(), "drained"); } #[test] - fn settings_switch_parts_off() { + fn switched_off_it_counts_nothing() { let r = Recorder::new(); - r.apply(&TrafficSettings { - client_addresses: false, - request_log: false, - ..TrafficSettings::default() - }); - assert!(!r.logs_requests()); - r.record(event(0, 200)); - let d = r.drain_all(); - assert!(d.log.is_empty()); - assert!(d.minutes[0].clients.is_empty(), "no addresses"); - - r.apply(&TrafficSettings { - enabled: false, - ..TrafficSettings::default() - }); - r.record(event(0, 200)); - assert!(r.drain_all().minutes.is_empty()); - } - - #[test] - fn long_values_are_clipped_on_a_char_boundary() { - assert_eq!(clip("äää".into(), 3), "ä"); - assert_eq!(clip("abc".into(), 10), "abc"); + r.set_enabled(false); + r.record(event(0, "192.0.2.1", None)); + assert!(r.drain().counts.is_empty()); } } diff --git a/crates/traffic/src/rows.rs b/crates/traffic/src/rows.rs index 6f39f6c4..2d91faab 100644 --- a/crates/traffic/src/rows.rs +++ b/crates/traffic/src/rows.rs @@ -1,9 +1,8 @@ -//! The counts and the rows of the Parquet files. +//! The rows of the Parquet files. //! //! The files are plain Parquet, readable by any tool (DuckDB, pandas). A //! column the record API cannot leave empty holds `""` or `0` for none: the -//! derived reader takes no `Option` field. Times are Unix seconds (`start`) -//! or milliseconds (`time_ms`), in UTC. +//! derived reader takes no `Option` field. use std::sync::Arc; @@ -15,211 +14,29 @@ use parquet::file::writer::SerializedFileWriter; use parquet::record::{RecordReader, RecordWriter}; use parquet_derive::{ParquetRecordReader, ParquetRecordWriter}; -/// The upper bounds of the latency histogram, in milliseconds: a request is -/// counted in the first bucket whose bound it does not exceed, or in the -/// last, above all of them. -pub const LATENCY_BOUNDS_MS: [u64; 13] = [ - 5, 10, 25, 50, 100, 250, 500, 1_000, 2_500, 5_000, 10_000, 30_000, 60_000, -]; - -/// The buckets of the latency histogram: one per bound, and one above. -pub const LATENCY_BUCKETS: usize = LATENCY_BOUNDS_MS.len() + 1; - -/// What requests are counted under within a time bucket. -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct BucketKey { - /// The surface (`forge_telemetry::http::Surface`), narrowed to the - /// format for a repository's requests. - pub surface: String, - /// The repository the request addressed; `""` for none. - pub repository: String, - /// The request method. - pub method: String, - /// The response status. - pub status: u16, -} - -/// The counts of a [`BucketKey`] in a time bucket. -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct Counts { - pub requests: u64, - /// Response body bytes sent. - pub bytes: u64, - /// Time to the response headers, summed, in microseconds. - pub duration_us: u64, - /// The latency histogram ([`LATENCY_BOUNDS_MS`]). - pub latency: [u64; LATENCY_BUCKETS], -} - -impl Counts { - /// Count one request. - pub fn observe(&mut self, duration: std::time::Duration, bytes: u64) { - self.requests += 1; - self.bytes += bytes; - let us = u64::try_from(duration.as_micros()).unwrap_or(u64::MAX); - self.duration_us = self.duration_us.saturating_add(us); - let bucket = LATENCY_BOUNDS_MS - .iter() - .position(|&bound| us <= bound * 1_000) - .unwrap_or(LATENCY_BOUNDS_MS.len()); - self.latency[bucket] += 1; - } - - /// Add `other`'s counts. - pub fn add(&mut self, other: &Counts) { - self.requests += other.requests; - self.bytes += other.bytes; - self.duration_us = self.duration_us.saturating_add(other.duration_us); - for (mine, theirs) in self.latency.iter_mut().zip(other.latency) { - *mine += theirs; - } - } -} - -/// A row of a `buckets` file: a key's counts in the bucket starting at -/// `start` (a minute or an hour). +/// A row of a day file: the requests one client network made to one +/// repository on one day, as one instance answered them. #[derive(Debug, Clone, Default, PartialEq, Eq, ParquetRecordWriter, ParquetRecordReader)] -pub(crate) struct BucketRow { - pub start: i64, - pub surface: String, +pub(crate) struct DayRow { + /// The start of the day (UTC), in Unix seconds. + pub day: i64, + /// The repository the requests addressed; `""` for requests that + /// addressed none (the web UI, the APIs, sign-in). pub repository: String, - pub method: String, - pub status: i32, - pub requests: i64, - pub bytes: i64, - pub duration_us: i64, - pub le_5ms: i64, - pub le_10ms: i64, - pub le_25ms: i64, - pub le_50ms: i64, - pub le_100ms: i64, - pub le_250ms: i64, - pub le_500ms: i64, - pub le_1s: i64, - pub le_2500ms: i64, - pub le_5s: i64, - pub le_10s: i64, - pub le_30s: i64, - pub le_60s: i64, - pub over_60s: i64, -} - -fn signed(n: u64) -> i64 { - i64::try_from(n).unwrap_or(i64::MAX) -} - -fn unsigned(n: i64) -> u64 { - u64::try_from(n).unwrap_or(0) -} - -impl BucketRow { - pub fn new(start: i64, key: &BucketKey, counts: &Counts) -> BucketRow { - let l = counts.latency.map(signed); - BucketRow { - start, - surface: key.surface.clone(), - repository: key.repository.clone(), - method: key.method.clone(), - status: i32::from(key.status), - requests: signed(counts.requests), - bytes: signed(counts.bytes), - duration_us: signed(counts.duration_us), - le_5ms: l[0], - le_10ms: l[1], - le_25ms: l[2], - le_50ms: l[3], - le_100ms: l[4], - le_250ms: l[5], - le_500ms: l[6], - le_1s: l[7], - le_2500ms: l[8], - le_5s: l[9], - le_10s: l[10], - le_30s: l[11], - le_60s: l[12], - over_60s: l[13], - } - } - - pub fn counts(&self) -> Counts { - Counts { - requests: unsigned(self.requests), - bytes: unsigned(self.bytes), - duration_us: unsigned(self.duration_us), - latency: [ - self.le_5ms, - self.le_10ms, - self.le_25ms, - self.le_50ms, - self.le_100ms, - self.le_250ms, - self.le_500ms, - self.le_1s, - self.le_2500ms, - self.le_5s, - self.le_10s, - self.le_30s, - self.le_60s, - self.over_60s, - ] - .map(unsigned), - } - } -} - -/// A row of a `clients` file: a client network's counts in the bucket -/// starting at `start`, as one of the bucket's top clients. -#[derive(Debug, Clone, Default, PartialEq, Eq, ParquetRecordWriter, ParquetRecordReader)] -pub(crate) struct ClientRow { - pub start: i64, - /// The network the requests are counted under. + /// The client network: the address, or an IPv6 address's /64; `""` with + /// client addresses switched off, `other` past the day's bound. pub network: String, - /// The last address of the network seen. + /// The last address of the network seen; `""` with addresses off. pub address: String, + /// The ISO 3166-1 code of the country the network is in; `""` unknown. + pub country: String, + /// The autonomous system the network belongs to; `0` unknown. + pub asn: i64, + /// The organisation of the autonomous system; `""` unknown. + pub as_org: String, pub requests: i64, + /// Response body bytes sent. pub bytes: i64, - /// 4xx answers, the next three included. - pub client_errors: i64, - /// 401 and 403 answers. - pub denied: i64, - pub not_found: i64, - /// 429 answers. - pub throttled: i64, - /// 5xx answers. - pub server_errors: i64, - /// The last user agent seen; `""` for none. - pub user_agent: String, - /// The last account kind seen ([`crate::Account::kind`]). - pub account: String, - /// The last account's user; `0` for none. - pub user_id: i64, -} - -/// A row of a `requests` file: one request of the request log. -#[derive(Debug, Clone, Default, PartialEq, Eq, ParquetRecordWriter, ParquetRecordReader)] -pub(crate) struct LogRow { - /// When the response was sent, in Unix milliseconds. - pub time_ms: i64, - /// The client network; `""` with client addresses switched off. - pub network: String, - /// The client address; `""` with client addresses switched off. - pub address: String, - pub method: String, - /// The path, without the query and with secrets replaced - /// (`forge-server`'s `recorded_path`). - pub path: String, - pub surface: String, - /// `""` for none. - pub repository: String, - pub status: i32, - pub bytes: i64, - /// Time to the response headers, in microseconds. - pub duration_us: i64, - /// `""` for none. - pub user_agent: String, - pub account: String, - /// `0` for none. - pub user_id: i64, } /// `rows` as a Parquet file, in one row group. @@ -259,54 +76,26 @@ where #[cfg(test)] mod tests { use super::*; - use std::time::Duration; - - #[test] - fn latency_lands_in_its_bucket() { - let mut c = Counts::default(); - c.observe(Duration::from_micros(4_999), 1); - c.observe(Duration::from_millis(5), 1); - c.observe(Duration::from_micros(5_001), 1); - c.observe(Duration::from_secs(61), 1); - assert_eq!(c.latency[0], 2, "up to and including 5 ms"); - assert_eq!(c.latency[1], 1); - assert_eq!(c.latency[LATENCY_BUCKETS - 1], 1, "above the last bound"); - assert_eq!(c.requests, 4); - assert_eq!(c.bytes, 4); - } #[test] fn rows_round_trip_through_parquet() { - let key = BucketKey { - surface: "nuget".into(), - repository: "nuget-hosted".into(), - method: "GET".into(), - status: 200, - }; - let mut counts = Counts::default(); - counts.observe(Duration::from_millis(30), 1234); - counts.observe(Duration::from_millis(3), 10); let rows = vec![ - BucketRow::new(1_700_000_040, &key, &counts), - BucketRow::new(1_700_000_100, &key, &Counts::default()), + DayRow { + day: 1_793_318_400, + repository: "npm-proxy".into(), + network: "2001:db8::/64".into(), + address: "2001:db8::7".into(), + country: "DE".into(), + asn: 64_500, + as_org: "Example AS".into(), + requests: 12, + bytes: 3_400, + }, + DayRow::default(), ]; - let back: Vec = decode(encode(&rows).unwrap().into()).unwrap(); + let back: Vec = decode(encode(&rows).unwrap().into()).unwrap(); assert_eq!(back, rows); - assert_eq!(back[0].counts(), counts); - - let log = vec![LogRow { - time_ms: 1_700_000_040_123, - network: "2001:db8::/64".into(), - address: "2001:db8::7".into(), - method: "GET".into(), - path: "/v2/".into(), - status: 401, - ..LogRow::default() - }]; - let back: Vec = decode(encode(&log).unwrap().into()).unwrap(); - assert_eq!(back, log); - - let none: Vec = decode(encode::(&[]).unwrap().into()).unwrap(); + let none: Vec = decode(encode::(&[]).unwrap().into()).unwrap(); assert!(none.is_empty()); } } diff --git a/crates/traffic/src/settings.rs b/crates/traffic/src/settings.rs index 42754dfa..1ece4299 100644 --- a/crates/traffic/src/settings.rs +++ b/crates/traffic/src/settings.rs @@ -4,8 +4,9 @@ pub const MAX_RETENTION_DAYS: u32 = 36_500; /// The traffic settings of every instance, set in the web UI (Admin → -/// Traffic) or through `/admin/settings/traffic` and kept in the database. -/// An install that never saved them uses [`TrafficSettings::default`]. +/// Traffic → Settings) or through `/admin/settings/traffic` and kept in the +/// database. An install that never saved them uses +/// [`TrafficSettings::default`]. /// /// Read leniently, like the other stored settings: a key left out takes its /// default and one this version does not know is ignored. @@ -15,17 +16,11 @@ pub struct TrafficSettings { /// Whether requests are counted at all. Off, nothing is recorded or /// written; what was written stays until it expires. pub enabled: bool, - /// Whether client addresses are recorded: the top clients and the - /// request log's addresses. + /// Whether client addresses are recorded. Off, requests are still + /// counted by repository, country and autonomous system. pub client_addresses: bool, - /// Whether every request is written to the request log. - pub request_log: bool, - /// How long the counts by hour are kept, in days; `0` keeps them. - pub statistics_retention_days: u32, - /// How long the top clients are kept, in days. - pub client_retention_days: u32, - /// How long the request log is kept, in days. - pub request_log_retention_days: u32, + /// How long the day files are kept, in days. + pub retention_days: u32, } impl Default for TrafficSettings { @@ -33,10 +28,7 @@ impl Default for TrafficSettings { TrafficSettings { enabled: true, client_addresses: true, - request_log: true, - statistics_retention_days: 400, - client_retention_days: 30, - request_log_retention_days: 7, + retention_days: 90, } } } @@ -44,22 +36,11 @@ impl Default for TrafficSettings { impl TrafficSettings { /// What is wrong with these settings, if anything. pub fn check(&self) -> Result<(), String> { - if self.statistics_retention_days > MAX_RETENTION_DAYS { + if !(1..=MAX_RETENTION_DAYS).contains(&self.retention_days) { return Err(format!( - "statistics_retention_days must be at most {MAX_RETENTION_DAYS}, or 0 to keep the statistics" + "retention_days must be between 1 and {MAX_RETENTION_DAYS}" )); } - for (name, days) in [ - ("client_retention_days", self.client_retention_days), - ( - "request_log_retention_days", - self.request_log_retention_days, - ), - ] { - if !(1..=MAX_RETENTION_DAYS).contains(&days) { - return Err(format!("{name} must be between 1 and {MAX_RETENTION_DAYS}")); - } - } Ok(()) } } diff --git a/crates/traffic/src/slot.rs b/crates/traffic/src/slot.rs index f565a502..b8d0dba6 100644 --- a/crates/traffic/src/slot.rs +++ b/crates/traffic/src/slot.rs @@ -1,11 +1,11 @@ //! What the layers of a request learn about it, for its count: the client -//! address, the account it ran as and the repository it addressed. +//! address and the repository it addressed. //! //! The outermost middleware opens a [`Slot`] around the request //! ([`Slot::scope`]); code running in the request's task fills it in with -//! [`note_client`], [`note_account`] and [`note_repository`] without the -//! values being threaded through every handler. Outside a slot (a worker, a -//! test calling a handler directly) noting does nothing. +//! [`note_client`] and [`note_repository`] without the values being threaded +//! through every handler. Outside a slot (a worker, a test calling a handler +//! directly) noting does nothing. use std::future::Future; use std::net::IpAddr; @@ -15,16 +15,6 @@ tokio::task_local! { static SLOT: Arc>; } -/// The account a request ran as, as the statistics name it. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Account { - /// How the caller authenticated: `anonymous`, `session`, `token`, - /// `password`, `trusted_header` or `workload`. - pub kind: &'static str, - /// The user, for the kinds that have one. - pub user_id: Option, -} - /// What was noted about a request. #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct Noted { @@ -34,8 +24,6 @@ pub struct Noted { /// The network the address is counted under (the address itself, an /// IPv6 address's /64), as text. pub network: Option, - /// The account the request ran as. - pub account: Option, /// The repository the request addressed, once it was found. pub repository: Option, } @@ -73,13 +61,6 @@ pub fn note_client(address: IpAddr, network: String) { }); } -/// Note the account the request runs as. A later note replaces an earlier -/// one (the registry's own token names the caller after the principal layer -/// found none). -pub fn note_account(account: Account) { - note(|n| n.account = Some(account)); -} - /// Note the repository the request addresses, once it is known to exist. pub fn note_repository(name: &str) { note(|n| n.repository = Some(name.to_string())); @@ -97,16 +78,11 @@ mod tests { note_client("192.0.2.7".parse().unwrap(), "192.0.2.7".into()); tokio::task::yield_now().await; note_repository("nuget"); - note_account(Account { - kind: "token", - user_id: Some(4), - }); }) .await; let noted = slot.take(); assert_eq!(noted.repository.as_deref(), Some("nuget")); assert_eq!(noted.network.as_deref(), Some("192.0.2.7")); - assert_eq!(noted.account.unwrap().user_id, Some(4)); assert_eq!(slot.take(), Noted::default(), "taken once"); } } diff --git a/crates/traffic/src/testing.rs b/crates/traffic/src/testing.rs new file mode 100644 index 00000000..d42a5861 --- /dev/null +++ b/crates/traffic/src/testing.rs @@ -0,0 +1,174 @@ +//! Small MaxMind databases for tests: an IPv4 search tree over a few +//! networks, their records and the metadata, in the MaxMind DB format +//! (), as GeoLite2 editions name +//! them. + +use std::net::Ipv4Addr; + +/// A value of the MaxMind DB data section. +enum Value<'a> { + Str(&'a str), + U16(u16), + U32(u32), + U64(u64), + Map(Vec<(&'a str, Value<'a>)>), + Array(Vec>), +} + +/// The control byte(s) of a value of `kind` (1–15) and `size`. +fn control(out: &mut Vec, kind: u8, size: usize) { + let (first, extra): (u8, Vec) = match size { + 0..=28 => (size as u8, vec![]), + 29..=284 => (29, vec![(size - 29) as u8]), + _ => { + let n = size - 285; + (30, vec![(n >> 8) as u8, n as u8]) + } + }; + if kind <= 7 { + out.push((kind << 5) | first); + } else { + out.push(first); + out.push(kind - 7); + } + out.extend(extra); +} + +fn uint(out: &mut Vec, kind: u8, n: u64) { + let bytes = n.to_be_bytes(); + let skip = bytes.iter().take_while(|b| **b == 0).count(); + control(out, kind, bytes.len() - skip); + out.extend(&bytes[skip..]); +} + +fn encode(out: &mut Vec, value: &Value<'_>) { + match value { + Value::Str(s) => { + control(out, 2, s.len()); + out.extend(s.as_bytes()); + } + Value::U16(n) => uint(out, 5, u64::from(*n)), + Value::U32(n) => uint(out, 6, u64::from(*n)), + Value::U64(n) => uint(out, 9, *n), + Value::Map(entries) => { + control(out, 7, entries.len()); + for (k, v) in entries { + encode(out, &Value::Str(k)); + encode(out, v); + } + } + Value::Array(items) => { + control(out, 11, items.len()); + for item in items { + encode(out, item); + } + } + } +} + +#[derive(Clone, Copy)] +enum Record { + Empty, + Node(usize), + Data(usize), +} + +/// A database of `database_type` mapping each network (`a.b.c.d/len`) to +/// its record. +fn database(database_type: &str, networks: Vec<(&str, Value<'_>)>) -> Vec { + let mut nodes: Vec<[Record; 2]> = vec![[Record::Empty; 2]]; + let mut data = Vec::new(); + for (network, record) in networks { + let (addr, len) = network.split_once('/').expect("a.b.c.d/len"); + let addr = u32::from(addr.parse::().expect("an IPv4 network")); + let len: usize = len.parse().expect("a prefix length"); + assert!((1..=32).contains(&len), "prefix length"); + let offset = data.len(); + encode(&mut data, &record); + let mut node = 0; + for depth in 0..len { + let bit = ((addr >> (31 - depth)) & 1) as usize; + if depth == len - 1 { + nodes[node][bit] = Record::Data(offset); + } else { + node = match nodes[node][bit] { + Record::Node(next) => next, + _ => { + nodes.push([Record::Empty; 2]); + let next = nodes.len() - 1; + nodes[node][bit] = Record::Node(next); + next + } + }; + } + } + } + let node_count = nodes.len(); + let mut out = Vec::new(); + for node in &nodes { + for record in node { + let value = match *record { + Record::Empty => node_count, + Record::Node(n) => n, + Record::Data(offset) => node_count + 16 + offset, + } as u32; + out.extend(&value.to_be_bytes()[1..]); + } + } + out.extend([0u8; 16]); + out.extend(data); + out.extend(b"\xAB\xCD\xEFMaxMind.com"); + encode( + &mut out, + &Value::Map(vec![ + ("binary_format_major_version", Value::U16(2)), + ("binary_format_minor_version", Value::U16(0)), + ("build_epoch", Value::U64(1_793_318_400)), + ("database_type", Value::Str(database_type)), + ("description", Value::Map(vec![("en", Value::Str("Test"))])), + ("ip_version", Value::U16(4)), + ("languages", Value::Array(vec![Value::Str("en")])), + ("node_count", Value::U32(node_count as u32)), + ("record_size", Value::U16(24)), + ]), + ); + out +} + +/// A `GeoLite2-Country` database of `(network, ISO code)`. +pub fn country_db(networks: &[(&str, &str)]) -> Vec { + database( + "GeoLite2-Country", + networks + .iter() + .map(|(net, code)| { + ( + *net, + Value::Map(vec![( + "country", + Value::Map(vec![("iso_code", Value::Str(code))]), + )]), + ) + }) + .collect(), + ) +} + +/// A `GeoLite2-ASN` database of `(network, number, organisation)`. +pub fn asn_db(networks: &[(&str, u32, &str)]) -> Vec { + database( + "GeoLite2-ASN", + networks + .iter() + .map(|(net, asn, org)| { + ( + *net, + Value::Map(vec![ + ("autonomous_system_number", Value::U32(*asn)), + ("autonomous_system_organization", Value::Str(org)), + ]), + ) + }) + .collect(), + ) +} diff --git a/crates/traffic/src/top.rs b/crates/traffic/src/top.rs deleted file mode 100644 index 9ca5b4cc..00000000 --- a/crates/traffic/src/top.rs +++ /dev/null @@ -1,190 +0,0 @@ -//! The clients with the most requests, counted in bounded memory. -//! -//! [`TopClients`] keeps at most twice its capacity of networks; when full, -//! it drops the half with the fewest requests. Whatever the number of -//! addresses a flood comes from, the memory stays bounded, and a network -//! with many requests stays in: it is dropped only while it has fewer -//! requests than half the networks kept. A network dropped and seen again -//! starts counting from zero, so a count can only be low, never high. - -use std::collections::HashMap; - -/// A client network's counts. -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct ClientCounts { - pub requests: u64, - pub bytes: u64, - /// 4xx answers (the next three included). - pub client_errors: u64, - /// 401 and 403 answers. - pub denied: u64, - pub not_found: u64, - /// 429 answers. - pub throttled: u64, - /// 5xx answers. - pub server_errors: u64, - /// The last address of the network seen. - pub address: String, - /// The last user agent seen; `""` for none. - pub user_agent: String, - /// The last account kind seen; `""` for none. - pub account: String, - /// The last account's user. - pub user_id: Option, -} - -impl ClientCounts { - /// Count one request answered with `status`. - pub fn observe(&mut self, status: u16, bytes: u64) { - self.requests += 1; - self.bytes += bytes; - match status { - 401 | 403 => self.denied += 1, - 404 => self.not_found += 1, - 429 => self.throttled += 1, - _ => {} - } - match status { - 400..=499 => self.client_errors += 1, - 500..=599 => self.server_errors += 1, - _ => {} - } - } - - /// Add `other`'s counts; its last-seen values win where it has them. - pub fn add(&mut self, other: &ClientCounts) { - self.requests += other.requests; - self.bytes += other.bytes; - self.client_errors += other.client_errors; - self.denied += other.denied; - self.not_found += other.not_found; - self.throttled += other.throttled; - self.server_errors += other.server_errors; - if !other.address.is_empty() { - self.address.clone_from(&other.address); - } - if !other.user_agent.is_empty() { - self.user_agent.clone_from(&other.user_agent); - } - if !other.account.is_empty() { - self.account.clone_from(&other.account); - self.user_id = other.user_id; - } - } -} - -/// The networks with the most requests, about `capacity` of them. -#[derive(Debug, Clone)] -pub struct TopClients { - capacity: usize, - counts: HashMap, -} - -impl TopClients { - /// Keep about `capacity` networks (at least one). - pub fn new(capacity: usize) -> TopClients { - TopClients { - capacity: capacity.max(1), - counts: HashMap::new(), - } - } - - /// The counts of `network`, added when new. - pub fn entry(&mut self, network: &str) -> &mut ClientCounts { - if !self.counts.contains_key(network) { - if self.counts.len() >= self.capacity * 2 { - self.shrink(); - } - self.counts - .insert(network.to_string(), ClientCounts::default()); - } - self.counts.get_mut(network).expect("just inserted") - } - - /// Add all of `other`'s networks. - pub fn merge(&mut self, other: &TopClients) { - for (network, counts) in &other.counts { - self.entry(network).add(counts); - } - } - - /// Whether no network was counted. - pub fn is_empty(&self) -> bool { - self.counts.is_empty() - } - - /// The `n` networks with the most requests, most first (ties by - /// network, for a stable order). - pub fn top(&self, n: usize) -> Vec<(&str, &ClientCounts)> { - let mut all: Vec<_> = self - .counts - .iter() - .map(|(network, counts)| (network.as_str(), counts)) - .collect(); - all.sort_by(|a, b| b.1.requests.cmp(&a.1.requests).then(a.0.cmp(b.0))); - all.truncate(n); - all - } - - /// Keep only the `n` networks with the most requests. - pub fn truncate(&mut self, n: usize) { - if self.counts.len() > n { - let keep: Vec = self - .top(n) - .into_iter() - .map(|(k, _)| k.to_string()) - .collect(); - let mut kept = HashMap::with_capacity(keep.len()); - for network in keep { - if let Some(counts) = self.counts.remove(&network) { - kept.insert(network, counts); - } - } - self.counts = kept; - } - } - - fn shrink(&mut self) { - self.truncate(self.capacity); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn heavy_clients_survive_a_flood_of_light_ones() { - let mut top = TopClients::new(8); - for round in 0..1_000 { - for heavy in ["192.0.2.1", "192.0.2.2"] { - top.entry(heavy).observe(200, 10); - } - top.entry(&format!("10.0.{}.{}", round / 250, round % 250)) - .observe(404, 1); - } - let best = top.top(2); - assert_eq!(best[0], ("192.0.2.1", &best[0].1.clone())); - assert_eq!(best[0].1.requests, 1_000); - assert_eq!(best[1].0, "192.0.2.2"); - assert!(top.counts.len() <= 16, "bounded: {}", top.counts.len()); - } - - #[test] - fn status_classes_are_counted() { - let mut c = ClientCounts::default(); - for status in [200, 401, 403, 404, 429, 500, 503, 302] { - c.observe(status, 0); - } - assert_eq!( - ( - c.client_errors, - c.denied, - c.not_found, - c.throttled, - c.server_errors - ), - (4, 2, 1, 1, 2) - ); - } -} diff --git a/docs/changes/2.2.0/709.feature.md b/docs/changes/2.2.0/709.feature.md index 67a08546..ab279c17 100644 --- a/docs/changes/2.2.0/709.feature.md +++ b/docs/changes/2.2.0/709.feature.md @@ -1,12 +1,11 @@ -**Admin → Traffic** shows what every instance answered: requests over time, -data sent, 4xx and 5xx answers and latency, by repository, surface and -status, and the 20 client networks with the most requests. Each client opens -its own view with its requests from a request log. Every instance writes its -counts to Forge's bucket once a minute, as Parquet files under `traffic/v1/` -that DuckDB or pandas read too, so the page covers all instances without -Redis. What is recorded (client addresses and the request log can be -switched off) and how long each part is kept (statistics 400 days, top -clients 30, request log 7 by default) is set under **Admin → Traffic → -Settings** (`/admin/settings/traffic`). The listing of these files makes -Forge call `ListObjectsV2`, which the documented `s3:ListBucket` permission -already allows. +**Admin → Traffic** shows how many requests each client made to each +repository, day by day, over every instance, and the countries and +autonomous systems they came from. Each list narrows the view: a repository +shows where its requests came from, a country, network or client what it +used. Every instance writes its daily counts to Forge's bucket once a +minute, as Parquet files under `traffic/v1/` that DuckDB or pandas read too, +so the page covers all instances without Redis. Countries and networks come +from MaxMind's GeoLite2 databases, which Forge downloads every three days +with a MaxMind account ID and license key set under **Admin → Traffic → +Settings**, or which can be put in the bucket by hand. Recording, client +addresses and the retention (90 days by default) are set there too. diff --git a/docs/deployment.md b/docs/deployment.md index c8840a01..5bbdda38 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -1301,51 +1301,72 @@ console log stays as it is. ## Traffic statistics -**Admin → Traffic** (`/manage/traffic`) shows what every instance answered: -requests over time, data sent, 4xx and 5xx answers and latency, by -repository, surface and status, and the 20 client networks with the most -requests. A client opens its own view with its requests from the request -log: time, method, path, status, size, duration, account, user agent and -the instance that answered. Health probes are not counted. - -Each instance counts its requests in memory and writes them to Forge's -bucket once a minute, as Parquet files under `traffic/v1/`. Every instance -writes files of its own, so the page reads the files of all of them from the -bucket and sums them: it needs no Redis and no call between instances, and -it shows every instance, or one, whichever answers it. The files are plain -Parquet, so DuckDB, pandas or Spark read them too: - -| Files | What they hold | Kept | -| --- | --- | --- | -| `minutes///.buckets.parquet` | An hour's requests by minute, repository, surface, method and status: count, bytes, a latency histogram | 3 days (the views of the last 48 hours) | -| `hours//.buckets.parquet` | The same of a day, by hour | **Statistics**, 400 days by default | -| `…/.clients.parquet` (both) | The top 100 client networks of each minute or hour | **Top clients**, 30 days by default | -| `requests///..parquet` | The request log: a file per instance and minute | **Request log**, 7 days by default | - -What is recorded and how long it is kept is set under **Admin → Traffic → -Settings** (`/admin/settings/traffic`), for every instance at once. Each -instance deletes the files past their retention about once an hour, -whichever instance wrote them. +**Admin → Traffic** (`/manage/traffic`) shows how many requests each client +made to each repository, day by day, over every instance, and where the +requests came from: the countries and autonomous systems (the provider or +company a network belongs to) of the clients. Every list narrows the view: +open a repository to see where its requests came from, or a country, a +network or a client to see what it used. Health probes are not counted. + +Each instance counts its requests in memory, by day (UTC), repository and +client network, and writes them to Forge's bucket once a minute, as one +Parquet file per instance and day: +`traffic/v1/days//.parquet`, a row per repository and +client with its country, autonomous system, requests and bytes sent. Every +instance writes files of its own, so the page reads the files of all of +them from the bucket and sums them: it needs no Redis and no call between +instances. The files are plain Parquet, so DuckDB, pandas or Spark read them +too. + +What is recorded and how long it is kept (90 days by default) is set under +**Admin → Traffic → Settings** (`/admin/settings/traffic`), for every +instance at once. Each instance deletes the days past the retention about +once an hour, whichever instance wrote them. - **Client addresses are personal data.** The address is the one Forge resolves for the login lockout: the socket peer, or the `X-Forwarded-For` of a [trusted proxy](#reverse-proxy). An IPv6 client is counted by its - /64. Addresses are kept only in the top clients and the request log, with - retentions of their own; **Client addresses** off records neither, and - **Request log** off keeps the counts and the top clients without the log. -- **Size.** The counts are a few small files per instance and hour. The - request log grows with the traffic: a row per request, with its path - (secrets in a path, such as a command-line login's, are replaced) and user - agent, compressed with Snappy. -- **Accuracy.** The last minute shows once the instances have written it. - An instance that stops writes what it counted on a graceful shutdown; one - that is killed loses its last minute. The top lists are counted in bounded - memory: under a flood of addresses a client's count can come out low, - never high. A client's view reads the request log of at most the last 24 - hours of its range, and shows the newest 500 requests. + /64. With **Client addresses** off, requests are still counted by + repository, country and autonomous system: the address is looked up in + memory and dropped before anything is written. +- **Accuracy.** Today fills in as the instances write it, once a minute. An + instance that stops writes what it counted on a graceful shutdown; one + that is killed loses its last minute. Past 500,000 clients a day on one + instance, new ones are counted together as `other`. The statistics are separate from the [OpenTelemetry metrics](#telemetry), -which carry no address, path or user and need a backend of their own. +which carry no address or repository per client and need a backend of their +own. + +### Countries and networks (MaxMind) + +The countries and autonomous systems come from MaxMind databases: the free +GeoLite2-Country and GeoLite2-ASN, or the commercial GeoIP2 editions. Forge +keeps them in its bucket, as `traffic/v1/geoip/country.mmdb` and +`traffic/v1/geoip/asn.mmdb`, and every instance loads them from there, +again within ten minutes of a change. A client is located when its row of +the day is first written; days written before a database was there stay +unknown. + +- **Downloaded by Forge.** Create a free MaxMind account and a license key + at [maxmind.com](https://www.maxmind.com/en/geolite2/signup) and enter + them under **Admin → Traffic → Settings → Geolocation**. The license key + is sealed like Forge's other secrets. One instance downloads the + databases at once with **Download now**, then every three days, and + stores them in the bucket; a failed download is retried six hours later, + and the settings show its error. Forge needs outbound HTTPS to + `download.maxmind.com` and the storage it redirects to; a mirror that + answers the same download API can be set as + `traffic.maxmind_url` (`FORGE_TRAFFIC__MAXMIND_URL`). **Turn off** forgets + the account and deletes the databases. +- **Put there yourself.** Without internet access, or for a commercial + edition, upload the `.mmdb` files to the bucket under those two keys (for + example with `geoipupdate` and `aws s3 cp`). A Country or City edition + fills `country.mmdb`, an ASN or ISP edition `asn.mmdb`; a file of another + edition is refused and logged. + +MaxMind's GeoLite license asks for its databases to be kept current and for +the credit the traffic screen shows next to the countries and networks. ## Running more than one instance diff --git a/forge.toml.example b/forge.toml.example index bb769146..03db509c 100644 --- a/forge.toml.example +++ b/forge.toml.example @@ -284,6 +284,13 @@ processed_retention_seconds = 604800 # (default: 604800 = 7 days) keep processe # bytes, and the outbox backlog for the metrics (database queries). # inventory_interval_seconds = 300 # (default: 300, at least 10) +[traffic] +# The traffic statistics (Admin → Traffic; docs/deployment.md → Traffic +# statistics). What they record and the MaxMind account their country and +# ASN databases come from are set in the web UI. Where Forge downloads those +# databases: MaxMind, or a mirror answering its download API. +# maxmind_url = "https://download.maxmind.com" # (default) + # Optional: OpenID Connect / Microsoft Entra interactive login. # This block is a one-shot SEED — on first boot it is imported into an # `auth_realm` row and authentication is managed in the admin UI from then on diff --git a/web/src/api/traffic.ts b/web/src/api/traffic.ts index 3ac30293..f541a123 100644 --- a/web/src/api/traffic.ts +++ b/web/src/api/traffic.ts @@ -4,32 +4,34 @@ import { httpClient } from '~/lib/httpClient' * Admin traffic API: the traffic statistics every instance writes to the * object store (`crates/server/src/admin/traffic.rs`, `crates/traffic`). * - * - `GET /admin/traffic` answers the traffic of a range over every instance - * (or the one `instance` names): counts by time bucket, surface, - * repository and status, latency, and the top 20 client networks. - * - `GET /admin/traffic/client` answers one client network's counts and - * its requests in the request log (the last 24 hours of the range at - * most, the newest 500). - * - `GET` / `PUT /admin/settings/traffic`: what is recorded and how long - * it is kept; `PUT` changes the fields it names. + * - `GET /admin/traffic` answers the requests of a range of days over every + * instance (or the one `instance` names), by day, repository, country, + * autonomous system and client network, narrowed by the filters given. + * - `GET` / `PUT /admin/settings/traffic`: what is recorded and how long it + * is kept; `PUT` changes the fields it names. + * - `/admin/traffic/geoip`: the geolocation databases and the MaxMind + * download (`PUT` the account, `DELETE` it and the databases, `POST + * …/update` to download now). */ -/** The ranges the views offer. */ -export type TrafficRange = '1h' | '6h' | '24h' | '7d' | '30d' | '90d' +/** The ranges the screen offers, in days up to today. */ +export type TrafficRange = 'today' | '7d' | '30d' | '90d' | '365d' export const TRAFFIC_RANGES: { id: TrafficRange; label: string }[] = [ - { id: '1h', label: '1 hour' }, - { id: '6h', label: '6 hours' }, - { id: '24h', label: '24 hours' }, + { id: 'today', label: 'Today' }, { id: '7d', label: '7 days' }, { id: '30d', label: '30 days' }, { id: '90d', label: '90 days' }, + { id: '365d', label: '1 year' }, ] export function isTrafficRange(value: unknown): value is TrafficRange { return TRAFFIC_RANGES.some((r) => r.id === value) } +/** What a filter value says for "none": no repository, no known country. */ +export const NONE = '-' + /** An instance whose files the range holds. */ export interface TrafficInstance { id: string @@ -37,166 +39,116 @@ export interface TrafficInstance { started_at: string | null } -/** A point of a series: the counts of one step. */ -export interface TrafficPoint { - start: string +/** A day of the range. */ +export interface TrafficDay { + /** `YYYY-MM-DD`, UTC. */ + day: string requests: number bytes: number - client_errors: number - server_errors: number -} - -/** What is recorded now (the settings), so a view can say why data is missing. */ -export interface TrafficRecording { - enabled: boolean - client_addresses: boolean - request_log: boolean -} - -/** The span a view covers. */ -interface TrafficSpan { - from: string - to: string - step_seconds: number - /** What the files read count by. */ - resolution: 'minute' | 'hour' + /** The client networks that made requests that day. */ + clients: number } -export interface TrafficTotals { +/** A share of the range. */ +export interface TrafficShare { requests: number bytes: number - /** 4xx answers (the next three included). */ - client_errors: number - /** 401 and 403 answers. */ - denied: number - not_found: number - /** 429 answers. */ - throttled: number - /** 5xx answers. */ - server_errors: number - mean_ms: number | null - /** The latency bound half, 95 % and 99 % of the requests stayed within. */ - p50_ms: number | null - p95_ms: number | null - p99_ms: number | null -} - -/** A client network's counts. */ + clients: number +} + +/** A client network's requests. */ export interface TrafficClient { network: string address: string + /** ISO 3166-1 alpha-2; `""` unknown. */ + country: string + /** `0` unknown. */ + asn: number + as_org: string requests: number bytes: number - client_errors: number - denied: number - not_found: number - throttled: number - server_errors: number - user_agent: string | null - account: string | null - user_id: number | null - username: string | null - hosts: string[] - first_seen: string | null - last_seen: string | null + /** The repositories it made the most requests to (`""`: none). */ + repositories: string[] + days: number + first_day: string | null + last_day: string | null } /** `GET /admin/traffic`. */ -export interface TrafficOverview extends TrafficSpan { +export interface TrafficOverview { + from: string + to: string instances: TrafficInstance[] - totals: TrafficTotals - series: TrafficPoint[] - surfaces: { surface: string; requests: number; bytes: number }[] - repositories: { - name: string + totals: { requests: number bytes: number - client_errors: number - server_errors: number - }[] - statuses: { status: number; requests: number }[] + clients: number + countries: number + asns: number + } + days: TrafficDay[] + /** `""` names the requests that addressed no repository. */ + repositories: ({ name: string } & TrafficShare)[] + countries: ({ country: string } & TrafficShare)[] + asns: ({ asn: number; org: string } & TrafficShare)[] clients: TrafficClient[] - recording: TrafficRecording -} - -/** A request of the request log. */ -export interface TrafficLogEntry { - time: string - address: string - method: string - path: string - surface: string - repository: string | null - status: number - bytes: number - duration_ms: number - user_agent: string | null - account: string | null - user_id: number | null - username: string | null - host: string -} - -/** How often a value occurs in a client's log. */ -export interface TrafficTally { - value: string - requests: number -} - -/** How often an account occurs in a client's log. */ -export interface TrafficAccountTally { - account: string | null - user_id: number | null - username: string | null - requests: number -} - -/** `GET /admin/traffic/client`. */ -export interface TrafficClientDetail extends TrafficSpan { - network: string - instances: TrafficInstance[] - /** Its counts in the time buckets it was among the top clients of. */ - totals: TrafficClient | null - series: TrafficPoint[] - log: { - from: string - to: string - matched: number - entries: TrafficLogEntry[] - paths: TrafficTally[] - user_agents: TrafficTally[] - accounts: TrafficAccountTally[] - statuses: TrafficTally[] + recording: { + enabled: boolean + client_addresses: boolean + /** Whether a country database is loaded. */ + countries: boolean + /** Whether an autonomous system database is loaded. */ + asns: boolean } - recording: TrafficRecording } /** `GET /admin/settings/traffic`. */ export interface TrafficSettings { enabled: boolean client_addresses: boolean - request_log: boolean - /** Days the counts by hour are kept; `0` keeps them. */ - statistics_retention_days: number - client_retention_days: number - request_log_retention_days: number + retention_days: number } /** `PUT /admin/settings/traffic` body: the fields to change. */ export type TrafficSettingsUpdate = Partial -/** What a view asks for. */ +/** A loaded geolocation database. */ +export interface GeoDatabase { + database_type: string + built_at: string | null + size: number +} + +/** `GET /admin/traffic/geoip`. */ +export interface GeoipStatus { + country: GeoDatabase | null + asn: GeoDatabase | null + maxmind: { + account_id: string | null + license_key_set: boolean + last_attempt: string | null + last_success: string | null + last_error: string | null + url: string + editions: string[] + } +} + +/** What the screen asks for: a range and the filters. */ export interface TrafficQuery { range: TrafficRange instance?: string repository?: string + network?: string + country?: string + asn?: string } export const trafficKeys = { all: ['traffic'] as const, overview: (q: TrafficQuery) => ['traffic', 'overview', q] as const, - client: (network: string, q: TrafficQuery) => ['traffic', 'client', network, q] as const, settings: () => ['settings', 'traffic'] as const, + geoip: () => ['traffic', 'geoip'] as const, } /** `GET /admin/traffic`. */ @@ -206,19 +158,14 @@ export function fetchTrafficOverview( ): Promise { return httpClient.get('/admin/traffic', { signal, - query: { range: q.range, instance: q.instance, repository: q.repository }, - }) -} - -/** `GET /admin/traffic/client`. */ -export function fetchTrafficClient( - network: string, - q: TrafficQuery, - signal?: AbortSignal, -): Promise { - return httpClient.get('/admin/traffic/client', { - signal, - query: { network, range: q.range, instance: q.instance }, + query: { + range: q.range, + instance: q.instance, + repository: q.repository, + network: q.network, + country: q.country, + asn: q.asn, + }, }) } @@ -231,3 +178,26 @@ export function fetchTrafficSettings(signal?: AbortSignal): Promise { return httpClient.put('/admin/settings/traffic', body) } + +/** `GET /admin/traffic/geoip`. */ +export function fetchGeoip(signal?: AbortSignal): Promise { + return httpClient.get('/admin/traffic/geoip', { signal }) +} + +/** `PUT /admin/traffic/geoip`: the license key left out keeps the stored one. */ +export function updateGeoip(body: { + account_id: string + license_key?: string +}): Promise { + return httpClient.put('/admin/traffic/geoip', body) +} + +/** `POST /admin/traffic/geoip/update`. */ +export function downloadGeoip(): Promise { + return httpClient.post('/admin/traffic/geoip/update') +} + +/** `DELETE /admin/traffic/geoip`. */ +export function removeGeoip(): Promise { + return httpClient.delete('/admin/traffic/geoip') +} diff --git a/web/src/components/TrafficChart.tsx b/web/src/components/TrafficChart.tsx index 9afa07b9..f39a6439 100644 --- a/web/src/components/TrafficChart.tsx +++ b/web/src/components/TrafficChart.tsx @@ -1,24 +1,23 @@ import { useEffect, useId, useRef, useState, type KeyboardEvent, type PointerEvent } from 'react' -import type { TrafficPoint } from '~/api/traffic' +import type { TrafficDay } from '~/api/traffic' import { formatBytes } from '~/lib/oci' +import { dayLabel } from '~/lib/traffic' import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '~/components/ui' -/** What the chart plots of each point. */ -export type TrafficMetric = 'requests' | 'bytes' | 'client_errors' | 'server_errors' +/** What the chart plots of each day. */ +export type TrafficMetric = 'requests' | 'bytes' | 'clients' export const TRAFFIC_METRICS: { id: TrafficMetric; label: string }[] = [ { id: 'requests', label: 'Requests' }, { id: 'bytes', label: 'Data sent' }, - { id: 'client_errors', label: '4xx' }, - { id: 'server_errors', label: '5xx' }, + { id: 'clients', label: 'Clients' }, ] const METRIC_NAMES: Record = { requests: 'Requests', bytes: 'Data sent', - client_errors: '4xx answers', - server_errors: '5xx answers', + clients: 'Client networks', } const compact = new Intl.NumberFormat(undefined, { notation: 'compact', maximumFractionDigits: 1 }) @@ -46,56 +45,42 @@ function niceTicks(max: number, count = 4): number[] { return ticks } -/** A point's time as the axis and the tooltip say it. */ -function timeLabel(iso: string, stepSeconds: number, withDate: boolean): string { - const d = new Date(iso) - if (stepSeconds >= 86_400) return d.toLocaleDateString(undefined, { month: 'short', day: 'numeric' }) - const time = d.toLocaleTimeString(undefined, { hour: '2-digit', minute: '2-digit' }) - return withDate - ? `${d.toLocaleDateString(undefined, { month: 'short', day: 'numeric' })} ${time}` - : time +/** A day on the axis: `Oct 30`. */ +function shortDay(day: string): string { + const d = new Date(`${day}T00:00:00Z`) + return Number.isNaN(d.getTime()) + ? day + : d.toLocaleDateString(undefined, { month: 'short', day: 'numeric', timeZone: 'UTC' }) } -/** The intervals time labels may be apart, in seconds. */ -const LABEL_INTERVALS = [ - 60, 300, 600, 900, 1_800, 3_600, 7_200, 10_800, 21_600, 43_200, 86_400, 172_800, 604_800, -] +/** How many days apart the axis labels may be. */ +const LABEL_EVERY = [1, 2, 7, 14, 30, 61, 91] const HEIGHT = 220 -const MARGIN = { top: 12, right: 12, bottom: 28, left: 64 } +const MARGIN = { top: 12, right: 16, bottom: 28, left: 64 } /** - * One series of a traffic view over time: an area with a 2px line, a - * crosshair that snaps to the nearest point and a tooltip with its value - * (the arrow keys step through the points), or the same points as a table. - * Drawn at the width it is given, so its text keeps its size. + * One measure of the traffic by day: an area with a 2px line, a crosshair + * that snaps to the nearest day and a tooltip with its value (the arrow keys + * step through the days), or the same days as a table. Drawn at the width it + * is given, so its text keeps its size. */ export function TrafficChart({ - points, - stepSeconds, + days, metric, view, }: { - points: TrafficPoint[] - stepSeconds: number + days: TrafficDay[] metric: TrafficMetric view: 'chart' | 'table' }) { if (view === 'table') { - return + return } - return + return } -function AreaChart({ - points, - stepSeconds, - metric, -}: { - points: TrafficPoint[] - stepSeconds: number - metric: TrafficMetric -}) { +function AreaChart({ days, metric }: { days: TrafficDay[]; metric: TrafficMetric }) { const box = useRef(null) const [width, setWidth] = useState(720) const [active, setActive] = useState(null) @@ -112,53 +97,44 @@ function AreaChart({ return () => observer.disconnect() }, []) - const values = points.map((p) => p[metric]) + const values = days.map((d) => d[metric]) const max = Math.max(0, ...values) const ticks = niceTicks(max) const top = ticks[ticks.length - 1] || 1 const plotW = Math.max(width - MARGIN.left - MARGIN.right, 1) const plotH = HEIGHT - MARGIN.top - MARGIN.bottom const x = (i: number) => - MARGIN.left + (points.length <= 1 ? plotW / 2 : (i / (points.length - 1)) * plotW) + MARGIN.left + (days.length <= 1 ? plotW / 2 : (i / (days.length - 1)) * plotW) const y = (v: number) => MARGIN.top + plotH - (v / top) * plotH - const line = points.map((_, i) => `${i === 0 ? 'M' : 'L'}${x(i)},${y(values[i])}`).join('') + const line = days.map((_, i) => `${i === 0 ? 'M' : 'L'}${x(i)},${y(values[i])}`).join('') const area = - points.length > 0 - ? `${line}L${x(points.length - 1)},${y(0)}L${x(0)},${y(0)}Z` - : '' - const multiDay = - points.length > 1 && - new Date(points[0].start).toDateString() !== - new Date(points[points.length - 1].start).toDateString() - // Room for a time label: about 6.5px a character, and a gap. - const withDate = multiDay && stepSeconds < 86_400 - const labelWidth = (points[0] ? timeLabel(points[0].start, stepSeconds, withDate).length : 5) * 6.5 + 24 - const perPoint = points.length <= 1 ? plotW : plotW / (points.length - 1) - // Labels on round times (every 3 hours, every day), as far apart as they need. - const wanted = Math.max(1, Math.ceil(labelWidth / perPoint)) * stepSeconds - const interval = - LABEL_INTERVALS.find((s) => s >= wanted && s % stepSeconds === 0) ?? wanted - const labelled = (iso: string) => (Date.parse(iso) / 1000) % interval === 0 + days.length > 0 ? `${line}L${x(days.length - 1)},${y(0)}L${x(0)},${y(0)}Z` : '' + // Room for a label: about 6.5px a character, and a gap; counted back from + // the last day, so today always has one. + const perDay = days.length <= 1 ? plotW : plotW / (days.length - 1) + const wanted = Math.ceil((shortDay(days[0]?.day ?? '').length * 6.5 + 24) / perDay) + const every = LABEL_EVERY.find((n) => n >= wanted) ?? wanted + const labelled = (i: number) => (days.length - 1 - i) % every === 0 const total = values.reduce((a, b) => a + b, 0) function nearest(e: PointerEvent): number | null { - if (points.length === 0) return null + if (days.length === 0) return null const rect = e.currentTarget.getBoundingClientRect() const px = e.clientX - rect.left - MARGIN.left - const i = points.length <= 1 ? 0 : Math.round((px / plotW) * (points.length - 1)) - return Math.min(points.length - 1, Math.max(0, i)) + const i = days.length <= 1 ? 0 : Math.round((px / plotW) * (days.length - 1)) + return Math.min(days.length - 1, Math.max(0, i)) } function onKey(e: KeyboardEvent) { - if (points.length === 0) return + if (days.length === 0) return const step = { ArrowLeft: -1, ArrowRight: 1, Home: -Infinity, End: Infinity }[e.key] if (step === undefined) return e.preventDefault() - const from = active ?? (step > 0 ? -1 : points.length) - setActive(Math.min(points.length - 1, Math.max(0, from + step))) + const from = active ?? (step > 0 ? -1 : days.length) + setActive(Math.min(days.length - 1, Math.max(0, from + step))) } - const tip = active !== null ? points[active] : null + const tip = active !== null ? days[active] : null const tipLeft = active !== null ? Math.min(Math.max(x(active), 80), width - 80) : 0 return ( @@ -168,7 +144,7 @@ function AreaChart({ height={HEIGHT} role="img" tabIndex={0} - aria-label={`${METRIC_NAMES[metric]} over time: ${formatMetric(metric, total)} in all, at most ${formatMetric(metric, max)} per point. Use the arrow keys to read each point.`} + aria-label={`${METRIC_NAMES[metric]} by day: ${formatMetric(metric, total)} in all, at most ${formatMetric(metric, max)} a day. Use the arrow keys to read each day.`} aria-describedby={tip ? tipId : undefined} onPointerMove={(e) => setActive(nearest(e))} onPointerLeave={() => setActive(null)} @@ -183,10 +159,10 @@ function AreaChart({ ))} - {points.map((p, i) => - labelled(p.start) ? ( - - {timeLabel(p.start, stepSeconds, withDate)} + {days.map((d, i) => + labelled(i) ? ( + + {shortDay(d.day)} ) : null, )} @@ -206,7 +182,7 @@ function AreaChart({ {formatMetric(metric, values[active])}
- {METRIC_NAMES[metric]}, {timeLabel(tip.start, stepSeconds, true)} + {METRIC_NAMES[metric]}, {dayLabel(tip.day)}
)} @@ -214,33 +190,23 @@ function AreaChart({ ) } -/** The points of the chart as rows, newest first. */ -function PointTable({ - points, - stepSeconds, - metric, -}: { - points: TrafficPoint[] - stepSeconds: number - metric: TrafficMetric -}) { +/** The days of the chart as rows, newest first. */ +function DayTable({ days, metric }: { days: TrafficDay[]; metric: TrafficMetric }) { return ( - +
- From + Day Requests Data sent - 4xx - 5xx + Clients - {[...points].reverse().map((p) => ( - - {timeLabel(p.start, stepSeconds, true)} - {full.format(p.requests)} - {formatBytes(p.bytes)} - {full.format(p.client_errors)} - {full.format(p.server_errors)} + {[...days].reverse().map((d) => ( + + {dayLabel(d.day)} + {full.format(d.requests)} + {formatBytes(d.bytes)} + {full.format(d.clients)} ))} diff --git a/web/src/lib/traffic.ts b/web/src/lib/traffic.ts index 8d15194d..7c7d4686 100644 --- a/web/src/lib/traffic.ts +++ b/web/src/lib/traffic.ts @@ -1,62 +1,35 @@ -import type { Tone } from '~/components/ui' - /** - * How the traffic views name what the statistics count: surfaces, accounts, - * statuses, instances. + * How the traffic views name what the statistics count: countries, + * autonomous systems, repositories, shares. */ -const SURFACES: Record = { - oci: 'Docker registry', - nuget: 'NuGet', - plugin_npm: 'npm', - plugin_raw: 'Raw', - plugin_other: 'Other formats', - repository: 'Unknown repositories', - admin: 'Admin API', - browse: 'Browse API', - auth: 'Sign-in and tokens', - ui: 'Web UI', -} - -/** A surface as people call it (`plugin_npm` → npm). */ -export function surfaceLabel(surface: string): string { - return SURFACES[surface] ?? surface -} - -const ACCOUNT_KINDS: Record = { - anonymous: 'anonymous', - session: 'web session', - token: 'API token', - password: 'password', - trusted_header: 'trusted proxy', - workload: 'workload identity', -} +const regions = (() => { + try { + return new Intl.DisplayNames(undefined, { type: 'region' }) + } catch { + return null + } +})() -/** Who a request ran as: `alice (API token)`, `anonymous`. */ -export function accountLabel( - account: string | null, - username: string | null, - userId: number | null, -): string { - const kind = account ? (ACCOUNT_KINDS[account] ?? account) : null - const who = username ?? (userId != null ? `user ${userId}` : null) - if (who && kind) return `${who} (${kind})` - return who ?? kind ?? '—' +/** A country's name from its ISO code (`DE` → Germany); `Unknown` for none. */ +export function countryName(code: string): string { + if (!code) return 'Unknown' + try { + return regions?.of(code) ?? code + } catch { + return code + } } -/** The tone of a status: 2xx ok, 3xx neutral, 4xx warn, 5xx danger. */ -export function statusTone(status: number): Tone { - if (status >= 500) return 'danger' - if (status >= 400) return 'warn' - if (status >= 300) return 'neutral' - return 'ok' +/** An autonomous system: `AS3320 Deutsche Telekom AG`; `Unknown` for none. */ +export function asnLabel(asn: number, org: string): string { + if (!asn) return 'Unknown' + return org ? `AS${asn} ${org}` : `AS${asn}` } -/** What the series' points are: "per minute", "per 5 minutes", "per day". */ -export function stepLabel(seconds: number): string { - if (seconds % 86_400 === 0) return seconds === 86_400 ? 'per day' : `per ${seconds / 86_400} days` - if (seconds % 3_600 === 0) return seconds === 3_600 ? 'per hour' : `per ${seconds / 3_600} hours` - return seconds === 60 ? 'per minute' : `per ${seconds / 60} minutes` +/** A repository's name; what `""` stands for: requests to no repository. */ +export function repositoryLabel(name: string): string { + return name || 'No repository (web UI, APIs, sign-in)' } const percent = new Intl.NumberFormat(undefined, { style: 'percent', maximumFractionDigits: 1 }) @@ -66,21 +39,11 @@ export function share(part: number, whole: number): string { return whole > 0 ? percent.format(part / whole) : '—' } -function bound(ms: number): string { - return ms >= 1_000 ? `${ms / 1_000} s` : `${ms} ms` -} - -/** - * How fast the requests were answered, from the latency histogram's bounds: - * `95 % answered within 250 ms, half within 10 ms`; `null` without requests. - */ -export function latencySummary(totals: { - requests: number - p50_ms: number | null - p95_ms: number | null -}): string | null { - if (totals.requests === 0) return null - const p95 = totals.p95_ms == null ? '5 % took over 60 s' : `95 % answered within ${bound(totals.p95_ms)}` - const p50 = totals.p50_ms == null ? 'half took over 60 s' : `half within ${bound(totals.p50_ms)}` - return `${p95}, ${p50}` +/** A `YYYY-MM-DD` day as the locale writes a date (`Oct 30, 2026`). */ +export function dayLabel(day: string | null | undefined): string { + if (!day) return '—' + const d = new Date(`${day}T00:00:00Z`) + return Number.isNaN(d.getTime()) + ? day + : d.toLocaleDateString(undefined, { dateStyle: 'medium', timeZone: 'UTC' }) } diff --git a/web/src/queries/useTraffic.ts b/web/src/queries/useTraffic.ts index 824d970c..80fa3744 100644 --- a/web/src/queries/useTraffic.ts +++ b/web/src/queries/useTraffic.ts @@ -2,24 +2,27 @@ import { keepPreviousData, useMutation, useQuery, useQueryClient } from '@tansta import type { ApiError } from '~/lib/httpClient' import { - fetchTrafficClient, + downloadGeoip, + fetchGeoip, fetchTrafficOverview, fetchTrafficSettings, + removeGeoip, trafficKeys, + updateGeoip, updateTrafficSettings, - type TrafficClientDetail, + type GeoipStatus, type TrafficOverview, type TrafficQuery, type TrafficSettings, type TrafficSettingsUpdate, } from '~/api/traffic' -/** How often an open traffic view is read again: the files change every minute. */ +/** How often an open traffic view is read again: today's files change every minute. */ const REFRESH_MS = 60_000 /** - * The traffic of a range (`GET /admin/traffic`). While another range or - * instance loads, the previous answer stays on screen. + * The requests of a range (`GET /admin/traffic`). While another range or + * filter loads, the previous answer stays on screen. */ export function useTrafficOverview(q: TrafficQuery) { return useQuery({ @@ -30,17 +33,6 @@ export function useTrafficOverview(q: TrafficQuery) { }) } -/** One client network's traffic and request log (`GET /admin/traffic/client`). */ -export function useTrafficClient(network: string, q: TrafficQuery) { - return useQuery({ - queryKey: trafficKeys.client(network, q), - queryFn: ({ signal }) => fetchTrafficClient(network, q, signal), - placeholderData: keepPreviousData, - refetchInterval: REFRESH_MS, - enabled: network.length > 0, - }) -} - /** What is recorded and how long it is kept (`GET /admin/settings/traffic`). */ export function useTrafficSettings() { return useQuery({ @@ -63,3 +55,40 @@ export function useUpdateTrafficSettings() { }, }) } + +/** The geolocation databases and the MaxMind download (`GET /admin/traffic/geoip`). */ +export function useGeoip() { + return useQuery({ + queryKey: trafficKeys.geoip(), + queryFn: ({ signal }) => fetchGeoip(signal), + }) +} + +/** Answers that replace the geolocation status, and make the traffic views be read again. */ +function useGeoipMutation(fn: (v: V) => Promise) { + const queryClient = useQueryClient() + return useMutation({ + mutationFn: fn, + onSuccess: (status) => { + if (status) queryClient.setQueryData(trafficKeys.geoip(), status) + void queryClient.invalidateQueries({ queryKey: trafficKeys.all }) + }, + }) +} + +/** Set the MaxMind account ID and license key. */ +export function useUpdateGeoip() { + return useGeoipMutation((body: { account_id: string; license_key?: string }) => + updateGeoip(body), + ) +} + +/** Download the databases now. */ +export function useDownloadGeoip() { + return useGeoipMutation((_: void) => downloadGeoip()) +} + +/** Forget the MaxMind account and delete the databases. */ +export function useRemoveGeoip() { + return useGeoipMutation((_: void) => removeGeoip()) +} diff --git a/web/src/routeTree.gen.ts b/web/src/routeTree.gen.ts index 4c7f49fd..c9cfcb1f 100644 --- a/web/src/routeTree.gen.ts +++ b/web/src/routeTree.gen.ts @@ -39,7 +39,6 @@ import { Route as adminServiceAccountsEditRouteImport } from './routes/admin/ser import { Route as adminServiceAccountsCreateRouteImport } from './routes/admin/service-accounts/Create' import { Route as adminSettingsEmailRouteImport } from './routes/admin/settings/Email' import { Route as adminSettingsRetentionSettingsRouteImport } from './routes/admin/settings/RetentionSettings' -import { Route as adminTrafficClientRouteImport } from './routes/admin/traffic/Client' import { Route as adminTrafficSettingsRouteImport } from './routes/admin/traffic/Settings' import { Route as adminUsersEditRouteImport } from './routes/admin/users/Edit' import { Route as adminUsersCreateRouteImport } from './routes/admin/users/Create' @@ -201,11 +200,6 @@ const adminSettingsRetentionSettingsRoute = path: '/settings/retention', getParentRoute: () => AdminLayoutRoute, } as any) -const adminTrafficClientRoute = adminTrafficClientRouteImport.update({ - id: '/traffic/client', - path: '/traffic/client', - getParentRoute: () => AdminLayoutRoute, -} as any) const adminTrafficSettingsRoute = adminTrafficSettingsRouteImport.update({ id: '/traffic/settings', path: '/traffic/settings', @@ -274,7 +268,6 @@ export interface FileRoutesByFullPath { '/manage/service-accounts/new': typeof adminServiceAccountsCreateRoute '/manage/settings/email': typeof adminSettingsEmailRoute '/manage/settings/retention': typeof adminSettingsRetentionSettingsRoute - '/manage/traffic/client': typeof adminTrafficClientRoute '/manage/traffic/settings': typeof adminTrafficSettingsRoute '/manage/users/$userId': typeof adminUsersEditRoute '/manage/users/new': typeof adminUsersCreateRoute @@ -312,7 +305,6 @@ export interface FileRoutesByTo { '/manage/service-accounts/new': typeof adminServiceAccountsCreateRoute '/manage/settings/email': typeof adminSettingsEmailRoute '/manage/settings/retention': typeof adminSettingsRetentionSettingsRoute - '/manage/traffic/client': typeof adminTrafficClientRoute '/manage/traffic/settings': typeof adminTrafficSettingsRoute '/manage/users/$userId': typeof adminUsersEditRoute '/manage/users/new': typeof adminUsersCreateRoute @@ -353,7 +345,6 @@ export interface FileRoutesById { '/_app/manage/service-accounts/new': typeof adminServiceAccountsCreateRoute '/_app/manage/settings/email': typeof adminSettingsEmailRoute '/_app/manage/settings/retention': typeof adminSettingsRetentionSettingsRoute - '/_app/manage/traffic/client': typeof adminTrafficClientRoute '/_app/manage/traffic/settings': typeof adminTrafficSettingsRoute '/_app/manage/users/$userId': typeof adminUsersEditRoute '/_app/manage/users/new': typeof adminUsersCreateRoute @@ -394,7 +385,6 @@ export interface FileRouteTypes { | '/manage/service-accounts/new' | '/manage/settings/email' | '/manage/settings/retention' - | '/manage/traffic/client' | '/manage/traffic/settings' | '/manage/users/$userId' | '/manage/users/new' @@ -432,7 +422,6 @@ export interface FileRouteTypes { | '/manage/service-accounts/new' | '/manage/settings/email' | '/manage/settings/retention' - | '/manage/traffic/client' | '/manage/traffic/settings' | '/manage/users/$userId' | '/manage/users/new' @@ -472,7 +461,6 @@ export interface FileRouteTypes { | '/_app/manage/service-accounts/new' | '/_app/manage/settings/email' | '/_app/manage/settings/retention' - | '/_app/manage/traffic/client' | '/_app/manage/traffic/settings' | '/_app/manage/users/$userId' | '/_app/manage/users/new' @@ -700,13 +688,6 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof adminSettingsRetentionSettingsRouteImport parentRoute: typeof AdminLayoutRoute } - '/_app/manage/traffic/client': { - id: '/_app/manage/traffic/client' - path: '/traffic/client' - fullPath: '/manage/traffic/client' - preLoaderRoute: typeof adminTrafficClientRouteImport - parentRoute: typeof AdminLayoutRoute - } '/_app/manage/traffic/settings': { id: '/_app/manage/traffic/settings' path: '/traffic/settings' @@ -781,7 +762,6 @@ interface AdminLayoutRouteChildren { adminServiceAccountsCreateRoute: typeof adminServiceAccountsCreateRoute adminSettingsEmailRoute: typeof adminSettingsEmailRoute adminSettingsRetentionSettingsRoute: typeof adminSettingsRetentionSettingsRoute - adminTrafficClientRoute: typeof adminTrafficClientRoute adminTrafficSettingsRoute: typeof adminTrafficSettingsRoute adminUsersEditRoute: typeof adminUsersEditRoute adminUsersCreateRoute: typeof adminUsersCreateRoute @@ -812,7 +792,6 @@ const AdminLayoutRouteChildren: AdminLayoutRouteChildren = { adminServiceAccountsCreateRoute: adminServiceAccountsCreateRoute, adminSettingsEmailRoute: adminSettingsEmailRoute, adminSettingsRetentionSettingsRoute: adminSettingsRetentionSettingsRoute, - adminTrafficClientRoute: adminTrafficClientRoute, adminTrafficSettingsRoute: adminTrafficSettingsRoute, adminUsersEditRoute: adminUsersEditRoute, adminUsersCreateRoute: adminUsersCreateRoute, diff --git a/web/src/routes.ts b/web/src/routes.ts index cb6bf333..116d860b 100644 --- a/web/src/routes.ts +++ b/web/src/routes.ts @@ -25,10 +25,10 @@ import { index, layout, rootRoute, route } from '@tanstack/virtual-file-routes' * web login: `npm login`) * /manage -> AdminLayout.tsx (beforeLoad: require is_admin; * /manage/traffic: every instance's - * traffic, /traffic/client: one - * client's, with its request log, - * /traffic/settings: what is - * recorded and kept) + * requests by day, repository and + * client; /traffic/settings: what + * is recorded and kept, and the + * geolocation databases) * * The admin screens live under `/manage/*`, NOT `/admin/*`: `/admin/*` is the * admin REST API, which the server routes before the SPA fallback, so a UI @@ -78,7 +78,6 @@ export const routes = rootRoute('root.tsx', [ route('/settings/email', 'admin/settings/Email.tsx'), route('/settings/retention', 'admin/settings/RetentionSettings.tsx'), route('/traffic', 'admin/traffic/Traffic.tsx'), - route('/traffic/client', 'admin/traffic/Client.tsx'), route('/traffic/settings', 'admin/traffic/Settings.tsx'), route('/system', 'admin/system/System.tsx'), ]), diff --git a/web/src/routes/admin/traffic/Client.tsx b/web/src/routes/admin/traffic/Client.tsx deleted file mode 100644 index 4005b3f2..00000000 --- a/web/src/routes/admin/traffic/Client.tsx +++ /dev/null @@ -1,271 +0,0 @@ -import { createFileRoute, useNavigate } from '@tanstack/react-router' - -import { - isTrafficRange, - type TrafficClientDetail, - type TrafficRange, - type TrafficTally, -} from '~/api/traffic' -import { formatBytes, formatDateTime } from '~/lib/oci' -import { accountLabel, share, stepLabel, statusTone, surfaceLabel } from '~/lib/traffic' -import { useTrafficClient } from '~/queries/useTraffic' -import { TrafficChart } from '~/components/TrafficChart' -import { TrafficFilters } from '~/components/TrafficFilters' -import { ErrorState } from '~/components/ErrorState' -import { - EmptyState, - Notice, - Page, - PageHeader, - Panel, - Skeleton, - Stat, - StatRow, - StatusBadge, - Table, - TableBody, - TableCell, - TableHead, - TableHeader, - TableRow, -} from '~/components/ui' - -/** The screen's URL state: the client network, the range and an instance. */ -export interface TrafficClientSearch { - network: string - range?: TrafficRange - instance?: string -} - -export const Route = createFileRoute('/_app/manage/traffic/client')({ - validateSearch: (search: Record): TrafficClientSearch => ({ - network: typeof search.network === 'string' ? search.network : '', - range: isTrafficRange(search.range) ? search.range : undefined, - instance: - typeof search.instance === 'string' && search.instance.length > 0 - ? search.instance - : undefined, - }), - component: TrafficClientScreen, -}) - -const count = new Intl.NumberFormat() - -/** - * One client network's traffic (`/admin/traffic/client`): its requests over - * time, from the time buckets it was among the top clients of, and its - * requests in the request log — what it asked for, as whom, with which - * client, and how each was answered. - */ -function TrafficClientScreen() { - const search = Route.useSearch() - const navigate = useNavigate({ from: Route.fullPath }) - const range = search.range ?? '24h' - const detail = useTrafficClient(search.network, { range, instance: search.instance }) - const set = (next: Partial) => - void navigate({ search: (prev) => ({ ...prev, ...next }) }) - - return ( - - - {!search.network ? ( - - Open a client from the top clients of the traffic screen. - - ) : ( - <> - set({ range: r })} - instance={search.instance} - instances={detail.data?.instances ?? []} - onInstance={(i) => set({ instance: i })} - /> - {detail.isError ? ( - void detail.refetch()} - /> - ) : !detail.data ? ( - - ) : ( -
- -
- )} - - )} - - ) -} - -function Body({ data }: { data: TrafficClientDetail }) { - const totals = data.totals - const log = data.log - return ( - <> - {!data.recording.client_addresses && ( - - Client addresses are not recorded now: the traffic settings switch - them off. What was recorded before stays until it expires. - - )} - - - - - - - - - - {!totals ? ( - - ) : ( - - )} - - - log.entries.length ? `, the newest ${log.entries.length} shown` : '' - }.`} - flush - > - {!data.recording.request_log && ( - - The request log is switched off in the traffic settings: new - requests are not logged. - - )} - {log.entries.length === 0 ? ( - - ) : ( - - )} - - - {log.matched > 0 && ( -
- - - ({ - value: accountLabel(a.account, a.username, a.user_id), - requests: a.requests, - }))} - total={log.matched} - /> - -
- )} - - ) -} - -function LogTable({ data }: { data: TrafficClientDetail }) { - return ( -
- - Time - Request - Status - Data sent - Took - Account - User agent - Instance - - - {data.log.entries.map((e, i) => ( - - {new Date(e.time).toLocaleString()} - - - {e.method} {e.path} - -
- {e.repository ?? surfaceLabel(e.surface)} - {e.address && e.address !== data.network ? ` · ${e.address}` : ''} -
- - - {String(e.status)} - - {formatBytes(e.bytes)} - - {e.duration_ms < 10 ? e.duration_ms.toFixed(1) : Math.round(e.duration_ms)} ms - - {accountLabel(e.account, e.username, e.user_id)} - - {e.user_agent ?? '—'} - - {e.host} - - ))} - -
- ) -} - -function TallyPanel({ - title, - tallies, - total, - mono = false, -}: { - title: string - tallies: TrafficTally[] - total: number - mono?: boolean -}) { - return ( - - - - {title.replace(/s$/, '')} - Requests - Share - - - {tallies.map((t) => ( - - {mono ? {t.value} : t.value} - {count.format(t.requests)} - {share(t.requests, total)} - - ))} - -
- - ) -} diff --git a/web/src/routes/admin/traffic/Settings.tsx b/web/src/routes/admin/traffic/Settings.tsx index ce72986e..26870f64 100644 --- a/web/src/routes/admin/traffic/Settings.tsx +++ b/web/src/routes/admin/traffic/Settings.tsx @@ -1,11 +1,22 @@ import { useState, type FormEvent } from 'react' import { createFileRoute } from '@tanstack/react-router' -import type { TrafficSettings } from '~/api/traffic' -import { useTrafficSettings, useUpdateTrafficSettings } from '~/queries/useTraffic' +import type { GeoDatabase, GeoipStatus, TrafficSettings } from '~/api/traffic' +import { formatBytes, formatDateTime } from '~/lib/oci' +import { + useDownloadGeoip, + useGeoip, + useRemoveGeoip, + useTrafficSettings, + useUpdateGeoip, + useUpdateTrafficSettings, +} from '~/queries/useTraffic' import { Button, - Checkbox, + Code, + Dialog, + FactList, + FormActions, Notice, Page, PageHeader, @@ -21,17 +32,17 @@ export const Route = createFileRoute('/_app/manage/traffic/settings')({ }) const DESCRIPTION = - 'What the traffic statistics record and how long they are kept, for every instance at once. Each instance follows a change within a minute.' + 'What the traffic statistics record, how long they are kept, and where the countries and networks of the clients come from, for every instance at once. Each instance follows a change within a minute.' /** The longest retention the server accepts, in days. */ const MAX_DAYS = 36_500 /** - * Traffic settings (`/admin/settings/traffic`): what is recorded (requests - * at all, client addresses, the request log) and how long each part is kept. - * Each is a form panel that saves its own fields (`PUT` changes only what - * it names), so saving one never undoes a change another administrator made - * to the other. + * Traffic settings (`/admin/settings/traffic`, `/admin/traffic/geoip`): + * what is recorded and how long it is kept, and the MaxMind account the + * geolocation databases are downloaded with. Each panel saves its own + * fields (`PUT` changes only what it names), so saving one never undoes a + * change another administrator made to the other. */ function TrafficSettingsScreen() { const settings = useTrafficSettings() @@ -55,31 +66,48 @@ function TrafficSettingsScreen() { ) : ( <> - + )} ) } -/** What is recorded. */ +/** A whole number of days, or the error to show. */ +function days(value: string): number | string { + const n = value.trim() + if (!/^\d+$/.test(n) || Number(n) < 1 || Number(n) > MAX_DAYS) { + return `Enter a whole number of days from 1 to ${MAX_DAYS.toLocaleString()}.` + } + return Number(n) +} + +/** What is recorded and how long it is kept. */ function RecordingForm({ settings }: { settings: TrafficSettings }) { const update = useUpdateTrafficSettings() const [enabled, setEnabled] = useState(settings.enabled) const [addresses, setAddresses] = useState(settings.client_addresses) - const [log, setLog] = useState(settings.request_log) + const [retention, setRetention] = useState(String(settings.retention_days)) + const [error, setError] = useState(null) function handleSubmit(e: FormEvent) { e.preventDefault() - update.mutate({ enabled, client_addresses: addresses, request_log: log }) + const kept = days(retention) + if (typeof kept === 'string') { + setError(kept) + return + } + setError(null) + update.mutate({ enabled, client_addresses: addresses, retention_days: kept }) } return ( {update.isPending ? 'Saving…' : 'Save recording'} @@ -97,13 +125,16 @@ function RecordingForm({ settings }: { settings: TrafficSettings }) { label="Client addresses" checked={addresses} onChange={setAddresses} - description="The top clients and the addresses in the request log. Addresses are personal data: they are kept apart from the counts, for as long as the retention below says." + description="Addresses are personal data. Off, requests are still counted by repository, country and network: the address is looked up in memory and dropped." /> - setRetention(e.target.value)} + error={error} + description="Each instance deletes older days about once an hour." /> {update.isError && ( {update.error.body || update.error.message} @@ -114,96 +145,168 @@ function RecordingForm({ settings }: { settings: TrafficSettings }) { ) } -/** A whole number of days, or the error to show. */ -function days(value: string, min: number): number | string { - const n = value.trim() - if (!/^\d+$/.test(n) || Number(n) < min || Number(n) > MAX_DAYS) { - return `Enter a whole number of days from ${min} to ${MAX_DAYS.toLocaleString()}.` - } - return Number(n) +function database(db: GeoDatabase | null): string { + if (!db) return 'Not loaded' + const built = db.built_at ? `, built ${formatDateTime(db.built_at)}` : '' + return `${db.database_type}${built} (${formatBytes(db.size)})` } -/** How long each part is kept. */ -function RetentionForm({ settings }: { settings: TrafficSettings }) { - const update = useUpdateTrafficSettings() - const keepsAll = settings.statistics_retention_days === 0 - const [keepAll, setKeepAll] = useState(keepsAll) - const [statistics, setStatistics] = useState(String(keepsAll ? 400 : settings.statistics_retention_days)) - const [clients, setClients] = useState(String(settings.client_retention_days)) - const [log, setLog] = useState(String(settings.request_log_retention_days)) - const [errors, setErrors] = useState>({}) +/** The geolocation databases and the MaxMind account they come from. */ +function GeolocationPanel() { + const status = useGeoip() + return ( + + The country and autonomous system (the provider or company) of each + client come from MaxMind's free GeoLite2 databases. With a + MaxMind account ID and license key, Forge downloads them every three + days and keeps them in its bucket for every instance. Without + internet access, put the .mmdb files there yourself as{' '} + traffic/v1/geoip/country.mmdb and{' '} + traffic/v1/geoip/asn.mmdb. + + } + > + {status.isError ? ( + void status.refetch()} + /> + ) : !status.data ? ( + + ) : ( + + )} + + ) +} + +function GeolocationBody({ status }: { status: GeoipStatus }) { + const update = useUpdateGeoip() + const download = useDownloadGeoip() + const remove = useRemoveGeoip() + const [accountId, setAccountId] = useState(status.maxmind.account_id ?? '') + const [licenseKey, setLicenseKey] = useState('') + const [errors, setErrors] = useState<{ account?: string; key?: string }>({}) + const [confirming, setConfirming] = useState(false) + const m = status.maxmind + const configured = m.account_id !== null && m.license_key_set function handleSubmit(e: FormEvent) { e.preventDefault() - const parsed = { - statistics: keepAll ? 0 : days(statistics, 1), - clients: days(clients, 1), - log: days(log, 1), - } - const found = Object.fromEntries( - Object.entries(parsed).filter((entry): entry is [string, string] => typeof entry[1] === 'string'), - ) + const found: { account?: string; key?: string } = {} + if (!/^\d+$/.test(accountId.trim())) found.account = 'Enter your MaxMind account ID, a number.' + if (!licenseKey.trim() && !m.license_key_set) found.key = 'Enter a license key.' setErrors(found) if (Object.keys(found).length > 0) return - update.mutate({ - statistics_retention_days: parsed.statistics as number, - client_retention_days: parsed.clients as number, - request_log_retention_days: parsed.log as number, - }) + update.mutate( + { + account_id: accountId.trim(), + ...(licenseKey.trim() ? { license_key: licenseKey.trim() } : {}), + }, + { onSuccess: () => setLicenseKey('') }, + ) } return ( - - {update.isPending ? 'Saving…' : 'Save retention'} - - } - > -
- - {!keepAll && ( +
+ + {m.last_error && ( + + {m.last_error} + + )} +
+
setStatistics(e.target.value)} - error={errors.statistics} + value={accountId} + onChange={(e) => setAccountId(e.target.value)} + error={errors.account} /> - )} - setClients(e.target.value)} - error={errors.clients} - /> - setLog(e.target.value)} - error={errors.log} - description="The largest part: a file per instance and minute." - /> + setLicenseKey(e.target.value)} + error={errors.key} + placeholder={m.license_key_set ? '••••••••' : undefined} + description={m.license_key_set ? 'Stored. Enter a new one to replace it.' : undefined} + /> +
+

+ Create a free account and a license key at{' '} + + maxmind.com + + . The key is stored encrypted. Forge downloads {m.editions.join(' and ')} from{' '} + {m.url}. +

{update.isError && ( {update.error.body || update.error.message} )} - {update.isSuccess && Saved.} -
- + {download.isError && ( + {download.error.body || download.error.message} + )} + {download.isSuccess && !m.last_error && ( + Downloaded: every instance uses the databases within ten minutes. + )} + + {configured && ( + + )} + {configured && ( + + )} + + + + setConfirming(false)} + icon="Globe" + tone="danger" + title="Turn geolocation off?" + description="Forge forgets the MaxMind account and deletes the databases from its bucket. Requests are no longer located; the countries already counted stay." + confirmLabel="Turn off" + onConfirm={() => { + remove.mutate(undefined, { + onSuccess: () => { + setAccountId('') + setLicenseKey('') + }, + }) + setConfirming(false) + }} + /> +
) } diff --git a/web/src/routes/admin/traffic/Traffic.tsx b/web/src/routes/admin/traffic/Traffic.tsx index 559bae6a..91420511 100644 --- a/web/src/routes/admin/traffic/Traffic.tsx +++ b/web/src/routes/admin/traffic/Traffic.tsx @@ -1,16 +1,15 @@ -import { useState } from 'react' +import { useState, type ReactNode } from 'react' import { createFileRoute, Link, useNavigate } from '@tanstack/react-router' -import { isTrafficRange, type TrafficOverview, type TrafficRange } from '~/api/traffic' -import { formatBytes, formatDateTime } from '~/lib/oci' -import { accountLabel, latencySummary, share, stepLabel, surfaceLabel } from '~/lib/traffic' +import { isTrafficRange, NONE, type TrafficOverview, type TrafficRange } from '~/api/traffic' +import { formatBytes } from '~/lib/oci' +import { asnLabel, countryName, dayLabel, repositoryLabel, share } from '~/lib/traffic' import { useTrafficOverview } from '~/queries/useTraffic' import { TRAFFIC_METRICS, TrafficChart, type TrafficMetric } from '~/components/TrafficChart' import { TrafficFilters } from '~/components/TrafficFilters' import { ErrorState } from '~/components/ErrorState' import { Button, - Chip, EmptyState, Notice, Page, @@ -28,11 +27,17 @@ import { TableRow, } from '~/components/ui' -/** The screen's URL state: the range, and an instance or repository to narrow to. */ +/** The screen's URL state: the range, an instance, and what to narrow to. */ export interface TrafficSearch { range?: TrafficRange instance?: string + /** `-` for the requests to no repository. */ repository?: string + network?: string + /** ISO code; `-` for unknown. */ + country?: string + /** `0` for unknown. */ + asn?: string } function text(value: unknown): string | undefined { @@ -44,27 +49,37 @@ export const Route = createFileRoute('/_app/manage/traffic')({ range: isTrafficRange(search.range) ? search.range : undefined, instance: text(search.instance), repository: text(search.repository), + network: text(search.network), + country: text(search.country), + asn: /^\d+$/.test(String(search.asn ?? '')) ? String(search.asn) : undefined, }), component: TrafficScreen, }) const count = new Intl.NumberFormat() +/** The filters a list row sets: what it is, in the URL's words. */ +type Narrow = Pick + /** - * The traffic screen (`/admin/traffic`): what every instance answered in a - * range — requests over time, their totals and latency, the repositories, - * surfaces and statuses, and the clients with the most requests, each - * opening its own view with its request log. The range, the instance and - * the repository are in the URL, so a view can be shared. + * The traffic screen (`/admin/traffic`): how many requests each client + * network made to each repository, by day, and where they came from + * (countries and autonomous systems, with a geolocation database). Every + * list narrows the view: open a repository to see where its requests came + * from, a country or a network to see what it asked for. The range, the + * instance and the filters are in the URL, so a view can be shared. */ function TrafficScreen() { const search = Route.useSearch() const navigate = useNavigate({ from: Route.fullPath }) - const range = search.range ?? '24h' + const range = search.range ?? '30d' const overview = useTrafficOverview({ range, instance: search.instance, repository: search.repository, + network: search.network, + country: search.country, + asn: search.asn, }) const set = (next: Partial) => void navigate({ search: (prev) => ({ ...prev, ...next }) }) @@ -74,7 +89,7 @@ function TrafficScreen() { Settings @@ -88,14 +103,7 @@ function TrafficScreen() { instances={overview.data?.instances ?? []} onInstance={(i) => set({ instance: i })} /> - {search.repository && ( -

- Only {search.repository}{' '} - -

- )} + {overview.isError ? ( ) : (
- +
)} ) } -function Body({ data, repository }: { data: TrafficOverview; repository?: string }) { +/** The filters in force, each with a button that removes it. */ +function ActiveFilters({ + search, + onClear, +}: { + search: TrafficSearch + onClear: (next: Partial) => void +}) { + const active: { key: keyof Narrow; label: string }[] = [] + if (search.repository) { + active.push({ + key: 'repository', + label: `Repository: ${search.repository === NONE ? repositoryLabel('') : search.repository}`, + }) + } + if (search.country) { + active.push({ + key: 'country', + label: `Country: ${countryName(search.country === NONE ? '' : search.country)}`, + }) + } + if (search.asn) { + active.push({ key: 'asn', label: `Network: ${search.asn === '0' ? 'Unknown' : `AS${search.asn}`}` }) + } + if (search.network) active.push({ key: 'network', label: `Client: ${search.network}` }) + if (active.length === 0) return null + return ( +
+ {active.map((f) => ( + + ))} + {active.length > 1 && ( + + )} +
+ ) +} + +/** A link that narrows the view to `narrow`, keeping the range and instance. */ +function NarrowLink({ narrow, children, mono }: { narrow: Narrow; children: ReactNode; mono?: boolean }) { + return ( + ({ ...prev, ...narrow })} + className={mono ? 'fg-link fg-mono' : 'fg-link'} + > + {children} + + ) +} + +function Body({ data }: { data: TrafficOverview }) { const [metric, setMetric] = useState('requests') const [view, setView] = useState<'chart' | 'table'>('chart') const { totals, recording } = data + const located = recording.countries || recording.asns || totals.countries > 0 || totals.asns > 0 return ( <> {!recording.enabled && ( @@ -127,30 +204,25 @@ function Body({ data, repository }: { data: TrafficOverview; repository?: string )} - + m.id === metric)?.label}, ${stepLabel(data.step_seconds)}, from ${formatDateTime(data.from)} to ${formatDateTime(data.to)}. The last minutes fill in as the instances write them.`} + title="By day" + description={`${TRAFFIC_METRICS.find((m) => m.id === metric)?.label}, from ${dayLabel(data.from)} to ${dayLabel(data.to)} (UTC). Today fills in as the instances write it, once a minute.`} actions={
@@ -172,161 +244,173 @@ function Body({ data, repository }: { data: TrafficOverview; repository?: string once a minute. ) : ( - - )} - - - - {!recording.client_addresses && ( - - Client addresses are not recorded: the traffic settings switch them off. - - )} - {data.clients.length === 0 ? ( - - ) : ( - - )} - {repository && ( -

- The top clients count every repository. -

+ )}
{data.repositories.length === 0 ? ( - + ) : ( Repository Requests Data sent - 4xx - 5xx + Clients {data.repositories.map((r) => ( - ({ ...prev, repository: r.name })} - className="fg-link" - > - {r.name} - + {repositoryLabel(r.name)} {count.format(r.requests)} {formatBytes(r.bytes)} - {count.format(r.client_errors)} - {count.format(r.server_errors)} + {count.format(r.clients)} + + ))} + +
+ )} + + c.country).length} + flush + footer={located ? : undefined} + > + {!located ? ( + + Set a MaxMind account in the{' '} + + traffic settings + {' '} + to see where the requests come from. + + ) : ( + + + Country + Requests + Share + Clients + + + {data.countries.map((c) => ( + + + {countryName(c.country)} + {c.country && {c.country}} + + {count.format(c.requests)} + {share(c.requests, totals.requests)} + {count.format(c.clients)} ))}
)} - - + + + {located && ( + } + > +
- Surface + Autonomous system Requests + Share Data sent + Clients - {data.surfaces.map((s) => ( - - {surfaceLabel(s.surface)} - {count.format(s.requests)} - {formatBytes(s.bytes)} + {data.asns.map((a) => ( + + + {asnLabel(a.asn, a.org)} + + {count.format(a.requests)} + {share(a.requests, totals.requests)} + {formatBytes(a.bytes)} + {count.format(a.clients)} ))}
- + + )} + + + {!recording.client_addresses && ( + + Client addresses are not recorded: the traffic settings switch them + off. Requests are still counted by repository and country. + + )} + {data.clients.length === 0 ? ( + + ) : ( +
- Status + Client Requests - Share + Data sent + {located && Country} + {located && Network} + Repositories + Days + Last seen - {data.statuses.map((s) => ( - - - {s.status} + {data.clients.map((c) => ( + + + + {c.network} + + {c.address && c.address !== c.network && ( +
{c.address}
+ )} - {count.format(s.requests)} - {share(s.requests, totals.requests)} + {count.format(c.requests)} + {formatBytes(c.bytes)} + {located && {countryName(c.country)}} + {located && {asnLabel(c.asn, c.as_org)}} + {c.repositories.map(repositoryLabel).join(', ')} + {count.format(c.days)} + {dayLabel(c.last_day)} ))}
- -
+ )} + ) } -function ClientTable({ data }: { data: TrafficOverview }) { +/** The credit MaxMind's GeoLite license asks for wherever its data is shown. */ +function Attribution() { return ( - - - Client - Requests - Data sent - 4xx - 5xx - Last account - Last user agent - Instances - Last seen - - - {data.clients.map((c) => ( - - - ({ - network: c.network, - range: prev.range, - instance: prev.instance, - })} - className="fg-link fg-mono" - > - {c.network} - - {c.address && c.address !== c.network && ( -
{c.address}
- )} - - {count.format(c.requests)} - {formatBytes(c.bytes)} - {count.format(c.client_errors)} - {count.format(c.server_errors)} - {accountLabel(c.account, c.username, c.user_id)} - - - {c.user_agent ? truncate(c.user_agent, 40) : '—'} - - - {c.hosts.join(', ')} - {formatDateTime(c.last_seen)} - - ))} - -
+

+ Countries and networks: GeoLite Data created by MaxMind, available from{' '} + + maxmind.com + + . +

) } - -function truncate(s: string, max: number): string { - return s.length > max ? `${s.slice(0, max - 1)}…` : s -} diff --git a/web/src/routes/admin/traffic/traffic.test.tsx b/web/src/routes/admin/traffic/traffic.test.tsx index cc181abe..99ec4167 100644 --- a/web/src/routes/admin/traffic/traffic.test.tsx +++ b/web/src/routes/admin/traffic/traffic.test.tsx @@ -4,16 +4,18 @@ import userEvent from '@testing-library/user-event' import { clearSession, setSession } from '~/lib/auth' import type { Me } from '~/api/me' -import type { TrafficClientDetail, TrafficOverview, TrafficSettings } from '~/api/traffic' +import type { GeoipStatus, TrafficOverview, TrafficSettings } from '~/api/traffic' import { renderApp } from '~/test/renderApp' /** - * The traffic screens (`/manage/traffic`, its client view and its settings), - * driven against the real route tree and query hooks: - * - the overview's totals, top clients and repositories, with the range, - * instance and repository sent as asked and kept in the URL; - * - a client's view with its request log, accounts named by user; - * - the settings, each panel saving only its own fields. + * The traffic screens (`/manage/traffic` and its settings), driven against + * the real route tree and query hooks: + * - the totals and the lists by repository, country, network and client; + * each row narrows the view, the filters are sent and kept in the URL and + * removed one by one; + * - without a geolocation database, the countries say how to get one; + * - the settings, the MaxMind account (the key never shown) and its + * download, and turning geolocation off after confirming. */ const adminMe: Me = { @@ -22,120 +24,63 @@ const adminMe: Me = { capabilities: ['repo.read', 'repo.admin'], } -const recording = { enabled: true, client_addresses: true, request_log: true } - -const point = (start: string, requests: number) => ({ - start, - requests, - bytes: requests * 1000, - client_errors: 1, - server_errors: 0, -}) +const recording = { enabled: true, client_addresses: true, countries: true, asns: true } const overview: TrafficOverview = { - from: '2026-10-30T13:00:00Z', - to: '2026-10-30T14:00:00Z', - step_seconds: 60, - resolution: 'minute', + from: '2026-10-24', + to: '2026-10-30', instances: [ { id: 'web-1_20261030T100000Z_00a1', host: 'web-1', started_at: '2026-10-30T10:00:00Z' }, { id: 'web-2_20261030T110000Z_00b2', host: 'web-2', started_at: '2026-10-30T11:00:00Z' }, ], - totals: { - requests: 1234, - bytes: 5 * 1024 ** 2, - client_errors: 40, - denied: 12, - not_found: 25, - throttled: 3, - server_errors: 2, - mean_ms: 18.25, - p50_ms: 10, - p95_ms: 250, - p99_ms: 1000, - }, - series: [point('2026-10-30T13:00:00Z', 600), point('2026-10-30T13:01:00Z', 634)], - surfaces: [ - { surface: 'plugin_npm', requests: 1000, bytes: 4 * 1024 ** 2 }, - { surface: 'oci', requests: 234, bytes: 1024 ** 2 }, + totals: { requests: 1234, bytes: 5 * 1024 ** 2, clients: 3, countries: 2, asns: 2 }, + days: [ + { day: '2026-10-29', requests: 600, bytes: 1024, clients: 2 }, + { day: '2026-10-30', requests: 634, bytes: 2048, clients: 3 }, ], repositories: [ - { name: 'npm-proxy', requests: 1000, bytes: 4 * 1024 ** 2, client_errors: 30, server_errors: 2 }, + { name: 'npm-proxy', requests: 1000, bytes: 4 * 1024 ** 2, clients: 2 }, + { name: '', requests: 234, bytes: 1024 ** 2, clients: 3 }, ], - statuses: [ - { status: 200, requests: 1192 }, - { status: 404, requests: 25 }, + countries: [ + { country: 'DE', requests: 900, bytes: 3 * 1024 ** 2, clients: 1 }, + { country: '', requests: 334, bytes: 2 * 1024 ** 2, clients: 2 }, ], + asns: [{ asn: 3320, org: 'Deutsche Telekom AG', requests: 900, bytes: 3 * 1024 ** 2, clients: 1 }], clients: [ { network: '2001:db8:1:2::/64', address: '2001:db8:1:2::7', + country: 'DE', + asn: 3320, + as_org: 'Deutsche Telekom AG', requests: 900, bytes: 3 * 1024 ** 2, - client_errors: 20, - denied: 5, - not_found: 15, - throttled: 0, - server_errors: 1, - user_agent: 'npm/10.8.2 node/v22', - account: 'token', - user_id: 7, - username: 'ci-bot', - hosts: ['web-1', 'web-2'], - first_seen: '2026-10-30T13:00:00Z', - last_seen: '2026-10-30T13:59:00Z', + repositories: ['npm-proxy', ''], + days: 2, + first_day: '2026-10-29', + last_day: '2026-10-30', }, ], recording, } -const clientDetail: TrafficClientDetail = { - network: '2001:db8:1:2::/64', - from: '2026-10-30T13:00:00Z', - to: '2026-10-30T14:00:00Z', - step_seconds: 60, - resolution: 'minute', - instances: overview.instances, - totals: overview.clients[0], - series: overview.series, - log: { - from: '2026-10-30T13:00:00Z', - to: '2026-10-30T14:00:00Z', - matched: 2, - entries: [ - { - time: '2026-10-30T13:59:00Z', - address: '2001:db8:1:2::7', - method: 'GET', - path: '/repository/npm-proxy/left-pad', - surface: 'plugin_npm', - repository: 'npm-proxy', - status: 404, - bytes: 21, - duration_ms: 3.5, - user_agent: 'npm/10.8.2 node/v22', - account: 'token', - user_id: 7, - username: 'ci-bot', - host: 'web-2', - }, - ], - paths: [{ value: '/repository/npm-proxy/left-pad', requests: 2 }], - user_agents: [{ value: 'npm/10.8.2 node/v22', requests: 2 }], - accounts: [{ account: 'token', user_id: 7, username: 'ci-bot', requests: 2 }], - statuses: [{ value: '404', requests: 2 }], - }, - recording, -} +const settings: TrafficSettings = { enabled: true, client_addresses: true, retention_days: 90 } -const settings: TrafficSettings = { - enabled: true, - client_addresses: true, - request_log: true, - statistics_retention_days: 400, - client_retention_days: 30, - request_log_retention_days: 7, -} +const geoip = (over: Partial = {}): GeoipStatus => ({ + country: { database_type: 'GeoLite2-Country', built_at: '2026-10-28T00:00:00Z', size: 9_000_000 }, + asn: { database_type: 'GeoLite2-ASN', built_at: '2026-10-28T00:00:00Z', size: 8_000_000 }, + maxmind: { + account_id: '123456', + license_key_set: true, + last_attempt: '2026-10-30T10:00:00Z', + last_success: '2026-10-30T10:00:00Z', + last_error: null, + url: 'https://download.maxmind.com', + editions: ['GeoLite2-Country', 'GeoLite2-ASN'], + ...over, + }, +}) function json(body: unknown, status = 200): Response { return new Response(JSON.stringify(body), { @@ -146,21 +91,34 @@ function json(body: unknown, status = 200): Response { const fetchMock = vi.fn() -function mockApi(h: { overview?: TrafficOverview; onPut?: (body: unknown) => Response } = {}) { +function mockApi(h: { overview?: TrafficOverview; geoip?: GeoipStatus } = {}) { + // The geolocation status, as the server keeps it between requests. + let geo = h.geoip ?? geoip() fetchMock.mockImplementation((input: RequestInfo | URL, init?: RequestInit) => { const url = new URL(String(input), 'http://localhost') const method = (init?.method ?? 'GET').toUpperCase() + const body = init?.body ? (JSON.parse(String(init.body)) as Record) : {} if (url.pathname.endsWith('/admin/me')) return Promise.resolve(json(adminMe)) if (url.pathname.endsWith('/admin/traffic')) return Promise.resolve(json(h.overview ?? overview)) - if (url.pathname.endsWith('/admin/traffic/client')) return Promise.resolve(json(clientDetail)) if (url.pathname.endsWith('/admin/settings/traffic')) { + return Promise.resolve(json(method === 'PUT' ? { ...settings, ...body } : settings)) + } + if (url.pathname.endsWith('/admin/traffic/geoip/update')) { + geo = { ...geo, maxmind: { ...geo.maxmind, last_error: 'MaxMind refused the account ID or the license key (401)' } } + return Promise.resolve(json(geo)) + } + if (url.pathname.endsWith('/admin/traffic/geoip')) { + if (method === 'DELETE') { + geo = { country: null, asn: null, maxmind: { ...geo.maxmind, account_id: null, license_key_set: false } } + return Promise.resolve(new Response(null, { status: 204 })) + } if (method === 'PUT') { - const body: unknown = JSON.parse(String(init?.body)) - return Promise.resolve( - h.onPut ? h.onPut(body) : json({ ...settings, ...(body as object) }), - ) + geo = { + ...geo, + maxmind: { ...geo.maxmind, account_id: String(body.account_id), license_key_set: true }, + } } - return Promise.resolve(json(settings)) + return Promise.resolve(json(geo)) } return Promise.resolve(new Response('not found', { status: 404 })) }) @@ -175,6 +133,19 @@ function sent(path: string, method = 'GET'): URLSearchParams[] { .map((u) => u.searchParams) } +/** The bodies of the requests to `path` sent with `method`. */ +function bodies(path: string, method: string): unknown[] { + return fetchMock.mock.calls + .filter(([input, init]) => { + const i = init as RequestInit | undefined + return i?.method === method && new URL(String(input), 'http://localhost').pathname === path + }) + .map(([, init]) => { + const raw = (init as RequestInit).body + return raw ? (JSON.parse(String(raw)) as unknown) : null + }) +} + describe('admin traffic screens', () => { beforeEach(() => { vi.stubGlobal('fetch', fetchMock) @@ -186,124 +157,142 @@ describe('admin traffic screens', () => { clearSession() }) - it('shows the totals, the top clients and the repositories', async () => { + it('shows the totals and where the requests came from', async () => { mockApi() await renderApp('/manage/traffic') expect(await screen.findByText('1,234')).toBeInTheDocument() - expect(screen.getByText('95 % answered within 250 ms, half within 10 ms')).toBeInTheDocument() - expect(sent('/admin/traffic')[0].get('range')).toBe('24h') + expect(sent('/admin/traffic')[0].get('range')).toBe('30d') + const repos = screen.getByRole('table', { name: 'Repositories by requests' }) + expect(within(repos).getByRole('link', { name: 'npm-proxy' })).toBeInTheDocument() + expect( + within(repos).getByRole('link', { name: 'No repository (web UI, APIs, sign-in)' }), + ).toBeInTheDocument() + const countries = screen.getByRole('table', { name: 'Countries by requests' }) + expect(within(countries).getByRole('link', { name: 'Germany' })).toBeInTheDocument() + expect(within(countries).getByRole('link', { name: 'Unknown' })).toBeInTheDocument() + const asns = screen.getByRole('table', { name: 'Autonomous systems by requests' }) + expect(within(asns).getByRole('link', { name: 'AS3320 Deutsche Telekom AG' })).toBeInTheDocument() const clients = screen.getByRole('table', { name: 'Top clients' }) const row = within(clients).getAllByRole('row')[1] expect(within(row).getByRole('link', { name: '2001:db8:1:2::/64' })).toBeInTheDocument() - expect(within(row).getByText('ci-bot (API token)')).toBeInTheDocument() - expect(within(row).getByText('web-1, web-2')).toBeInTheDocument() - - const repos = screen.getByRole('table', { name: 'Repositories by requests' }) - expect(within(repos).getByRole('link', { name: 'npm-proxy' })).toBeInTheDocument() - const surfaces = screen.getByRole('table', { name: 'Requests by surface' }) - expect(within(surfaces).getByText('Docker registry')).toBeInTheDocument() - expect(screen.getByRole('img', { name: /Requests over time: 1,234 in all/ })).toBeInTheDocument() + expect(within(row).getByText('Germany')).toBeInTheDocument() + expect(within(row).getByText('npm-proxy, No repository (web UI, APIs, sign-in)')).toBeInTheDocument() + expect(screen.getAllByText(/GeoLite Data created by MaxMind/).length).toBeGreaterThan(0) + expect(screen.getByRole('img', { name: /Requests by day: 1,234 in all/ })).toBeInTheDocument() }) - it('asks for the range, instance and repository chosen', async () => { + it('narrows the view from its lists, and removes the filters one by one', async () => { mockApi() const user = userEvent.setup() const router = await renderApp('/manage/traffic') await screen.findByText('1,234') await user.click(screen.getByRole('radio', { name: '7 days' })) - await waitFor(() => expect(sent('/admin/traffic').some((q) => q.get('range') === '7d')).toBe(true)) - - await user.selectOptions(screen.getByRole('combobox', { name: 'Instance' }), 'web-2_20261030T110000Z_00b2') - await waitFor(() => - expect(sent('/admin/traffic').some((q) => q.get('instance') === 'web-2_20261030T110000Z_00b2')).toBe(true), - ) - await user.click(screen.getByRole('link', { name: 'npm-proxy' })) + await user.click(await screen.findByRole('link', { name: 'Germany' })) await waitFor(() => - expect(sent('/admin/traffic').some((q) => q.get('repository') === 'npm-proxy')).toBe(true), + expect( + sent('/admin/traffic').some( + (q) => q.get('range') === '7d' && q.get('repository') === 'npm-proxy' && q.get('country') === 'DE', + ), + ).toBe(true), ) - expect(router.state.location.search).toMatchObject({ - range: '7d', - instance: 'web-2_20261030T110000Z_00b2', - repository: 'npm-proxy', - }) + expect(router.state.location.search).toMatchObject({ range: '7d', repository: 'npm-proxy', country: 'DE' }) + + await user.click(screen.getByRole('button', { name: 'Remove the filter Repository: npm-proxy' })) + await waitFor(() => expect(router.state.location.search).not.toHaveProperty('repository')) + expect(router.state.location.search).toMatchObject({ country: 'DE' }) + + // The requests to no repository and the unknown country are filters too. + await user.click(screen.getByRole('link', { name: 'No repository (web UI, APIs, sign-in)' })) + await waitFor(() => expect(sent('/admin/traffic').some((q) => q.get('repository') === '-')).toBe(true)) + await user.click(screen.getByRole('button', { name: 'Clear all' })) + await waitFor(() => expect(router.state.location.search).not.toHaveProperty('country')) }) - it('shows the points as a table on request', async () => { + it('shows the days as a table on request', async () => { mockApi() const user = userEvent.setup() await renderApp('/manage/traffic') await screen.findByText('1,234') await user.click(screen.getByRole('radio', { name: 'Table' })) - const table = screen.getByRole('table', { name: 'Requests over time' }) + const table = screen.getByRole('table', { name: 'Requests by day' }) const rows = within(table).getAllByRole('row').slice(1) expect(rows.map((r) => within(r).getAllByRole('cell')[1].textContent)).toEqual(['634', '600']) }) - it('says when nothing is recorded', async () => { - mockApi({ overview: { ...overview, recording: { ...recording, enabled: false } } }) + it('says how to look countries up without a database, and when nothing is recorded', async () => { + mockApi({ + overview: { + ...overview, + totals: { ...overview.totals, countries: 0, asns: 0 }, + countries: [{ country: '', requests: 1234, bytes: 0, clients: 3 }], + asns: [{ asn: 0, org: '', requests: 1234, bytes: 0, clients: 3 }], + recording: { ...recording, enabled: false, countries: false, asns: false }, + }, + }) await renderApp('/manage/traffic') expect(await screen.findByText('Traffic is not recorded')).toBeInTheDocument() + expect(screen.getByText('Countries are not looked up')).toBeInTheDocument() + expect(screen.queryByRole('table', { name: 'Autonomous systems by requests' })).toBeNull() }) - it("opens a client's view with its request log", async () => { + it('saves the recording settings', async () => { mockApi() const user = userEvent.setup() - await renderApp('/manage/traffic?range=6h') - await user.click(await screen.findByRole('link', { name: '2001:db8:1:2::/64' })) + await renderApp('/manage/traffic/settings') - const log = await screen.findByRole('table', { name: 'Request log' }) - const row = within(log).getAllByRole('row')[1] - expect(within(row).getByText('GET /repository/npm-proxy/left-pad')).toBeInTheDocument() - expect(within(row).getByText('404')).toBeInTheDocument() - expect(within(row).getByText('ci-bot (API token)')).toBeInTheDocument() - const accounts = screen.getByRole('table', { name: 'Accounts in the request log' }) - expect(within(accounts).getByText('ci-bot (API token)')).toBeInTheDocument() + await user.click(await screen.findByRole('switch', { name: /Client addresses/ })) + const retention = screen.getByRole('textbox', { name: 'Keep the statistics for (days)' }) + await user.clear(retention) + await user.type(retention, '0') + await user.click(screen.getByRole('button', { name: 'Save recording' })) + expect(await screen.findByText(/Enter a whole number of days from 1/)).toBeInTheDocument() + expect(bodies('/admin/settings/traffic', 'PUT')).toEqual([]) - const asked = sent('/admin/traffic/client')[0] - expect(asked.get('network')).toBe('2001:db8:1:2::/64') - expect(asked.get('range')).toBe('6h') + await user.clear(retention) + await user.type(retention, '30') + await user.click(screen.getByRole('button', { name: 'Save recording' })) + await waitFor(() => + expect(bodies('/admin/settings/traffic', 'PUT')).toEqual([ + { enabled: true, client_addresses: false, retention_days: 30 }, + ]), + ) }) - it('saves each settings panel on its own', async () => { - mockApi() + it('sets the MaxMind account, downloads, and turns geolocation off', async () => { + mockApi({ geoip: { country: null, asn: null, maxmind: { ...geoip().maxmind, account_id: null, license_key_set: false, last_success: null } } }) const user = userEvent.setup() await renderApp('/manage/traffic/settings') - await user.click(await screen.findByRole('switch', { name: /Request log/ })) - await user.click(screen.getByRole('button', { name: 'Save recording' })) - await waitFor(() => expect(sent('/admin/settings/traffic', 'PUT')).toHaveLength(1)) - const puts = fetchMock.mock.calls.filter( - ([, init]) => (init as RequestInit | undefined)?.method === 'PUT', + expect(await screen.findAllByText('Not loaded')).toHaveLength(2) + await user.click(screen.getByRole('button', { name: 'Save account' })) + expect(screen.getByText('Enter your MaxMind account ID, a number.')).toBeInTheDocument() + expect(screen.getByText('Enter a license key.')).toBeInTheDocument() + + await user.type(screen.getByRole('textbox', { name: 'MaxMind account ID' }), '123456') + await user.type(screen.getByLabelText('License key'), 'abc_secret') + await user.click(screen.getByRole('button', { name: 'Save account' })) + await waitFor(() => + expect(bodies('/admin/traffic/geoip', 'PUT')).toEqual([ + { account_id: '123456', license_key: 'abc_secret' }, + ]), ) - expect(JSON.parse(String((puts[0][1] as RequestInit).body))).toEqual({ - enabled: true, - client_addresses: true, - request_log: false, - }) + // The key is not kept on screen once stored. + await waitFor(() => expect(screen.getByLabelText('License key')).toHaveValue('')) - const log = screen.getByRole('textbox', { name: 'Keep the request log for (days)' }) - await user.clear(log) - await user.type(log, '0') - await user.click(screen.getByRole('button', { name: 'Save retention' })) - expect(await screen.findByText(/Enter a whole number of days from 1/)).toBeInTheDocument() - expect(sent('/admin/settings/traffic', 'PUT')).toHaveLength(1) + await user.click(await screen.findByRole('button', { name: 'Download now' })) + expect(await screen.findByText('The last download failed')).toBeInTheDocument() + expect(screen.getByText(/refused the account ID or the license key/)).toBeInTheDocument() - await user.clear(log) - await user.type(log, '14') - await user.click(screen.getByRole('button', { name: 'Save retention' })) - await waitFor(() => expect(sent('/admin/settings/traffic', 'PUT')).toHaveLength(2)) - const second = fetchMock.mock.calls.filter( - ([, init]) => (init as RequestInit | undefined)?.method === 'PUT', - )[1] - expect(JSON.parse(String((second[1] as RequestInit).body))).toEqual({ - statistics_retention_days: 400, - client_retention_days: 30, - request_log_retention_days: 14, - }) + await user.click(screen.getByRole('button', { name: 'Turn off' })) + const dialog = screen.getByRole('dialog', { name: 'Turn geolocation off?' }) + expect(bodies('/admin/traffic/geoip', 'DELETE')).toEqual([]) + await user.click(within(dialog).getByRole('button', { name: 'Turn off' })) + await waitFor(() => expect(bodies('/admin/traffic/geoip', 'DELETE')).toHaveLength(1)) + expect(await screen.findByText('Not set')).toBeInTheDocument() }) }) From 96b85cc98c296f119e8d2165fb6783e53d596bb1 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 10:05:28 +0000 Subject: [PATCH 3/3] Storage: box the S3 upload futures, so requests that store a blob fit the stack The raw group test overflowed its 2 MiB test thread in CI: a debug build of a request that fetches a file through a group's proxy and stores it needed about 1.95 MiB of stack on main already, and the traffic statistics' per-request layers took it over. Every future up that request held the AWS SDK's upload future inline (put_if_absent, the CAS commit, the file store, the proxy fill, the group walk, the plugin dispatch), so each of their poll frames carried a copy. put_if_absent and the conditional PUT now return boxed futures: the SDK's future lives in one heap allocation, and the request needs about 1.55 MiB, measured with RUST_MIN_STACK on the test. Nothing else changes. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012e6DJbpgZmQnhz5Jjjhug6 --- crates/storage/src/s3.rs | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/crates/storage/src/s3.rs b/crates/storage/src/s3.rs index 308e8b22..62734944 100644 --- a/crates/storage/src/s3.rs +++ b/crates/storage/src/s3.rs @@ -405,7 +405,21 @@ impl BlobStore { /// Single-part bodies may use a conditional PUT (`If-None-Match: *`) /// instead of HEAD-then-PUT, per [`ConditionalWrites`]; the two paths are /// observably identical, the conditional one just saves a round trip. - pub async fn put_if_absent(&self, key: &str, path: &Path) -> Result { + /// + /// The upload is boxed: the SDK's request futures are large, and a caller + /// would otherwise hold one inline, and so would every future awaiting + /// it, up the whole request (a deep stack in a debug build). + pub fn put_if_absent<'a>( + &'a self, + key: &'a str, + path: &'a Path, + ) -> std::pin::Pin> + Send + 'a>> + { + Box::pin(self.upload_if_absent(key, path)) + } + + /// [`Self::put_if_absent`], unboxed. + async fn upload_if_absent(&self, key: &str, path: &Path) -> Result { self.admit(S3Op::Put)?; let size = tokio::fs::metadata(path).await?.len(); @@ -472,8 +486,19 @@ impl BlobStore { /// One conditional PUT attempt. Maps the interesting backend answers /// (412 / NotImplemented / 409) into [`ConditionalPut`]; anything else is - /// a real error. - async fn put_conditional(&self, key: &str, path: &Path) -> Result { + /// a real error. Boxed, like [`Self::put_if_absent`]. + fn put_conditional<'a>( + &'a self, + key: &'a str, + path: &'a Path, + ) -> std::pin::Pin< + Box> + Send + 'a>, + > { + Box::pin(self.send_conditional(key, path)) + } + + /// [`Self::put_conditional`], unboxed. + async fn send_conditional(&self, key: &str, path: &Path) -> Result { let body = body_from_path(path).await?; let call = S3Call::start(S3Op::PutConditional, &self.bucket); let result = self