diff --git a/Cargo.lock b/Cargo.lock index d8dad922..71d5529b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -45,6 +45,20 @@ dependencies = [ "zeroize", ] +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "const-random", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.5" @@ -1083,6 +1097,26 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -1206,6 +1240,12 @@ version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-common" version = "0.1.7" @@ -1846,6 +1886,7 @@ dependencies = [ "forge-plugin", "forge-storage", "forge-telemetry", + "forge-traffic", "futures-util", "hex", "http", @@ -1878,6 +1919,7 @@ dependencies = [ "sha2 0.11.1", "sqlx", "subtle", + "tar", "tempfile", "testcontainers", "testcontainers-modules", @@ -1933,6 +1975,27 @@ dependencies = [ "tracing", ] +[[package]] +name = "forge-traffic" +version = "2.1.0" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "chrono", + "forge-storage", + "futures-util", + "maxminddb", + "moka", + "parquet", + "parquet_derive", + "rand 0.10.3", + "serde", + "serde_json", + "tokio", + "tracing", +] + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -2138,6 +2201,18 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -2575,6 +2650,12 @@ version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" +[[package]] +name = "ipnetwork" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf370abdafd54d13e54a620e8c3e1145f28e46cc9d704bc6d94414559df41763" + [[package]] name = "itertools" version = "0.14.0" @@ -2848,6 +2929,18 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "maxminddb" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b43d2458d977b610b04a3858e6504f06133663d303a1a0606601b530bfc9bc6" +dependencies = [ + "ipnetwork", + "memchr", + "serde", + "thiserror", +] + [[package]] name = "md-5" version = "0.11.0" @@ -3267,6 +3360,36 @@ dependencies = [ "windows-link", ] +[[package]] +name = "parquet" +version = "60.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "806eaa0f1c0436dcb43bee8e6a82bf2efff90521a60172f8281cb9ef577b43a4" +dependencies = [ + "ahash", + "bytes", + "chrono", + "half", + "hashbrown 0.17.1", + "num-bigint 0.5.1", + "num-integer", + "num-traits", + "seq-macro", + "snap", + "twox-hash", +] + +[[package]] +name = "parquet_derive" +version = "60.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c6a0cc5516d0c382595f124ba4f88fdfff723adec2adf58c7d34730d0c30b53" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "parse-display" version = "0.9.1" @@ -4115,6 +4238,12 @@ version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + [[package]] name = "serde" version = "1.0.229" @@ -4368,6 +4497,12 @@ dependencies = [ "serde", ] +[[package]] +name = "snap" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "199905e6153d6405f9728fe44daace35f8f837bbf830bb6e85fbd5828709a886" + [[package]] name = "socket2" version = "0.6.5" @@ -4856,6 +4991,15 @@ dependencies = [ "time-core", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tinystr" version = "0.8.4" @@ -5172,6 +5316,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "twox-hash" +version = "2.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86a801b3cea342a06d468c8710662aa29e5e05e4f5c0d62f00bbb7f2ad7941c2" + [[package]] name = "typed-path" version = "0.12.3" diff --git a/Cargo.toml b/Cargo.toml index f5717aac..f9123884 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "crates/core", "crates/crypto", "crates/telemetry", + "crates/traffic", "crates/plugin-api", "crates/plugin", "crates/server", @@ -192,6 +193,14 @@ jsonwebtoken = { version = "11", default-features = false, features = ["use_pem" # already links, used directly where `jsonwebtoken` has no key generation. aws-lc-rs = { version = "1.18" } +# Traffic statistics (crates/traffic): Parquet files in the object store. +# Without the Arrow layer (its record API suffices) and with Snappy alone, a +# pure-Rust codec. +parquet = { version = "60", default-features = false, features = ["snap"] } +parquet_derive = { version = "60" } +# Country and ASN lookups in MaxMind databases (crates/traffic). +maxminddb = { version = "0.32" } + # Temp files tempfile = { version = "3" } diff --git a/crates/db/src/secret_dao.rs b/crates/db/src/secret_dao.rs index 6a157ae4..ef9831c3 100644 --- a/crates/db/src/secret_dao.rs +++ b/crates/db/src/secret_dao.rs @@ -22,7 +22,8 @@ //! [`insert_realm_with_secrets`] / [`realm_config`]). //! //! Secrets kept in `forge_setting` ([`SECRET_SETTINGS`]: the generated -//! registry-token key, the OIDC issuer keys) are text values, +//! registry-token key, the OIDC issuer keys, the MaxMind license key) are +//! text values, //! `sealed:`, AAD-bound to `('forge_setting', 'value', 0)` //! plus the setting's key. They take part in the re-encrypt sweep and the DEK //! reference scan like the secret columns ([`store_secret_setting`], @@ -1073,8 +1074,16 @@ pub const REGISTRY_TOKEN_KEY_SETTING: &str = "docker_token_key"; /// assertions with. pub const OIDC_ISSUER_KEYS_SETTING: &str = "oidc_issuer_keys"; +/// `forge_setting` key of the MaxMind license key the traffic statistics' +/// geolocation databases are downloaded with. +pub const MAXMIND_LICENSE_KEY_SETTING: &str = "traffic_maxmind_license_key"; + /// The `forge_setting` keys whose values are secrets (module docs). -pub const SECRET_SETTINGS: &[&str] = &[REGISTRY_TOKEN_KEY_SETTING, OIDC_ISSUER_KEYS_SETTING]; +pub const SECRET_SETTINGS: &[&str] = &[ + REGISTRY_TOKEN_KEY_SETTING, + OIDC_ISSUER_KEYS_SETTING, + MAXMIND_LICENSE_KEY_SETTING, +]; /// Prefix of a sealed secret setting: `sealed:`. const SEALED_SETTING: &str = "sealed:"; diff --git a/crates/server/Cargo.toml b/crates/server/Cargo.toml index 1b7da79a..608e876b 100644 --- a/crates/server/Cargo.toml +++ b/crates/server/Cargo.toml @@ -22,6 +22,7 @@ forge-core = { path = "../core" } forge-crypto = { path = "../crypto" } forge-plugin = { path = "../plugin" } forge-telemetry = { path = "../telemetry" } +forge-traffic = { path = "../traffic" } # The built-in format plugins (crates/formats/docker, crates/formats/nuget, # crates/formats/npm, crates/formats/raw), linked in. forge-format-docker = { path = "../formats/docker" } @@ -79,6 +80,8 @@ tokio-util = { workspace = true } multer = "3" # npm audits: a gzip-compressed request body. flate2 = { version = "1.1", default-features = false, features = ["zlib-rs"] } +# The MaxMind databases of the traffic statistics come as `.tar.gz`. +tar = { version = "0.4", default-features = false } # The file store: the MD5 of uploads (the `.md5` sidecar), file names in # headers, HTTP dates (`Last-Modified`, `If-Modified-Since`) and content # types guessed from file names. @@ -110,6 +113,9 @@ figment = { version = "0.10", features = ["toml", "env"] } tikv-jemallocator = { version = "0.7", features = ["background_threads"] } [dev-dependencies] +# Small MaxMind databases for the traffic statistics' tests. +forge-traffic = { path = "../traffic", features = ["testing"] } +tar = { version = "0.4", default-features = false } # Builds Helm chart archives for the registry tests. flate2 = { version = "1.1", default-features = false, features = ["zlib-rs"] } tokio = { workspace = true, features = ["rt-multi-thread", "macros", "fs", "io-util", "time", "test-util"] } diff --git a/crates/server/src/admin/mod.rs b/crates/server/src/admin/mod.rs index af2dcdda..0e979f20 100644 --- a/crates/server/src/admin/mod.rs +++ b/crates/server/src/admin/mod.rs @@ -70,6 +70,7 @@ pub mod roles; pub mod settings; pub mod token_scope; pub mod tokens; +pub mod traffic; pub mod trusted_publishers; pub mod user_roles; pub mod users; @@ -351,6 +352,22 @@ pub fn admin_router(state: AppState) -> Router { .route("/crypto/keys", get(crypto::list_keys)) // --- local blob cache ---------------------------------------------- .route("/blob-cache", get(blob_cache::stats)) + // --- traffic statistics -------------------------------------------- + .route("/traffic", get(traffic::overview)) + .route( + "/traffic/geoip", + get(traffic::get_geoip) + .put(traffic::put_geoip) + .delete(traffic::delete_geoip), + ) + .route( + "/traffic/geoip/update", + axum::routing::post(traffic::update_geoip), + ) + .route( + "/settings/traffic", + get(traffic::get_settings).put(traffic::put_settings), + ) // --- the email service --------------------------------------------- .route( "/settings/email", diff --git a/crates/server/src/admin/traffic.rs b/crates/server/src/admin/traffic.rs new file mode 100644 index 00000000..c0dfb95b --- /dev/null +++ b/crates/server/src/admin/traffic.rs @@ -0,0 +1,481 @@ +//! The traffic statistics (Admin → Traffic). **Admin-only.** +//! +//! * `GET /admin/traffic?range=&instance=&repository=&network=&country=&asn=` +//! — the requests of a range of days ([`forge_traffic::query::Overview`]) +//! over every instance or the one `instance` names: by day, repository, +//! country, autonomous system and client network. `range` is one of +//! [`RANGES`] (default `30d`), or `from` and `to` (dates, both included) +//! give it. `repository=-` keeps the requests that addressed no +//! repository, `country=-` and `asn=0` the ones from nowhere known. +//! * `GET /admin/settings/traffic` answers the [`TrafficSettings`]; `PUT` +//! changes the fields a body names, for every instance (each follows with +//! its next flush). +//! * `GET /admin/traffic/geoip` — the geolocation databases this instance +//! loaded and the MaxMind download's account and outcome; `PUT` sets the +//! account ID and license key (sealed), `DELETE` forgets them and deletes +//! the databases, `POST /admin/traffic/geoip/update` downloads them now. +//! +//! The files are read from the object store, so the answers cover every +//! instance; a store that cannot be read answers 503. + +use axum::extract::{Query, State}; +use axum::http::StatusCode; +use axum::response::{IntoResponse, Json, Response}; +use chrono::{Duration, NaiveDate, Utc}; +use forge_core::auth::principal::Principal; +use forge_traffic::query::{self, Filter}; +use forge_traffic::{DbInfo, GeoKind, TrafficSettings}; +use serde::{Deserialize, Serialize}; + +use crate::state::AppState; +use crate::traffic::maxmind::{self, MaxmindState}; +use crate::traffic::{read_settings, write_settings}; + +/// The ranges `range` names, in days up to today (included). +pub const RANGES: [(&str, i64); 5] = [ + ("today", 1), + ("7d", 7), + ("30d", 30), + ("90d", 90), + ("365d", 365), +]; + +/// What a filter value says for "none" (no repository, no known country). +const NONE: &str = "-"; + +/// The query of the overview. +#[derive(Debug, Default, Deserialize)] +pub struct OverviewQuery { + pub range: Option, + pub from: Option, + pub to: Option, + /// Only this instance's files (an id of the answer's `instances`). + pub instance: Option, + /// Only this repository's requests; `-` for the ones to none. + pub repository: Option, + /// Only this client network's requests. + pub network: Option, + /// Only this country's requests (ISO code); `-` for unknown. + pub country: Option, + /// Only this autonomous system's requests; `0` for unknown. + pub asn: Option, +} + +fn bad_request(msg: impl Into) -> Response { + (StatusCode::BAD_REQUEST, msg.into()).into_response() +} + +fn unavailable(e: anyhow::Error) -> Response { + tracing::warn!(error = %format!("{e:#}"), "cannot read the traffic statistics"); + ( + StatusCode::SERVICE_UNAVAILABLE, + "cannot read the traffic statistics from the object store", + ) + .into_response() +} + +fn some(value: &Option) -> Option { + value.as_deref().filter(|v| !v.is_empty()).map(|v| { + if v == NONE { + String::new() + } else { + v.to_string() + } + }) +} + +impl OverviewQuery { + /// The days the query covers, both included. + fn days(&self, today: NaiveDate) -> Result<(NaiveDate, NaiveDate), Response> { + match (self.from, self.to) { + (Some(from), Some(to)) => { + if to < from { + return Err(bad_request("to must not be before from")); + } + if (to - from).num_days() >= query::MAX_DAYS { + return Err(bad_request(format!( + "a range is at most {} days", + query::MAX_DAYS + ))); + } + Ok((from, to)) + } + (None, None) => { + let name = self.range.as_deref().unwrap_or("30d"); + let Some((_, days)) = RANGES.iter().find(|(n, _)| *n == name) else { + let names: Vec<&str> = RANGES.iter().map(|(n, _)| *n).collect(); + return Err(bad_request(format!( + "range must be one of {}", + names.join(", ") + ))); + }; + Ok((today - Duration::days(days - 1), today)) + } + _ => Err(bad_request("give both from and to, or neither")), + } + } + + fn filter(&self) -> Filter { + Filter { + instance: self.instance.clone().filter(|i| !i.is_empty()), + repository: some(&self.repository), + network: self.network.clone().filter(|n| !n.is_empty()), + country: some(&self.country), + asn: self.asn, + } + } +} + +/// What is recorded now and what can be looked up, so the page can tell why +/// something is missing. +#[derive(Debug, Serialize)] +pub struct Recording { + pub enabled: bool, + pub client_addresses: bool, + /// Whether this instance has a country database loaded. + pub countries: bool, + /// Whether this instance has an autonomous system database loaded. + pub asns: bool, +} + +/// `GET /admin/traffic`. +#[derive(Debug, Serialize)] +pub struct OverviewView { + #[serde(flatten)] + pub overview: query::Overview, + pub recording: Recording, +} + +/// `GET /admin/traffic` +pub async fn overview(State(state): State, Query(q): Query) -> Response { + let (from, to) = match q.days(Utc::now().date_naive()) { + Ok(days) => days, + Err(resp) => return resp, + }; + let settings = match stored(&state).await { + Ok(settings) => settings, + Err(resp) => return resp, + }; + let traffic = &state.traffic; + let overview = + match query::overview(&*traffic.objects, &traffic.cache, from, to, &q.filter()).await { + Ok(overview) => overview, + Err(e) => return unavailable(e), + }; + Json(OverviewView { + overview, + recording: Recording { + enabled: settings.enabled, + client_addresses: settings.client_addresses, + countries: traffic.geo.info(GeoKind::Country).is_some(), + asns: traffic.geo.info(GeoKind::Asn).is_some(), + }, + }) + .into_response() +} + +/// The stored settings, or the 500 answer when they cannot be read. +async fn stored(state: &AppState) -> Result { + read_settings(&state.pool).await.map_err(|e| { + tracing::error!(error = %e, "cannot read the traffic settings"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + "cannot read the traffic settings", + ) + .into_response() + }) +} + +/// `PUT /admin/settings/traffic` body: the fields to change. Unknown fields +/// are refused, so a misspelled one is not dropped. +#[derive(Debug, Default, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TrafficSettingsUpdate { + pub enabled: Option, + pub client_addresses: Option, + pub retention_days: Option, +} + +/// `GET /admin/settings/traffic` +pub async fn get_settings(State(state): State) -> Response { + match stored(&state).await { + Ok(settings) => Json(settings).into_response(), + Err(resp) => resp, + } +} + +/// `PUT /admin/settings/traffic` — change the fields the body names. This +/// instance applies a change at once, the others with their next flush. +pub async fn put_settings( + State(state): State, + axum::Extension(principal): axum::Extension, + Json(update): Json, +) -> Response { + let current = match stored(&state).await { + Ok(settings) => settings, + Err(resp) => return resp, + }; + let TrafficSettingsUpdate { + enabled, + client_addresses, + retention_days, + } = update; + let next = TrafficSettings { + enabled: enabled.unwrap_or(current.enabled), + client_addresses: client_addresses.unwrap_or(current.client_addresses), + retention_days: retention_days.unwrap_or(current.retention_days), + }; + if let Err(msg) = next.check() { + return bad_request(msg); + } + if next != current { + if let Err(e) = write_settings(&state.pool, &next).await { + return super::db_error(e); + } + state.traffic.recorder.set_enabled(next.enabled); + tracing::info!( + target: "forge::audit", + settings = ?next, + principal = %principal.audit_label(), + "traffic settings changed" + ); + } + Json(next).into_response() +} + +/// `GET /admin/traffic/geoip`. +#[derive(Debug, Serialize)] +pub struct GeoipView { + /// The databases the answering instance loaded. + pub country: Option, + pub asn: Option, + /// The MaxMind download. + pub maxmind: MaxmindView, +} + +/// The MaxMind download, without the license key. +#[derive(Debug, Serialize)] +pub struct MaxmindView { + pub account_id: Option, + pub license_key_set: bool, + pub last_attempt: Option>, + pub last_success: Option>, + pub last_error: Option, + /// Where the databases are downloaded from (`traffic.maxmind_url`). + pub url: String, + /// The editions downloaded. + pub editions: Vec<&'static str>, +} + +async fn geoip_view(state: &AppState) -> Result { + let (stored, _) = maxmind::read_state(&state.pool) + .await + .map_err(super::db_error)?; + let license_key_set = maxmind::has_license_key(&state.pool) + .await + .map_err(super::db_error)?; + let MaxmindState { + account_id, + last_attempt, + last_success, + last_error, + } = stored; + Ok(GeoipView { + country: state.traffic.geo.info(GeoKind::Country), + asn: state.traffic.geo.info(GeoKind::Asn), + maxmind: MaxmindView { + account_id, + license_key_set, + last_attempt, + last_success, + last_error, + url: state.config.traffic.maxmind_base().to_string(), + editions: maxmind::EDITIONS.iter().map(|(_, e)| *e).collect(), + }, + }) +} + +/// `GET /admin/traffic/geoip` +pub async fn get_geoip(State(state): State) -> Response { + // Pick up databases another instance stored a moment ago. + crate::traffic::refresh_geo(&state.traffic).await; + match geoip_view(&state).await { + Ok(view) => Json(view).into_response(), + Err(resp) => resp, + } +} + +/// `PUT /admin/traffic/geoip` body. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct GeoipCredentials { + pub account_id: String, + /// Left out (or empty) keeps the stored one. + #[serde(default)] + pub license_key: Option, +} + +/// `PUT /admin/traffic/geoip` — set the MaxMind account ID and license key. +/// The databases are downloaded with the worker's next check, or at once +/// with `POST /admin/traffic/geoip/update`. +pub async fn put_geoip( + State(state): State, + axum::Extension(principal): axum::Extension, + Json(body): Json, +) -> Response { + let account_id = body.account_id.trim(); + if account_id.is_empty() + || account_id.len() > 32 + || !account_id.chars().all(|c| c.is_ascii_digit()) + { + return bad_request("account_id must be the MaxMind account ID, a number"); + } + let license_key = body + .license_key + .as_deref() + .map(str::trim) + .filter(|k| !k.is_empty()); + if let Some(key) = license_key { + if key.len() > 256 || !key.chars().all(|c| c.is_ascii_graphic()) { + return bad_request("license_key must be a MaxMind license key"); + } + } else { + match maxmind::has_license_key(&state.pool).await { + Ok(true) => {} + Ok(false) => return bad_request("license_key is required"), + Err(e) => return super::db_error(e), + } + } + if let Err(e) = maxmind::set_credentials(&state, account_id, license_key).await { + tracing::error!(error = %format!("{e:#}"), "cannot store the MaxMind credentials"); + return ( + StatusCode::INTERNAL_SERVER_ERROR, + "cannot store the MaxMind credentials", + ) + .into_response(); + } + tracing::info!( + target: "forge::audit", + account_id, + license_key_changed = license_key.is_some(), + principal = %principal.audit_label(), + "MaxMind credentials changed" + ); + match geoip_view(&state).await { + Ok(view) => Json(view).into_response(), + Err(resp) => resp, + } +} + +/// `DELETE /admin/traffic/geoip` — forget the MaxMind credentials and delete +/// the databases: requests are no longer located. +pub async fn delete_geoip( + State(state): State, + axum::Extension(principal): axum::Extension, +) -> Response { + if let Err(e) = maxmind::forget(&state).await { + tracing::error!(error = %format!("{e:#}"), "cannot remove the geolocation databases"); + return ( + StatusCode::SERVICE_UNAVAILABLE, + "cannot remove the geolocation databases", + ) + .into_response(); + } + tracing::info!( + target: "forge::audit", + principal = %principal.audit_label(), + "MaxMind credentials and geolocation databases removed" + ); + StatusCode::NO_CONTENT.into_response() +} + +/// `POST /admin/traffic/geoip/update` — download the databases now. Answers +/// the status, the download's error included; 409 without credentials. +pub async fn update_geoip(State(state): State) -> Response { + let (stored, raw) = match maxmind::read_state(&state.pool).await { + Ok(found) => found, + Err(e) => return super::db_error(e), + }; + match maxmind::has_license_key(&state.pool).await { + Ok(true) if stored.account_id.is_some() => {} + Ok(_) => { + return ( + StatusCode::CONFLICT, + "set the MaxMind account ID and license key first", + ) + .into_response() + } + Err(e) => return super::db_error(e), + } + match maxmind::claim(&state.pool, &stored, raw.as_deref(), Utc::now()).await { + Ok(true) => {} + Ok(false) => { + return ( + StatusCode::CONFLICT, + "another instance is downloading the databases; try again in a moment", + ) + .into_response() + } + Err(e) => return super::db_error(e), + } + // The outcome, a failure included, is in the answer's `last_error`. + let _ = maxmind::update(&state).await; + match geoip_view(&state).await { + Ok(view) => Json(view).into_response(), + Err(resp) => resp, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn d(s: &str) -> NaiveDate { + NaiveDate::parse_from_str(s, "%Y-%m-%d").unwrap() + } + + #[test] + fn ranges_are_named_or_given() { + let today = d("2026-10-30"); + assert_eq!( + OverviewQuery::default().days(today).unwrap(), + (d("2026-10-01"), today), + "30 days by default, today included" + ); + let q = OverviewQuery { + range: Some("today".into()), + ..OverviewQuery::default() + }; + assert_eq!(q.days(today).unwrap(), (today, today)); + let q = OverviewQuery { + range: Some("2h".into()), + ..OverviewQuery::default() + }; + assert!(q.days(today).is_err()); + let q = OverviewQuery { + from: Some(d("2026-10-10")), + to: Some(d("2026-10-12")), + ..OverviewQuery::default() + }; + assert_eq!(q.days(today).unwrap(), (d("2026-10-10"), d("2026-10-12"))); + let q = OverviewQuery { + from: Some(d("2026-10-10")), + ..OverviewQuery::default() + }; + assert!(q.days(today).is_err(), "from alone"); + } + + #[test] + fn a_dash_filters_for_none() { + let q = OverviewQuery { + repository: Some("-".into()), + country: Some("DE".into()), + network: Some(String::new()), + ..OverviewQuery::default() + }; + let f = q.filter(); + assert_eq!(f.repository.as_deref(), Some("")); + assert_eq!(f.country.as_deref(), Some("DE")); + assert_eq!(f.network, None); + } +} diff --git a/crates/server/src/app.rs b/crates/server/src/app.rs index 2ed7e519..01539c7a 100644 --- a/crates/server/src/app.rs +++ b/crates/server/src/app.rs @@ -178,6 +178,7 @@ pub fn build_router(state: AppState, layer: PrincipalLayer) -> Router { let app = app.fallback_service(spa_service(&state.config.web)); let hsts = state.cookie_secure; + let recorder = state.traffic.recorder.clone(); let public_base_url = state.config.public_base_url(); let images = images.map(|p| p.implementation().clone()); let server = state.config.server.clone(); @@ -249,7 +250,10 @@ pub fn build_router(state: AppState, layer: PrincipalLayer) -> Router { // The request's OpenTelemetry span and metrics, outermost so they // time and count every answer, the ones the layers above give // themselves (a timeout) included. - .layer(axum::middleware::from_fn(crate::telemetry::track)) + .layer(axum::middleware::from_fn_with_state( + recorder, + crate::telemetry::track, + )) .with_state(state) } diff --git a/crates/server/src/config.rs b/crates/server/src/config.rs index 4028484c..65cfbfa8 100644 --- a/crates/server/src/config.rs +++ b/crates/server/src/config.rs @@ -96,6 +96,10 @@ pub struct Config { /// metrics report. #[serde(default)] pub telemetry: TelemetryConfig, + /// The traffic statistics (Admin → Traffic): where the geolocation + /// databases are downloaded from. What is recorded is set in the web UI. + #[serde(default)] + pub traffic: TrafficConfig, } /// `server.*` @@ -1460,6 +1464,50 @@ impl TelemetryConfig { } } +/// `[traffic]`: the traffic statistics ([`crate::traffic`]). What they +/// record, how long they keep it and the MaxMind account the geolocation +/// databases are downloaded with are set in the web UI (Admin → Traffic → +/// Settings). +#[derive(Debug, Clone, Deserialize)] +pub struct TrafficConfig { + /// Where the MaxMind databases are downloaded from: MaxMind's download + /// service, or a mirror that answers its API + /// (`{url}/geoip/databases/{edition}/download?suffix=tar.gz`, HTTP Basic + /// with the account ID and license key). Default + /// `https://download.maxmind.com`. + #[serde(default = "default_maxmind_url")] + pub maxmind_url: String, +} + +impl Default for TrafficConfig { + fn default() -> Self { + TrafficConfig { + maxmind_url: default_maxmind_url(), + } + } +} + +fn default_maxmind_url() -> String { + "https://download.maxmind.com".into() +} + +impl TrafficConfig { + fn validate(&self) -> Result<(), ConfigError> { + match url::Url::parse(&self.maxmind_url) { + Ok(url) if matches!(url.scheme(), "https" | "http") && url.host().is_some() => Ok(()), + _ => Err(ConfigError::Invalid(format!( + "traffic.maxmind_url must be an http(s) URL, not {:?}", + self.maxmind_url + ))), + } + } + + /// [`Self::maxmind_url`] without a trailing `/`. + pub fn maxmind_base(&self) -> &str { + self.maxmind_url.trim_end_matches('/') + } +} + /// `[docker]`: the Docker / OCI registry served under `/v2`. #[derive(Debug, Clone, Deserialize)] pub struct DockerConfig { @@ -2377,6 +2425,7 @@ impl Config { fn validate(&self) -> Result<(), ConfigError> { self.server.bind_addr()?; self.telemetry.validate()?; + self.traffic.validate()?; self.validate_public_base_url()?; // The DNS-rebinding guard's host list, checked before migrations run // rather than when the router is built. diff --git a/crates/server/src/images/dispatch.rs b/crates/server/src/images/dispatch.rs index d97aa34e..80c77607 100644 --- a/crates/server/src/images/dispatch.rs +++ b/crates/server/src/images/dispatch.rs @@ -379,6 +379,7 @@ impl Registry { /// grammar already. pub(crate) async fn resolve(&self, site: &Site, name: &str) -> ImgResult> { if let Some(repo) = &site.host_repo { + forge_traffic::note_repository(&repo.name); return Ok(Some(Target { repo: repo.clone(), image: name.to_string(), @@ -392,7 +393,11 @@ impl Registry { })); }; let repo = repository_dao::find_repository_by_name(&self.inner.pool, repo_name).await?; - Ok(repo.filter(|r| self.serves(r)).map(|repo| Target { + let repo = repo.filter(|r| self.serves(r)); + if let Some(repo) = &repo { + forge_traffic::note_repository(&repo.name); + } + Ok(repo.map(|repo| Target { repo, image: image.to_string(), name: name.to_string(), diff --git a/crates/server/src/lib.rs b/crates/server/src/lib.rs index ef0198b2..b6d077e4 100644 --- a/crates/server/src/lib.rs +++ b/crates/server/src/lib.rs @@ -38,5 +38,6 @@ pub mod registry; pub mod serve; pub mod state; pub mod telemetry; +pub mod traffic; pub mod web_ui; pub mod workers; diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs index e641369a..ada10c33 100644 --- a/crates/server/src/main.rs +++ b/crates/server/src/main.rs @@ -181,6 +181,7 @@ async fn main() -> anyhow::Result<()> { let workers = workers::spawn_all(&state); let downloads = state.downloads.clone(); + let (traffic, pool) = (state.traffic.clone(), state.pool.clone()); let app = build_router(state, layer); let listener = tokio::net::TcpListener::bind(bind_addr) @@ -196,6 +197,8 @@ async fn main() -> anyhow::Result<()> { workers.abort(); // Download counts not written yet would be lost with the process. downloads.write_pending().await; + // So would the traffic counted since the last minute's flush. + forge_server::traffic::flush_on_shutdown(&traffic, &pool).await; tracing::info!("forge-server stopped"); // Export what the telemetry still buffers. tokio::task::spawn_blocking(move || telemetry.shutdown()) diff --git a/crates/server/src/middleware/principal.rs b/crates/server/src/middleware/principal.rs index b7f5d00c..7aee09e4 100644 --- a/crates/server/src/middleware/principal.rs +++ b/crates/server/src/middleware/principal.rs @@ -278,6 +278,9 @@ pub async fn attach_principal( next: Next, ) -> Response { let peer = peer_ip(&req); + let xff = forwarded_for(req.headers()); + let client_ip = resolve_client_ip(peer, xff.as_deref(), &layer.trusted_proxies); + crate::traffic::note_client(client_ip); if !req.method().is_safe() && cross_site_browser_request(req.headers(), layer.public_origin.as_deref()) @@ -296,9 +299,6 @@ pub async fn attach_principal( return next.run(req).await; } - let xff = forwarded_for(req.headers()); - let client_ip = resolve_client_ip(peer, xff.as_deref(), &layer.trusted_proxies); - let anon = match &layer.anon { AnonSource::Fixed(policy) => *policy, AnonSource::Settings(cache) => cache.current().await.anon, diff --git a/crates/server/src/plugins.rs b/crates/server/src/plugins.rs index 5fe19579..7bea6684 100644 --- a/crates/server/src/plugins.rs +++ b/crates/server/src/plugins.rs @@ -271,7 +271,10 @@ async fn dispatch_request(state: AppState, req: Request) -> Response { }; let repo = match repository_dao::find_repository_by_name(&state.pool, repo_name).await { - Ok(Some(r)) => r, + Ok(Some(r)) => { + forge_traffic::note_repository(&r.name); + r + } // As for the NuGet feed: an unknown name answers like a repository // the caller may not read, except to a global admin. Ok(None) if state.authz.is_global_admin(&principal).await => { diff --git a/crates/server/src/state.rs b/crates/server/src/state.rs index f771a682..c838cea8 100644 --- a/crates/server/src/state.rs +++ b/crates/server/src/state.rs @@ -101,6 +101,9 @@ pub struct AppState { /// This process among the instances of an install, as per-instance /// reports (the local blob cache's statistics) name it. pub instance: Arc, + /// The traffic statistics (Admin → Traffic): this instance's counts, + /// written to the object store by the traffic worker. + pub traffic: forge_traffic::Traffic, /// Forge's own OIDC issuer: the keys a `federated_credential` OIDC realm /// signs its client assertions with, published under `/.well-known/`. pub assertion_issuer: AssertionIssuer, @@ -349,6 +352,11 @@ impl AppState { )); let (pool_for_mailer, config_for_mailer) = (pool.clone(), config.clone()); + let instance = Arc::new(Instance::current()); + let traffic = forge_traffic::Traffic::new( + Arc::new(blob_store.clone()), + forge_traffic::instance_id(instance.host.as_deref(), instance.started_at), + ); AppState { downloads: DownloadTally::new(pool.clone()), pool, @@ -375,7 +383,8 @@ impl AppState { rand::rng().fill_bytes(&mut key); key }]), - instance: Arc::new(Instance::current()), + instance, + traffic, mailer: Arc::new(crate::email::Mailer::new( pool_for_mailer, envelope.clone(), diff --git a/crates/server/src/telemetry.rs b/crates/server/src/telemetry.rs index 12a2f061..e2fde0be 100644 --- a/crates/server/src/telemetry.rs +++ b/crates/server/src/telemetry.rs @@ -13,7 +13,9 @@ //! every request (joining the caller's trace from its `traceparent`) and //! the HTTP metrics: request duration by surface (from the route, never //! the raw path), method and status, the requests in flight and the -//! response body sizes. +//! response body sizes. It also counts the request in the traffic +//! statistics ([`crate::traffic`]) by day, repository and client address, +//! which the metrics' bounded attributes leave out. //! * [`install_live_source`]: what the SDK reads from the process when it //! collects (the connection pool, the blob cache's counters, those of `GET //! /admin/blob-cache`, the circuit breaker, the staging space, the @@ -27,18 +29,20 @@ use std::borrow::Cow; use std::pin::Pin; +use std::sync::Arc; use std::task::{Context, Poll}; use std::time::{Duration, Instant}; use anyhow::Context as _; use axum::body::{Body, Bytes}; -use axum::extract::{MatchedPath, Request}; +use axum::extract::{MatchedPath, Request, State}; use axum::http::{header, HeaderMap}; use axum::middleware::Next; use axum::response::Response; use forge_db::model::RepoKind; use forge_telemetry::http::Surface; use forge_telemetry::{inventory, live, storage}; +use forge_traffic::Recorder; use http_body::{Frame, SizeHint}; use opentelemetry::metrics::MeterProvider as _; use opentelemetry::propagation::Extractor; @@ -476,8 +480,9 @@ fn unsampled() -> opentelemetry::Context { } /// Middleware of the main listener: the request's server span and its -/// metrics, and the size of its response body. -pub async fn track(req: Request, next: Next) -> Response { +/// metrics, the size of its response body, and its count in the traffic +/// statistics ([`forge_traffic`]). +pub async fn track(State(recorder): State>, req: Request, next: Next) -> Response { let surface = surface_of(req.uri().path()); let method = forge_telemetry::http::method(req.method().as_str()); let route = req @@ -485,9 +490,18 @@ pub async fn track(req: Request, next: Next) -> Response { .get::() .map(|p| p.as_str().to_owned()); let span = request_span(&req, surface, route.as_deref()); + // Health probes are no traffic. + let counted = (surface != Surface::Health && recorder.enabled()).then(|| Counted { + slot: forge_traffic::Slot::new(), + recorder, + }); let started = Instant::now(); let active = forge_telemetry::http::Active::start(method); - let mut response = next.run(req).instrument(span.clone()).await; + let run = next.run(req).instrument(span.clone()); + let mut response = match &counted { + Some(counted) => counted.slot.scope(run).await, + None => run.await, + }; drop(active); let surface = response .extensions_mut() @@ -507,11 +521,19 @@ pub async fn track(req: Request, next: Next) -> Response { method, status, bytes: 0, + counted, _span: span, }) }) } +/// A request to count in the traffic statistics once its body is sent. +struct Counted { + recorder: Arc, + /// What the layers inside noted. + slot: forge_traffic::Slot, +} + /// A response body that counts the bytes it yields: their total is /// recorded, and the request's span ends, when the body is done or /// dropped (a client that went away). @@ -521,6 +543,7 @@ struct CountingBody { method: &'static str, status: u16, bytes: u64, + counted: Option, _span: tracing::Span, } @@ -554,6 +577,13 @@ impl http_body::Body for CountingBody { impl Drop for CountingBody { fn drop(&mut self) { forge_telemetry::http::response_size(self.surface, self.method, self.status, self.bytes); + if let Some(counted) = self.counted.take() { + counted.recorder.record(forge_traffic::Event { + at: chrono::Utc::now(), + bytes: self.bytes, + noted: counted.slot.take(), + }); + } } } diff --git a/crates/server/src/traffic/maxmind.rs b/crates/server/src/traffic/maxmind.rs new file mode 100644 index 00000000..9589c56c --- /dev/null +++ b/crates/server/src/traffic/maxmind.rs @@ -0,0 +1,378 @@ +//! The geolocation databases of the traffic statistics, downloaded from +//! MaxMind: GeoLite2-Country and GeoLite2-ASN, with the account ID and +//! license key an administrator sets (Admin → Traffic → Settings). +//! +//! * The license key is sealed in `forge_setting` +//! ([`forge_db::secret_dao::MAXMIND_LICENSE_KEY_SETTING`]); the account ID +//! and the downloads' outcome are kept next to it ([`STATE_SETTING`]). +//! * One instance downloads (the others see its claim on the state) and puts +//! the databases in the bucket ([`GeoKind::key`]); every instance loads them +//! from there ([`forge_traffic::geo::refresh`]). +//! * The databases are downloaded again every [`REFRESH_AFTER`] (MaxMind +//! publishes GeoLite2 twice a week, and its license asks for current +//! databases), at most every [`RETRY_AFTER`] after a failure. + +use std::io::Read; + +use chrono::{DateTime, Duration, Utc}; +use forge_crypto::{ExposeSecret, PlaintextSecret}; +use forge_db::authz_dao; +use forge_db::secret_dao::{self, MAXMIND_LICENSE_KEY_SETTING}; +use forge_traffic::GeoKind; +use serde::{Deserialize, Serialize}; + +use crate::state::AppState; + +/// The `forge_setting` key of the [`MaxmindState`]. +pub const STATE_SETTING: &str = "traffic_maxmind"; + +/// The editions downloaded, by the database they fill. +pub const EDITIONS: [(GeoKind, &str); 2] = [ + (GeoKind::Country, "GeoLite2-Country"), + (GeoKind::Asn, "GeoLite2-ASN"), +]; + +/// How old the databases may get before they are downloaded again. +pub const REFRESH_AFTER: Duration = Duration::days(3); +/// How long after an attempt the next one waits: a failure is not retried +/// every hour (MaxMind limits the downloads a day). +pub const RETRY_AFTER: Duration = Duration::hours(6); +/// The largest archive accepted, and the largest database in it. +const MAX_ARCHIVE_BYTES: usize = 256 << 20; +const MAX_DATABASE_BYTES: u64 = 512 << 20; +/// A download's deadline. +const DOWNLOAD_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(300); +/// The longest error kept. +const MAX_ERROR_BYTES: usize = 500; + +/// The MaxMind account and how its downloads went, kept in `forge_setting`. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct MaxmindState { + /// The MaxMind account ID; `None` without credentials. + pub account_id: Option, + /// When an instance last started a download. + pub last_attempt: Option>, + /// When the databases were last downloaded and stored. + pub last_success: Option>, + /// Why the last download failed, until one succeeds. + pub last_error: Option, +} + +/// The stored state and its raw value (for a compare-and-swap). +pub async fn read_state(pool: &sqlx::PgPool) -> sqlx::Result<(MaxmindState, Option)> { + let raw = authz_dao::get_setting(pool, STATE_SETTING).await?; + let state = match &raw { + None => MaxmindState::default(), + Some(raw) => serde_json::from_str(raw).map_err(|e| sqlx::Error::Decode(Box::new(e)))?, + }; + Ok((state, raw)) +} + +async fn write_state(pool: &sqlx::PgPool, state: &MaxmindState) -> sqlx::Result<()> { + let json = serde_json::to_string(state).map_err(|e| sqlx::Error::Encode(Box::new(e)))?; + authz_dao::set_setting(pool, STATE_SETTING, &json).await +} + +/// Store `state` while the stored value is still `expected` (`None`: unset). +/// Returns whether it was stored. +async fn swap_state( + pool: &sqlx::PgPool, + expected: Option<&str>, + state: &MaxmindState, +) -> sqlx::Result { + let json = serde_json::to_string(state).map_err(|e| sqlx::Error::Encode(Box::new(e)))?; + match expected { + Some(expected) => authz_dao::replace_setting(pool, STATE_SETTING, expected, &json).await, + None => authz_dao::insert_setting_if_absent(pool, STATE_SETTING, &json).await, + } +} + +/// The stored license key, opened; `None` when there is none. +pub async fn license_key(state: &AppState) -> anyhow::Result> { + let Some(value) = authz_dao::get_setting(&state.pool, MAXMIND_LICENSE_KEY_SETTING).await? + else { + return Ok(None); + }; + Ok(Some( + secret_dao::open_secret_setting(&state.envelope, MAXMIND_LICENSE_KEY_SETTING, &value) + .await?, + )) +} + +/// Whether a license key is stored. +pub async fn has_license_key(pool: &sqlx::PgPool) -> sqlx::Result { + Ok(authz_dao::get_setting(pool, MAXMIND_LICENSE_KEY_SETTING) + .await? + .is_some()) +} + +/// Store the account ID and, when given, the license key (sealed). The +/// downloads' outcome is cleared: the next check downloads with these. +pub async fn set_credentials( + state: &AppState, + account_id: &str, + license_key: Option<&str>, +) -> anyhow::Result<()> { + if let Some(key) = license_key { + let current = authz_dao::get_setting(&state.pool, MAXMIND_LICENSE_KEY_SETTING).await?; + let stored = secret_dao::store_secret_setting( + &state.pool, + &state.envelope, + MAXMIND_LICENSE_KEY_SETTING, + &PlaintextSecret::from(key.to_string()), + current.as_deref(), + ) + .await?; + if stored.is_none() { + anyhow::bail!("the license key changed while it was being saved; try again"); + } + } + write_state( + &state.pool, + &MaxmindState { + account_id: Some(account_id.to_string()), + ..MaxmindState::default() + }, + ) + .await?; + Ok(()) +} + +/// Forget the credentials and delete the databases from the bucket (every +/// instance unloads them with its next refresh). +pub async fn forget(state: &AppState) -> anyhow::Result<()> { + authz_dao::delete_setting(&state.pool, MAXMIND_LICENSE_KEY_SETTING).await?; + authz_dao::delete_setting(&state.pool, STATE_SETTING).await?; + for kind in GeoKind::ALL { + state.traffic.objects.delete(kind.key()).await?; + state.traffic.geo.unload(kind); + } + Ok(()) +} + +/// The HTTP client for MaxMind: bounded time, redirects followed (the +/// download answers with one to its storage; the credentials are not sent +/// along to another host). +fn http_client() -> reqwest::Client { + reqwest::Client::builder() + .timeout(DOWNLOAD_TIMEOUT) + .user_agent(concat!("forge/", env!("CARGO_PKG_VERSION"))) + .build() + .expect("the MaxMind HTTP client builds") +} + +fn clip(mut text: String) -> String { + if text.len() > MAX_ERROR_BYTES { + let mut end = MAX_ERROR_BYTES; + while !text.is_char_boundary(end) { + end -= 1; + } + text.truncate(end); + } + text +} + +/// The `.mmdb` file in the `.tar.gz` archive `archive`. +fn extract(archive: &[u8]) -> anyhow::Result> { + let mut tar = tar::Archive::new(flate2::read::GzDecoder::new(archive)); + for entry in tar.entries()? { + let entry = entry?; + let is_database = entry.path()?.extension().is_some_and(|ext| ext == "mmdb"); + if !is_database { + continue; + } + let size = entry.header().size()?; + if size > MAX_DATABASE_BYTES { + anyhow::bail!("the database in the archive is larger than {MAX_DATABASE_BYTES} bytes"); + } + let mut bytes = Vec::with_capacity(usize::try_from(size).unwrap_or(0)); + entry.take(MAX_DATABASE_BYTES).read_to_end(&mut bytes)?; + return Ok(bytes); + } + anyhow::bail!("the archive holds no .mmdb file") +} + +/// Download `edition` with the account `account_id` and `license_key`. +async fn download( + http: &reqwest::Client, + base: &str, + account_id: &str, + license_key: &PlaintextSecret, + edition: &str, +) -> anyhow::Result> { + let url = format!("{base}/geoip/databases/{edition}/download?suffix=tar.gz"); + let response = http + .get(&url) + .basic_auth(account_id, Some(license_key.expose_secret())) + .send() + .await + .map_err(|e| anyhow::anyhow!("cannot reach {base}: {e}"))?; + let status = response.status(); + if status == reqwest::StatusCode::UNAUTHORIZED { + anyhow::bail!("MaxMind refused the account ID or the license key (401)"); + } + if !status.is_success() { + let body = response.text().await.unwrap_or_default(); + anyhow::bail!("MaxMind answered {status} for {edition}: {}", clip(body)); + } + let mut archive = Vec::new(); + let mut response = response; + while let Some(chunk) = response.chunk().await? { + if archive.len() + chunk.len() > MAX_ARCHIVE_BYTES { + anyhow::bail!("the {edition} archive is larger than {MAX_ARCHIVE_BYTES} bytes"); + } + archive.extend_from_slice(&chunk); + } + tokio::task::spawn_blocking(move || extract(&archive)).await? +} + +/// Download both databases, check them, store them in the bucket and load +/// them here, and record the outcome. The caller has claimed the attempt. +pub async fn update(state: &AppState) -> anyhow::Result<()> { + let result = async { + let (stored, _) = read_state(&state.pool).await?; + let account_id = stored + .account_id + .ok_or_else(|| anyhow::anyhow!("no MaxMind account ID is set"))?; + let key = license_key(state) + .await? + .ok_or_else(|| anyhow::anyhow!("no MaxMind license key is set"))?; + let http = http_client(); + let base = state.config.traffic.maxmind_base(); + for (kind, edition) in EDITIONS { + let bytes = download(&http, base, &account_id, &key, edition).await?; + forge_traffic::geo::check(kind, bytes.clone()) + .map_err(|e| anyhow::anyhow!("the {edition} download is no database: {e}"))?; + state.traffic.objects.put(kind.key(), bytes).await?; + } + forge_traffic::geo::refresh(&*state.traffic.objects, &state.traffic.geo).await?; + anyhow::Ok(()) + } + .await; + let (mut recorded, _) = read_state(&state.pool).await.unwrap_or_default(); + match &result { + Ok(()) => { + recorded.last_success = Some(Utc::now()); + recorded.last_error = None; + tracing::info!("downloaded the MaxMind geolocation databases"); + } + Err(e) => { + recorded.last_error = Some(clip(format!("{e:#}"))); + tracing::warn!(error = %format!("{e:#}"), "cannot download the MaxMind databases"); + } + } + if let Err(e) = write_state(&state.pool, &recorded).await { + tracing::warn!(error = %e, "cannot record the MaxMind download"); + } + result +} + +/// Claim an attempt: record `now` as the last attempt, unless another +/// instance did since `raw` was read. Returns whether this instance won. +pub async fn claim( + pool: &sqlx::PgPool, + stored: &MaxmindState, + raw: Option<&str>, + now: DateTime, +) -> sqlx::Result { + let claimed = MaxmindState { + last_attempt: Some(now), + ..stored.clone() + }; + swap_state(pool, raw, &claimed).await +} + +/// Whether the databases are due for a download: credentials are set, the +/// last download is older than [`REFRESH_AFTER`] (or a database is missing), +/// and the last attempt is older than [`RETRY_AFTER`]. +pub fn due(stored: &MaxmindState, has_key: bool, missing: bool, now: DateTime) -> bool { + let stale = missing || stored.last_success.is_none_or(|t| now - t >= REFRESH_AFTER); + let rested = stored.last_attempt.is_none_or(|t| now - t >= RETRY_AFTER); + stored.account_id.is_some() && has_key && stale && rested +} + +/// The worker's check: download the databases if they are due and this +/// instance claims the download. +pub async fn update_if_due(state: &AppState) -> anyhow::Result<()> { + let (stored, raw) = read_state(&state.pool).await?; + if stored.account_id.is_none() { + return Ok(()); + } + let has_key = has_license_key(&state.pool).await?; + let listed = state.traffic.objects.list("traffic/v1/geoip/").await?; + let missing = GeoKind::ALL + .iter() + .any(|k| !listed.iter().any(|o| o.key == k.key())); + let now = Utc::now(); + if !due(&stored, has_key, missing, now) { + return Ok(()); + } + if !claim(&state.pool, &stored, raw.as_deref(), now).await? { + return Ok(()); + } + update(state).await +} + +#[cfg(test)] +mod tests { + use super::*; + + fn archive(name: &str, body: &[u8]) -> Vec { + let mut tar = tar::Builder::new(Vec::new()); + let mut header = tar::Header::new_gnu(); + header.set_size(body.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + tar.append_data(&mut header, name, body).unwrap(); + let tar = tar.into_inner().unwrap(); + let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + std::io::Write::write_all(&mut gz, &tar).unwrap(); + gz.finish().unwrap() + } + + #[test] + fn the_database_is_taken_from_its_archive() { + let a = archive("GeoLite2-ASN_20261030/GeoLite2-ASN.mmdb", b"database"); + assert_eq!(extract(&a).unwrap(), b"database"); + let none = archive("GeoLite2-ASN_20261030/LICENSE.txt", b"text"); + assert!(extract(&none).unwrap_err().to_string().contains("no .mmdb")); + assert!(extract(b"not gzip").is_err()); + } + + #[test] + fn downloads_are_due_when_old_and_rested() { + let now = Utc::now(); + let set = MaxmindState { + account_id: Some("1".into()), + ..MaxmindState::default() + }; + assert!(due(&set, true, false, now), "never downloaded"); + assert!(!due(&set, false, false, now), "no license key"); + assert!( + !due(&MaxmindState::default(), true, true, now), + "no account" + ); + let fresh = MaxmindState { + last_success: Some(now - Duration::hours(1)), + last_attempt: Some(now - Duration::hours(1)), + ..set.clone() + }; + assert!(!due(&fresh, true, false, now)); + let old = MaxmindState { + last_success: Some(now - Duration::days(4)), + last_attempt: Some(now - Duration::days(4)), + ..set.clone() + }; + assert!(due(&old, true, false, now)); + let failed_recently = MaxmindState { + last_attempt: Some(now - Duration::hours(1)), + last_error: Some("401".into()), + ..set + }; + assert!( + !due(&failed_recently, true, true, now), + "rests after a failure" + ); + } +} diff --git a/crates/server/src/traffic/mod.rs b/crates/server/src/traffic/mod.rs new file mode 100644 index 00000000..69f70c14 --- /dev/null +++ b/crates/server/src/traffic/mod.rs @@ -0,0 +1,215 @@ +//! The traffic statistics of Admin → Traffic ([`forge_traffic`]): what the +//! server notes about a request for its count, the settings, and the worker +//! that writes the files, loads the geolocation databases and deletes the +//! expired files. +//! +//! * The client address is the one the principal layer resolves (the socket +//! peer, or what a trusted proxy forwarded), counted under its network as +//! the login lockout counts it: an IPv6 address by its /64 +//! ([`client_network`]). +//! * The settings ([`TrafficSettings`]) are kept in `forge_setting` under +//! [`SETTINGS_KEY`]; every instance follows a change with its next flush. +//! * The worker flushes once a minute, a couple of seconds after the minute +//! turns; it loads the geolocation databases from the bucket when they +//! change ([`GEO_REFRESH_INTERVAL`]), and about once an hour, at a random +//! offset per instance, prunes and downloads the databases from MaxMind +//! when they are due ([`maxmind`]). + +pub mod maxmind; + +use std::net::IpAddr; +use std::time::Duration; + +use chrono::{DateTime, Utc}; +use forge_core::auth::login_throttle::client_network; +use forge_traffic::{Traffic, TrafficSettings}; +use sqlx::PgPool; +use tokio::task::JoinHandle; +use tracing::Instrument; + +use crate::state::AppState; + +/// The `forge_setting` key of the [`TrafficSettings`]. +pub const SETTINGS_KEY: &str = "traffic"; + +/// The worker's name (`forge.worker`). +pub const WORKER: &str = "traffic"; + +/// How often the worker prunes and checks the MaxMind download. +const HOURLY: Duration = Duration::from_secs(3_600); + +/// How often the worker looks for new geolocation databases in the bucket. +pub const GEO_REFRESH_INTERVAL: Duration = Duration::from_secs(600); + +/// How long the last flush on shutdown may take. +const SHUTDOWN_FLUSH: Duration = Duration::from_secs(10); + +/// The network `ip` is counted under, as text: the address itself for IPv4, +/// `/64` for IPv6. +pub fn network_label(ip: IpAddr) -> String { + match client_network(ip) { + IpAddr::V6(prefix) => format!("{prefix}/64"), + v4 => v4.to_string(), + } +} + +/// Note the request's client address. +pub fn note_client(ip: IpAddr) { + forge_traffic::note_client(ip.to_canonical(), network_label(ip)); +} + +/// The stored [`TrafficSettings`], or the defaults when never saved. +pub async fn read_settings(pool: &PgPool) -> sqlx::Result { + match forge_db::authz_dao::get_setting(pool, SETTINGS_KEY).await? { + None => Ok(TrafficSettings::default()), + Some(raw) => serde_json::from_str(&raw).map_err(|e| sqlx::Error::Decode(Box::new(e))), + } +} + +/// Store the [`TrafficSettings`] for every instance. +pub async fn write_settings(pool: &PgPool, settings: &TrafficSettings) -> sqlx::Result<()> { + let json = serde_json::to_string(settings).map_err(|e| sqlx::Error::Encode(Box::new(e)))?; + forge_db::authz_dao::set_setting(pool, SETTINGS_KEY, &json).await +} + +/// The stored settings, applied to this instance's recorder; the ones +/// applied before when they cannot be read. +async fn follow_settings( + pool: &PgPool, + traffic: &Traffic, + last: &TrafficSettings, +) -> TrafficSettings { + match read_settings(pool).await { + Ok(settings) => { + traffic.recorder.set_enabled(settings.enabled); + settings + } + Err(e) => { + forge_db::observe_error(&e); + tracing::warn!(error = %e, "cannot read the traffic settings; keeping the ones in force"); + last.clone() + } + } +} + +/// Load the geolocation databases from the bucket when they changed. +pub async fn refresh_geo(traffic: &Traffic) -> bool { + match forge_traffic::geo::refresh(&*traffic.objects, &traffic.geo).await { + Ok(()) => true, + Err(e) => { + tracing::warn!(error = %format!("{e:#}"), "cannot look for geolocation databases"); + false + } + } +} + +/// How long until `now`'s next minute is two seconds old. +fn until_next_flush(now: DateTime) -> Duration { + let into_minute = + now.timestamp().rem_euclid(60) * 1_000 + i64::from(now.timestamp_subsec_millis()); + Duration::from_millis(u64::try_from(62_000 - into_minute).unwrap_or(0)) +} + +/// The worker: flush every minute, look for new geolocation databases every +/// ten, prune and check the MaxMind download every hour. +pub fn spawn_worker(state: AppState) -> JoinHandle<()> { + tokio::spawn(async move { + let traffic = state.traffic.clone(); + let mut settings = + follow_settings(&state.pool, &traffic, &TrafficSettings::default()).await; + refresh_geo(&traffic).await; + let mut next_geo = tokio::time::Instant::now() + GEO_REFRESH_INTERVAL; + // Instances started together prune and download at different times. + let mut next_hourly = tokio::time::Instant::now() + + Duration::from_secs(rand::random_range(60..HOURLY.as_secs())); + loop { + tokio::time::sleep(until_next_flush(Utc::now())).await; + let run = forge_telemetry::workers::Run::start(WORKER); + let ok = async { + settings = follow_settings(&state.pool, &traffic, &settings).await; + let mut ok = true; + if tokio::time::Instant::now() >= next_geo { + next_geo = tokio::time::Instant::now() + GEO_REFRESH_INTERVAL; + ok &= refresh_geo(&traffic).await; + } + ok &= traffic.flush(&settings, Utc::now()).await.failures == 0; + if tokio::time::Instant::now() >= next_hourly { + next_hourly = tokio::time::Instant::now() + HOURLY; + match forge_traffic::layout::prune(&*traffic.objects, &settings, Utc::now()) + .await + { + Ok(0) => {} + Ok(deleted) => { + tracing::info!(deleted, "deleted expired traffic statistics"); + } + Err(e) => { + ok = false; + tracing::warn!(error = %e, "cannot delete expired traffic statistics"); + } + } + if let Err(e) = maxmind::update_if_due(&state).await { + ok = false; + tracing::warn!( + error = %format!("{e:#}"), + "the MaxMind geolocation download failed" + ); + } + } + ok + } + .instrument(run.span()) + .await; + run.finish(ok); + } + }) +} + +/// Write what this instance counted and has not written yet, before it +/// exits: at most [`SHUTDOWN_FLUSH`]. +pub async fn flush_on_shutdown(traffic: &Traffic, pool: &PgPool) { + let flush = async { + let settings = read_settings(pool).await.unwrap_or_default(); + traffic.flush(&settings, Utc::now()).await + }; + match tokio::time::timeout(SHUTDOWN_FLUSH, flush).await { + Ok(report) if report.failures > 0 => { + tracing::warn!( + ?report, + "could not write all the traffic statistics on shutdown" + ); + } + Ok(_) => {} + Err(_) => tracing::warn!("writing the traffic statistics on shutdown timed out"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ipv6_clients_are_counted_by_their_64() { + assert_eq!(network_label("192.0.2.7".parse().unwrap()), "192.0.2.7"); + assert_eq!( + network_label("::ffff:192.0.2.7".parse().unwrap()), + "192.0.2.7" + ); + assert_eq!( + network_label("2001:db8:1:2:3:4:5:6".parse().unwrap()), + "2001:db8:1:2::/64" + ); + } + + #[test] + fn the_flush_waits_for_the_minute_to_turn() { + let at = |s: &str| DateTime::parse_from_rfc3339(s).unwrap().to_utc(); + assert_eq!( + until_next_flush(at("2026-10-30T14:00:00Z")), + Duration::from_secs(62) + ); + assert_eq!( + until_next_flush(at("2026-10-30T14:00:59.500Z")), + Duration::from_millis(2_500) + ); + } +} diff --git a/crates/server/src/workers.rs b/crates/server/src/workers.rs index 3bdf678c..6fb09d08 100644 --- a/crates/server/src/workers.rs +++ b/crates/server/src/workers.rs @@ -60,6 +60,11 @@ //! bytes, and the outbox backlog for the metrics //! ([`crate::telemetry::spawn_inventory`]). Read-only, per instance. //! +//! 10. **Traffic statistics** — writes this instance's request counts, top +//! clients and request log to the object store every minute, and deletes +//! the files past their retention every hour ([`crate::traffic`]). Each +//! instance writes only files of its own; deleting is idempotent. +//! //! Every loop reports to the metrics (`forge.worker.*`): its restarts, and //! for the periodic ones each pass's outcome and duration, the time of the //! last successful one and the interval it runs at. Each pass is a trace of @@ -121,6 +126,8 @@ pub struct WorkerHandles { /// Counts the inventory for the metrics. Present only with the metrics /// exported (`telemetry.metrics`). pub metrics_inventory: Option>, + /// Writes and prunes the traffic statistics. + pub traffic: JoinHandle<()>, } impl WorkerHandles { @@ -147,6 +154,7 @@ impl WorkerHandles { if let Some(h) = &self.metrics_inventory { h.abort(); } + self.traffic.abort(); } } @@ -337,6 +345,15 @@ pub fn spawn_all(state: &AppState) -> WorkerHandles { }) }); + // The traffic statistics: every instance writes its own files. + let traffic = { + let state = state.clone(); + forge_telemetry::workers::set_interval(crate::traffic::WORKER, Duration::from_secs(60)); + supervise(crate::traffic::WORKER, move || { + crate::traffic::spawn_worker(state.clone()) + }) + }; + WorkerHandles { outbox, reconciler, @@ -351,6 +368,7 @@ pub fn spawn_all(state: &AppState) -> WorkerHandles { blob_cache, import, metrics_inventory, + traffic, } } diff --git a/crates/server/tests/traffic.rs b/crates/server/tests/traffic.rs new file mode 100644 index 00000000..020f2c44 --- /dev/null +++ b/crates/server/tests/traffic.rs @@ -0,0 +1,381 @@ +//! E2E: the traffic statistics (Admin → Traffic). Requires Docker +//! (Postgres + versitygw via the shared harness), or +//! `FORGE_TEST_DATABASE_URL` / `FORGE_TEST_S3_ENDPOINT`. +//! +//! Requests are counted by day, repository and client network, written to +//! the object store by a flush and read back by the admin API; the +//! geolocation databases are downloaded from a mock of MaxMind's download +//! service, stored in the bucket and used to locate the clients. + +mod common; + +use common::{unique, TestServer}; +use forge_traffic::testing::{asn_db, country_db}; +use forge_traffic::TrafficSettings; +use reqwest::StatusCode; +use serde_json::{json, Value}; +use wiremock::matchers::{header, method, path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +async fn server(extra_toml: Option) -> TestServer { + common::start_with( + common::fresh_pool().await, + common::ServerOpts { + bootstrap_admin: Some((format!("admin{}@example.com", unique()), "adminpw".into())), + extra_toml, + ..Default::default() + }, + ) + .await +} + +async fn admin( + srv: &TestServer, + verb: &str, + path: &str, + body: Option, +) -> (StatusCode, Value) { + let (u, p) = srv.admin_basic(); + let url = format!("{}{path}", srv.base_url); + let client = srv.client(); + let request = match verb { + "GET" => client.get(url), + "PUT" => client.put(url), + "POST" => client.post(url), + "DELETE" => client.delete(url), + _ => unreachable!(), + }; + let request = match body { + Some(body) => request.json(&body), + None => request, + }; + let resp = request.basic_auth(&u, Some(&p)).send().await.unwrap(); + let status = resp.status(); + let text = resp.text().await.unwrap(); + ( + status, + serde_json::from_str(&text).unwrap_or(Value::String(text)), + ) +} + +async fn create_raw(srv: &TestServer) -> String { + let name = format!("traffic{}", unique()); + let (status, body) = admin( + srv, + "POST", + "/admin/repositories", + Some(json!({"name": name, "kind": "hosted", "format": "raw"})), + ) + .await; + assert_eq!(status, StatusCode::CREATED, "{body}"); + name +} + +/// A `.tar.gz` holding `database` as MaxMind ships it. +fn archive(edition: &str, database: &[u8]) -> Vec { + let mut tar = tar::Builder::new(Vec::new()); + let mut header = tar::Header::new_gnu(); + header.set_size(database.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + tar.append_data( + &mut header, + format!("{edition}_20261030/{edition}.mmdb"), + database, + ) + .unwrap(); + let tar = tar.into_inner().unwrap(); + let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + std::io::Write::write_all(&mut gz, &tar).unwrap(); + gz.finish().unwrap() +} + +#[tokio::test] +async fn requests_are_counted_by_day_repository_and_client() { + let srv = server(None).await; + let (u, p) = srv.admin_basic(); + let client = srv.client(); + let name = create_raw(&srv).await; + + // Three requests to the repository, a health probe that is not counted, + // and a repository that does not exist (its name is not recorded). + let file = format!("{}/repository/{name}/dir/hello.txt", srv.base_url); + let resp = client + .put(&file) + .basic_auth(&u, Some(&p)) + .body("hello") + .send() + .await + .unwrap(); + assert!(resp.status().is_success(), "upload: {}", resp.status()); + let resp = client + .get(&file) + .basic_auth(&u, Some(&p)) + .send() + .await + .unwrap(); + assert_eq!(resp.text().await.unwrap(), "hello"); + let resp = client.get(&file).send().await.unwrap(); + assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); + client + .get(format!("{}/healthz", srv.base_url)) + .send() + .await + .unwrap(); + client + .get(format!( + "{}/repository/no-such-{}/x", + srv.base_url, + unique() + )) + .send() + .await + .unwrap(); + + // What a minute's flush writes, written now. + let traffic = &srv.state.traffic; + let report = traffic + .flush(&TrafficSettings::default(), chrono::Utc::now()) + .await; + assert_eq!(report.failures, 0, "{report:?}"); + let instance = traffic.instance.to_string(); + + let (status, overview) = admin( + &srv, + "GET", + &format!("/admin/traffic?range=today&instance={instance}"), + None, + ) + .await; + assert_eq!(status, StatusCode::OK, "{overview}"); + assert!( + overview["instances"] + .as_array() + .unwrap() + .iter() + .any(|i| i["id"] == instance.as_str()), + "{overview}" + ); + assert_eq!(overview["days"].as_array().unwrap().len(), 1); + let repositories = overview["repositories"].as_array().unwrap(); + let repo = repositories + .iter() + .find(|r| r["name"] == name.as_str()) + .expect("the repository is listed"); + assert_eq!(repo["requests"], 3); + assert_eq!(repo["clients"], 1); + assert!( + repositories + .iter() + .all(|r| !r["name"].as_str().unwrap().starts_with("no-such-")), + "only repositories that exist: {overview}" + ); + let top = &overview["clients"][0]; + assert_eq!(top["network"], "127.0.0.1"); + assert_eq!(top["repositories"][0], name.as_str()); + assert_eq!(overview["recording"]["countries"], false, "no database yet"); + + // One repository's requests, then the requests to none. + let (_, only) = admin( + &srv, + "GET", + &format!("/admin/traffic?range=today&instance={instance}&repository={name}"), + None, + ) + .await; + assert_eq!(only["totals"]["requests"], 3); + assert!( + only["totals"]["bytes"].as_u64().unwrap() >= 5, + "the response bodies, the download's included: {only}" + ); + let (_, none) = admin( + &srv, + "GET", + &format!("/admin/traffic?range=today&instance={instance}&repository=-"), + None, + ) + .await; + assert!(none["totals"]["requests"].as_u64().unwrap() >= 2, "{none}"); + assert!(none["repositories"] + .as_array() + .unwrap() + .iter() + .all(|r| r["name"] == "")); + + let (status, _) = admin(&srv, "GET", "/admin/traffic?range=2h", None).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + // Admins only. + let resp = client + .get(format!("{}/admin/traffic", srv.base_url)) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); +} + +#[tokio::test] +async fn settings_are_checked_and_applied() { + let srv = server(None).await; + let (status, settings) = admin(&srv, "GET", "/admin/settings/traffic", None).await; + assert_eq!(status, StatusCode::OK); + assert_eq!( + serde_json::from_value::(settings).unwrap(), + TrafficSettings::default() + ); + + let (status, settings) = admin( + &srv, + "PUT", + "/admin/settings/traffic", + Some(json!({"enabled": false, "retention_days": 30})), + ) + .await; + assert_eq!(status, StatusCode::OK, "{settings}"); + assert_eq!(settings["retention_days"], 30); + assert_eq!(settings["client_addresses"], true, "untouched"); + assert!(!srv.state.traffic.recorder.enabled(), "applied at once"); + + let (status, _) = admin( + &srv, + "PUT", + "/admin/settings/traffic", + Some(json!({"retention_days": 0})), + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST); + let (status, _) = admin( + &srv, + "PUT", + "/admin/settings/traffic", + Some(json!({"unknown": true})), + ) + .await; + assert!(status.is_client_error()); + let (_, settings) = admin(&srv, "GET", "/admin/settings/traffic", None).await; + assert_eq!(settings["enabled"], false, "stored"); +} + +#[tokio::test] +async fn maxmind_databases_are_downloaded_and_locate_clients() { + let maxmind = MockServer::start().await; + // "1234:secret", as MaxMind's download expects it. + let auth = "Basic MTIzNDpzZWNyZXQ="; + for (edition, database) in [ + ("GeoLite2-Country", country_db(&[("127.0.0.0/8", "DE")])), + ( + "GeoLite2-ASN", + asn_db(&[("127.0.0.0/8", 64_500, "Example AS")]), + ), + ] { + Mock::given(method("GET")) + .and(path(format!("/geoip/databases/{edition}/download"))) + .and(header("authorization", auth)) + .respond_with(ResponseTemplate::new(200).set_body_bytes(archive(edition, &database))) + .mount(&maxmind) + .await; + } + Mock::given(method("GET")) + .respond_with(ResponseTemplate::new(401).set_body_string("Invalid license key")) + .mount(&maxmind) + .await; + let srv = server(Some(format!( + "[traffic]\nmaxmind_url = \"{}\"\n", + maxmind.uri() + ))) + .await; + + let (status, view) = admin(&srv, "GET", "/admin/traffic/geoip", None).await; + assert_eq!(status, StatusCode::OK, "{view}"); + assert_eq!(view["maxmind"]["license_key_set"], false); + let (status, _) = admin(&srv, "POST", "/admin/traffic/geoip/update", None).await; + assert_eq!(status, StatusCode::CONFLICT, "no credentials yet"); + let (status, _) = admin( + &srv, + "PUT", + "/admin/traffic/geoip", + Some(json!({"account_id": "1234"})), + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST, "a license key is needed"); + + // A wrong key: the download fails and says why. + let (status, view) = admin( + &srv, + "PUT", + "/admin/traffic/geoip", + Some(json!({"account_id": "1234", "license_key": "wrong"})), + ) + .await; + assert_eq!(status, StatusCode::OK, "{view}"); + assert_eq!(view["maxmind"]["license_key_set"], true); + assert!( + view["maxmind"].get("license_key").is_none(), + "never answered" + ); + let (_, view) = admin(&srv, "POST", "/admin/traffic/geoip/update", None).await; + assert!( + view["maxmind"]["last_error"] + .as_str() + .unwrap() + .contains("refused the account ID or the license key"), + "{view}" + ); + let sealed = forge_db::authz_dao::get_setting(&srv.pool, "traffic_maxmind_license_key") + .await + .unwrap() + .unwrap(); + assert!( + sealed.starts_with("sealed:") && !sealed.contains("wrong"), + "sealed at rest" + ); + + // The right key (the account ID alone keeps it otherwise). + admin( + &srv, + "PUT", + "/admin/traffic/geoip", + Some(json!({"account_id": "1234", "license_key": "secret"})), + ) + .await; + let (status, view) = admin(&srv, "POST", "/admin/traffic/geoip/update", None).await; + assert_eq!(status, StatusCode::OK, "{view}"); + assert_eq!(view["maxmind"]["last_error"], Value::Null, "{view}"); + assert_eq!(view["country"]["database_type"], "GeoLite2-Country"); + assert_eq!(view["asn"]["database_type"], "GeoLite2-ASN"); + + // A request is now located. + let name = create_raw(&srv).await; + srv.client() + .get(format!("{}/repository/{name}/x", srv.base_url)) + .send() + .await + .unwrap(); + let traffic = &srv.state.traffic; + traffic + .flush(&TrafficSettings::default(), chrono::Utc::now()) + .await; + let instance = traffic.instance.to_string(); + let (_, overview) = admin( + &srv, + "GET", + &format!("/admin/traffic?range=today&instance={instance}&country=DE"), + None, + ) + .await; + assert!( + overview["totals"]["requests"].as_u64().unwrap() >= 1, + "{overview}" + ); + assert_eq!(overview["countries"][0]["country"], "DE"); + assert_eq!(overview["asns"][0]["asn"], 64_500); + assert_eq!(overview["asns"][0]["org"], "Example AS"); + assert_eq!(overview["clients"][0]["country"], "DE"); + assert_eq!(overview["recording"]["countries"], true); + + // Turned off: the credentials and the databases are gone. + let (status, _) = admin(&srv, "DELETE", "/admin/traffic/geoip", None).await; + assert_eq!(status, StatusCode::NO_CONTENT); + let (_, view) = admin(&srv, "GET", "/admin/traffic/geoip", None).await; + assert_eq!(view["country"], Value::Null); + assert_eq!(view["maxmind"]["license_key_set"], false); +} diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index 4b9c4f96..efa14ae3 100644 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -31,4 +31,4 @@ pub use blob_cache::{BlobCache, BlobCacheConfig, BlobCacheStats, BlobStream, Cac pub use cas::{Cas, CasError, SizedStage, StageReader, StagedBlob, Staging, StoredBlob}; pub use credentials::{S3Credentials, WebIdentity}; pub use read::BlobRef; -pub use s3::{BlobStore, ConditionalWrites, S3Config, S3Error}; +pub use s3::{BlobStore, ConditionalWrites, ListedObject, Listing, S3Config, S3Error}; diff --git a/crates/storage/src/s3.rs b/crates/storage/src/s3.rs index b2e239c7..62734944 100644 --- a/crates/storage/src/s3.rs +++ b/crates/storage/src/s3.rs @@ -89,6 +89,23 @@ pub enum ConditionalWrites { Off, } +/// The answer of [`BlobStore::list`]. +#[derive(Debug, Clone, Default)] +pub struct Listing { + pub objects: Vec, + /// The common prefixes, each ending with the delimiter. + pub prefixes: Vec, +} + +/// An object [`BlobStore::list`] found. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ListedObject { + pub key: String, + pub size: u64, + /// Changes whenever the object is replaced. + pub etag: Option, +} + /// Connection + behavior configuration for [`BlobStore`]. `Debug` omits the /// secret key. #[derive(Debug, Clone)] @@ -388,7 +405,21 @@ impl BlobStore { /// Single-part bodies may use a conditional PUT (`If-None-Match: *`) /// instead of HEAD-then-PUT, per [`ConditionalWrites`]; the two paths are /// observably identical, the conditional one just saves a round trip. - pub async fn put_if_absent(&self, key: &str, path: &Path) -> Result { + /// + /// The upload is boxed: the SDK's request futures are large, and a caller + /// would otherwise hold one inline, and so would every future awaiting + /// it, up the whole request (a deep stack in a debug build). + pub fn put_if_absent<'a>( + &'a self, + key: &'a str, + path: &'a Path, + ) -> std::pin::Pin> + Send + 'a>> + { + Box::pin(self.upload_if_absent(key, path)) + } + + /// [`Self::put_if_absent`], unboxed. + async fn upload_if_absent(&self, key: &str, path: &Path) -> Result { self.admit(S3Op::Put)?; let size = tokio::fs::metadata(path).await?.len(); @@ -455,8 +486,19 @@ impl BlobStore { /// One conditional PUT attempt. Maps the interesting backend answers /// (412 / NotImplemented / 409) into [`ConditionalPut`]; anything else is - /// a real error. - async fn put_conditional(&self, key: &str, path: &Path) -> Result { + /// a real error. Boxed, like [`Self::put_if_absent`]. + fn put_conditional<'a>( + &'a self, + key: &'a str, + path: &'a Path, + ) -> std::pin::Pin< + Box> + Send + 'a>, + > { + Box::pin(self.send_conditional(key, path)) + } + + /// [`Self::put_conditional`], unboxed. + async fn send_conditional(&self, key: &str, path: &Path) -> Result { let body = body_from_path(path).await?; let call = S3Call::start(S3Op::PutConditional, &self.bucket); let result = self @@ -667,6 +709,94 @@ impl BlobStore { Ok(out.body) } + /// Store `body` at `key`, replacing what is there: for objects Forge + /// rewrites (the traffic statistics), never for content-addressed blobs + /// ([`Self::put_if_absent`]). + pub async fn put_bytes(&self, key: &str, body: Vec) -> Result<(), S3Error> { + self.admit(S3Op::Put)?; + self.call( + S3Op::Put, + self.client + .put_object() + .bucket(&self.bucket) + .key(key) + .body(ByteStream::from(body)) + .send(), + ) + .await + .map(|_| ()) + } + + /// The whole object at `key`, or `None` when there is none. + pub async fn get_bytes(&self, key: &str) -> Result, S3Error> { + self.admit(S3Op::Get)?; + let call = S3Call::start(S3Op::Get, &self.bucket); + let result = self + .client + .get_object() + .bucket(&self.bucket) + .key(key) + .send() + .instrument(call.span()) + .await; + self.observe(call, &result); + let out = match result { + Ok(out) => out, + Err(e) if is_not_found(&e) => return Ok(None), + Err(e) => return Err(self.error(e)), + }; + let body = out + .body + .collect() + .await + .map_err(|e| S3Error::S3(format!("read {key}: {e}")))?; + Ok(Some(body.into_bytes())) + } + + /// The objects whose keys start with `prefix`. With `delimiter`, keys + /// that continue past the next delimiter after the prefix are listed + /// once each, as a common prefix (a "directory"), instead. + pub async fn list(&self, prefix: &str, delimiter: Option<&str>) -> Result { + let mut listing = Listing::default(); + let mut token: Option = None; + loop { + self.admit(S3Op::List)?; + let page = self + .call( + S3Op::List, + self.client + .list_objects_v2() + .bucket(&self.bucket) + .prefix(prefix) + .set_delimiter(delimiter.map(str::to_string)) + .set_continuation_token(token.take()) + .send(), + ) + .await?; + listing + .objects + .extend(page.contents().iter().filter_map(|object| { + Some(ListedObject { + key: object.key()?.to_string(), + size: object + .size() + .and_then(|s| u64::try_from(s).ok()) + .unwrap_or(0), + etag: object.e_tag().map(str::to_string), + }) + })); + listing.prefixes.extend( + page.common_prefixes() + .iter() + .filter_map(|p| p.prefix().map(str::to_string)), + ); + match page.next_continuation_token() { + Some(next) if page.is_truncated() == Some(true) => token = Some(next.to_string()), + _ => return Ok(listing), + } + } + } + /// Delete the object at `key`. Idempotent: a missing object is `Ok(())`. pub async fn delete(&self, key: &str) -> Result<(), S3Error> { self.admit(S3Op::Delete)?; diff --git a/crates/telemetry/src/lib.rs b/crates/telemetry/src/lib.rs index 76d0815f..9af9a6d3 100644 --- a/crates/telemetry/src/lib.rs +++ b/crates/telemetry/src/lib.rs @@ -641,6 +641,7 @@ pub mod storage { CompleteMultipart => "CompleteMultipartUpload", AbortMultipart => "AbortMultipartUpload", Get => "GetObject", + List => "ListObjectsV2", Delete => "DeleteObject", } } diff --git a/crates/traffic/Cargo.toml b/crates/traffic/Cargo.toml new file mode 100644 index 00000000..4b897f4a --- /dev/null +++ b/crates/traffic/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "forge-traffic" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +# The object store the statistics are kept in (BlobStore implements +# `Objects`). +forge-storage = { path = "../storage" } + +tokio = { workspace = true, features = ["sync", "time", "rt"] } +parquet = { workspace = true } +parquet_derive = { workspace = true } +# Country and ASN lookups in MaxMind databases (GeoLite2, GeoIP2). +maxminddb = { workspace = true } +bytes = "1" +chrono = { workspace = true, features = ["serde"] } +serde = { workspace = true } +anyhow = { workspace = true } +async-trait = { workspace = true } +futures-util = { workspace = true } +moka = { workspace = true } +tracing = { workspace = true } +rand = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["rt-multi-thread", "macros", "sync", "time"] } +serde_json = { workspace = true } + +[features] +# `testing`: a writer of small MaxMind databases, for the tests of the crates +# that use this one. +testing = [] diff --git a/crates/traffic/src/geo.rs b/crates/traffic/src/geo.rs new file mode 100644 index 00000000..43c8efb2 --- /dev/null +++ b/crates/traffic/src/geo.rs @@ -0,0 +1,349 @@ +//! The country and autonomous system of a client address, from MaxMind +//! databases (`.mmdb`: GeoLite2 or GeoIP2). +//! +//! Two databases, each optional: one with countries (a Country or City +//! edition) and one with autonomous systems (an ASN or ISP edition). They +//! are kept in Forge's bucket ([`GeoKind::key`]), where Forge's download from +//! MaxMind puts them or an operator does, and every instance loads them from +//! there ([`refresh`]): a database replaced in the bucket is loaded again, +//! one removed is unloaded. + +use std::net::IpAddr; +use std::sync::{Arc, RwLock}; + +use chrono::{DateTime, TimeZone, Utc}; +use maxminddb::{geoip2, Reader}; +use serde::Serialize; + +use crate::objects::Objects; + +/// The two databases. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum GeoKind { + Country, + Asn, +} + +impl GeoKind { + pub const ALL: [GeoKind; 2] = [GeoKind::Country, GeoKind::Asn]; + + /// Where the database is kept in the bucket. + pub fn key(self) -> &'static str { + match self { + GeoKind::Country => "traffic/v1/geoip/country.mmdb", + GeoKind::Asn => "traffic/v1/geoip/asn.mmdb", + } + } + + /// Whether a database of `database_type` (its metadata's) answers this + /// kind: a Country or City edition for countries, an ASN or ISP edition + /// for autonomous systems. + fn accepts(self, database_type: &str) -> bool { + let t = database_type.to_ascii_lowercase(); + match self { + GeoKind::Country => t.contains("country") || t.contains("city"), + GeoKind::Asn => t.contains("asn") || t.contains("isp"), + } + } +} + +/// What a loaded database is. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct DbInfo { + /// Its edition, as its metadata names it (`GeoLite2-Country`). + pub database_type: String, + /// When MaxMind built it. + pub built_at: Option>, + pub size: u64, + /// The bucket object's ETag it was loaded from. + #[serde(skip)] + pub etag: String, +} + +struct Db { + reader: Reader>, + info: DbInfo, +} + +/// Where an address is. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Location { + /// ISO 3166-1 alpha-2; `""` unknown. + pub country: String, + /// `0` unknown. + pub asn: u32, + /// `""` unknown. + pub as_org: String, +} + +/// The loaded databases. +#[derive(Default)] +pub struct Geo { + country: RwLock>>, + asn: RwLock>>, +} + +impl std::fmt::Debug for Geo { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Geo") + .field("country", &self.info(GeoKind::Country)) + .field("asn", &self.info(GeoKind::Asn)) + .finish() + } +} + +/// `bytes` as a database of `kind`, checked: it parses, and its edition +/// answers `kind`. +fn open(kind: GeoKind, bytes: Vec, etag: String) -> anyhow::Result { + let size = bytes.len() as u64; + let reader = Reader::from_source(bytes)?; + let meta = reader.metadata(); + let database_type = meta.database_type.clone(); + if !kind.accepts(&database_type) { + anyhow::bail!( + "{database_type} is not a {} database", + match kind { + GeoKind::Country => "country (Country or City)", + GeoKind::Asn => "autonomous system (ASN or ISP)", + } + ); + } + let built_at = i64::try_from(meta.build_epoch) + .ok() + .and_then(|secs| Utc.timestamp_opt(secs, 0).single()); + Ok(Db { + reader, + info: DbInfo { + database_type, + built_at, + size, + etag, + }, + }) +} + +/// Check that `bytes` is a database of `kind`; what it is. +pub fn check(kind: GeoKind, bytes: Vec) -> anyhow::Result { + Ok(open(kind, bytes, String::new())?.info) +} + +impl Geo { + fn slot(&self, kind: GeoKind) -> &RwLock>> { + match kind { + GeoKind::Country => &self.country, + GeoKind::Asn => &self.asn, + } + } + + fn get(&self, kind: GeoKind) -> Option> { + self.slot(kind) + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() + } + + fn set(&self, kind: GeoKind, db: Option>) { + *self + .slot(kind) + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) = db; + } + + /// Load `bytes`, the bucket object of ETag `etag`, as the database of + /// `kind`. + pub fn load(&self, kind: GeoKind, bytes: Vec, etag: String) -> anyhow::Result { + let db = open(kind, bytes, etag)?; + let info = db.info.clone(); + self.set(kind, Some(Arc::new(db))); + Ok(info) + } + + /// Unload the database of `kind`. + pub fn unload(&self, kind: GeoKind) { + self.set(kind, None); + } + + /// What is loaded as the database of `kind`. + pub fn info(&self, kind: GeoKind) -> Option { + self.get(kind).map(|db| db.info.clone()) + } + + /// Whether any database is loaded. + pub fn any(&self) -> bool { + GeoKind::ALL.iter().any(|k| self.get(*k).is_some()) + } + + /// Where `ip` is, as far as the loaded databases know. + pub fn locate(&self, ip: IpAddr) -> Location { + let ip = ip.to_canonical(); + let mut location = Location::default(); + if let Some(db) = self.get(GeoKind::Country) { + if let Ok(Some(c)) = db + .reader + .lookup(ip) + .and_then(|r| r.decode::()) + { + location.country = c + .country + .iso_code + .or(c.registered_country.iso_code) + .unwrap_or_default() + .to_string(); + } + } + if let Some(db) = self.get(GeoKind::Asn) { + if let Ok(Some(a)) = db.reader.lookup(ip).and_then(|r| r.decode::()) { + location.asn = a.autonomous_system_number.unwrap_or(0); + location.as_org = a + .autonomous_system_organization + .unwrap_or_default() + .to_string(); + } + } + location + } +} + +/// Load the databases kept in the bucket that changed since they were +/// loaded, and unload the ones removed. A database that cannot be read is +/// logged and left as it was. +pub async fn refresh(objects: &dyn Objects, geo: &Geo) -> anyhow::Result<()> { + let listed = objects.list("traffic/v1/geoip/").await?; + for kind in GeoKind::ALL { + let Some(object) = listed.iter().find(|o| o.key == kind.key()) else { + if geo.info(kind).is_some() { + tracing::info!(?kind, "the geolocation database was removed; unloading it"); + geo.unload(kind); + } + continue; + }; + let etag = object.etag.clone().unwrap_or_default(); + if geo + .info(kind) + .is_some_and(|i| i.etag == etag && !etag.is_empty()) + { + continue; + } + let Some(bytes) = objects.get(kind.key()).await? else { + continue; + }; + match geo.load(kind, bytes.to_vec(), etag) { + Ok(info) => tracing::info!( + database = %info.database_type, + built_at = ?info.built_at, + "loaded the geolocation database" + ), + Err(e) => tracing::warn!( + error = %e, + key = kind.key(), + "cannot load the geolocation database" + ), + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::objects::MemoryObjects; + use crate::testing::{asn_db, country_db}; + + #[test] + fn databases_locate_addresses() { + let geo = Geo::default(); + assert_eq!( + geo.locate("192.0.2.1".parse().unwrap()), + Location::default() + ); + geo.load( + GeoKind::Country, + country_db(&[("192.0.2.0/24", "DE")]), + "1".into(), + ) + .unwrap(); + geo.load( + GeoKind::Asn, + asn_db(&[("192.0.2.0/24", 64_500, "Example AS")]), + "1".into(), + ) + .unwrap(); + assert_eq!( + geo.locate("192.0.2.77".parse().unwrap()), + Location { + country: "DE".into(), + asn: 64_500, + as_org: "Example AS".into() + } + ); + assert_eq!( + geo.locate("::ffff:192.0.2.77".parse().unwrap()).country, + "DE", + "a mapped address is its IPv4 address" + ); + assert_eq!( + geo.locate("198.51.100.1".parse().unwrap()), + Location::default() + ); + assert_eq!( + geo.locate("2001:db8::1".parse().unwrap()), + Location::default(), + "an IPv6 address in an IPv4 database is unknown" + ); + assert_eq!( + geo.info(GeoKind::Asn).unwrap().database_type, + "GeoLite2-ASN" + ); + } + + #[test] + fn a_database_of_the_other_kind_is_refused() { + let geo = Geo::default(); + let err = geo + .load( + GeoKind::Country, + asn_db(&[("192.0.2.0/24", 1, "x")]), + "1".into(), + ) + .unwrap_err(); + assert!(err.to_string().contains("not a country"), "{err}"); + assert!(geo + .load(GeoKind::Asn, b"no database".to_vec(), "1".into()) + .is_err()); + assert!(!geo.any()); + } + + #[tokio::test] + async fn refresh_follows_the_bucket() { + let objects = MemoryObjects::default(); + let geo = Geo::default(); + objects + .put( + GeoKind::Country.key(), + country_db(&[("192.0.2.0/24", "DE")]), + ) + .await + .unwrap(); + refresh(&objects, &geo).await.unwrap(); + assert_eq!(geo.locate("192.0.2.1".parse().unwrap()).country, "DE"); + + objects + .put( + GeoKind::Country.key(), + country_db(&[("192.0.2.0/24", "FR")]), + ) + .await + .unwrap(); + refresh(&objects, &geo).await.unwrap(); + assert_eq!( + geo.locate("192.0.2.1".parse().unwrap()).country, + "FR", + "replaced" + ); + + objects.delete(GeoKind::Country.key()).await.unwrap(); + refresh(&objects, &geo).await.unwrap(); + assert!(!geo.any(), "removed"); + } +} diff --git a/crates/traffic/src/layout.rs b/crates/traffic/src/layout.rs new file mode 100644 index 00000000..6c431621 --- /dev/null +++ b/crates/traffic/src/layout.rs @@ -0,0 +1,538 @@ +//! Where the files are, what writes them and what deletes them. +//! +//! Every key is under `traffic/v1/` in Forge's bucket: +//! +//! * `days/{date}/{instance}.parquet`: one instance's requests of a day +//! (UTC), a row per repository and client network with their country and +//! autonomous system; rewritten every minute while the day lasts. Kept as +//! long as the settings say ([`TrafficSettings::retention_days`]). +//! * `geoip/{country,asn}.mmdb`: the geolocation databases +//! ([`crate::geo`]). +//! +//! Each instance writes only files named after itself ([`instance_id`]), so +//! two instances never write the same object and a restarted process (a new +//! name) never overwrites what its predecessor wrote. + +use std::collections::{BTreeMap, HashMap}; +use std::sync::Arc; + +use chrono::{DateTime, NaiveDate, TimeZone, Utc}; +use futures_util::{stream, StreamExt}; + +use crate::geo::{Geo, Location}; +use crate::objects::Objects; +use crate::record::{Drained, OTHER}; +use crate::rows::{encode, DayRow}; +use crate::TrafficSettings; + +/// The prefix of the day files. +pub const DAYS: &str = "traffic/v1/days/"; +/// The rows a day file holds before new client networks are counted as +/// [`OTHER`]: a bound on the memory and the file whatever the number of +/// addresses. +const MAX_DAY_ROWS: usize = 500_000; +/// Deletes and reads in flight at once. +pub(crate) const CONCURRENCY: usize = 16; + +const DAY: i64 = 86_400; + +/// The UTC date of `day` (Unix seconds), as the keys write it. +pub(crate) fn date(day: i64) -> String { + Utc.timestamp_opt(day, 0) + .single() + .map(|t| t.format("%Y-%m-%d").to_string()) + .unwrap_or_default() +} + +/// The start of the day `date` (`YYYY-MM-DD`) names, in Unix seconds. +pub(crate) fn day_of(date: &str) -> Option { + Some( + NaiveDate::parse_from_str(date, "%Y-%m-%d") + .ok()? + .and_hms_opt(0, 0, 0)? + .and_utc() + .timestamp(), + ) +} + +/// The prefix of the files of `day`. +pub fn day_prefix(day: i64) -> String { + format!("{DAYS}{}/", date(day)) +} + +/// The key of `instance`'s file of `day`. +pub fn day_key(day: i64, instance: &str) -> String { + format!("{}{instance}.parquet", day_prefix(day)) +} + +/// The day and the instance `key`, a day file's key, names; `None` for a +/// key Forge did not write. +pub fn parse_day_key(key: &str) -> Option<(i64, String)> { + let (date, name) = key.strip_prefix(DAYS)?.split_once('/')?; + let instance = name.strip_suffix(".parquet")?; + if instance.is_empty() || instance.contains('/') { + return None; + } + Some((day_of(date)?, instance.to_string())) +} + +/// This process's name in the files: its host name (`-` for anything a +/// host name does not hold), when it started and a random suffix, so that +/// two processes never share one. No `.`, `_` or `/` but the separators. +pub fn instance_id(host: Option<&str>, started_at: DateTime) -> String { + let host: String = host + .unwrap_or("forge") + .chars() + .map(|c| { + if c.is_ascii_alphanumeric() || c == '-' { + c + } else { + '-' + } + }) + .take(63) + .collect(); + let host = if host.is_empty() { + "forge".into() + } else { + host + }; + format!( + "{host}_{}_{:04x}", + started_at.format("%Y%m%dT%H%M%SZ"), + rand::random::() + ) +} + +/// The host name and start time an [`instance_id`] holds. +pub fn instance_parts(id: &str) -> (String, Option>) { + let mut parts = id.split('_'); + let host = parts.next().unwrap_or_default().to_string(); + let started = parts.next().and_then(|t| { + chrono::NaiveDateTime::parse_from_str(t, "%Y%m%dT%H%M%SZ") + .ok() + .map(|t| t.and_utc()) + }); + (host, started) +} + +/// What a day's row is counted under. With client addresses, the network +/// (its country and autonomous system are looked up when the file is +/// written); without, the country and autonomous system, looked up before +/// the address is dropped. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +struct RowKey { + repository: String, + network: String, + country: String, + asn: u32, +} + +#[derive(Debug, Clone, Default)] +struct RowCounts { + requests: u64, + bytes: u64, + /// The last address seen; `""` without addresses. + address: String, + /// The autonomous system's organisation, for a row without a network. + as_org: String, +} + +/// One instance's counts of a day. +#[derive(Debug, Default)] +struct Day { + rows: BTreeMap, + /// Where each network is, looked up once a day (again while unknown). + located: HashMap, + dirty: bool, +} + +/// What a flush did. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct FlushReport { + /// Files written. + pub files: usize, + /// Files that could not be written (written with the next flush). + pub failures: usize, +} + +/// Writes one instance's day files. +pub struct Flusher { + objects: Arc, + instance: Arc, + days: BTreeMap, +} + +fn signed(n: u64) -> i64 { + i64::try_from(n).unwrap_or(i64::MAX) +} + +impl Flusher { + /// The flusher of `instance`, writing to `objects`. + pub fn new(objects: Arc, instance: Arc) -> Flusher { + Flusher { + objects, + instance, + days: BTreeMap::new(), + } + } + + /// Add `drained` to its days, locating the client networks with `geo`, + /// and write the files of the days that changed. A file that cannot be + /// written is written with the next flush. + pub async fn flush( + &mut self, + drained: Drained, + settings: &TrafficSettings, + geo: &Geo, + now: DateTime, + ) -> FlushReport { + if drained.overflowed > 0 { + tracing::warn!( + requests = drained.overflowed, + "more client networks than the traffic statistics hold between two flushes; \ + counted as `other`" + ); + } + for (key, counts) in drained.counts { + let day = self.days.entry(key.day).or_default(); + day.dirty = true; + let address = counts.address.map(|a| a.to_string()).unwrap_or_default(); + let mut row = if !settings.client_addresses || key.network.is_empty() { + // No network to keep: locate the address now, then drop it. + let at = counts.address.map(|a| geo.locate(a)).unwrap_or_default(); + ( + RowKey { + repository: key.repository, + network: String::new(), + country: at.country, + asn: at.asn, + }, + String::new(), + at.as_org, + ) + } else { + ( + RowKey { + repository: key.repository, + network: key.network, + country: String::new(), + asn: 0, + }, + address, + String::new(), + ) + }; + if day.rows.len() >= MAX_DAY_ROWS && !day.rows.contains_key(&row.0) { + row.0.network = OTHER.to_string(); + row.1.clear(); + } + let (row_key, address, as_org) = row; + let entry = day.rows.entry(row_key).or_default(); + entry.requests += counts.requests; + entry.bytes += counts.bytes; + if !address.is_empty() { + entry.address = address; + } + if !as_org.is_empty() { + entry.as_org = as_org; + } + } + + let mut report = FlushReport::default(); + for (start, day) in self.days.iter_mut().filter(|(_, d)| d.dirty) { + let rows = day_rows(*start, day, geo); + let key = day_key(*start, &self.instance); + let written = match encode(&rows) { + Ok(body) => self.objects.put(&key, body).await, + Err(e) => Err(e), + }; + match written { + Ok(()) => { + day.dirty = false; + report.files += 1; + } + Err(e) => { + report.failures += 1; + tracing::warn!(error = %e, key, "cannot write the traffic statistics"); + } + } + } + // Keep today and yesterday (a request that ended after midnight + // counts in its own day), and any day not written yet but for a + // week at most. + let today = now.timestamp().div_euclid(DAY) * DAY; + self.days + .retain(|start, day| *start >= today - DAY || (day.dirty && *start >= today - 7 * DAY)); + report + } +} + +/// The rows of `day`, its networks located with `geo`. +fn day_rows(start: i64, day: &mut Day, geo: &Geo) -> Vec { + let mut rows = Vec::with_capacity(day.rows.len()); + for (key, counts) in &day.rows { + let at = if key.network.is_empty() || key.network == OTHER || counts.address.is_empty() { + Location { + country: key.country.clone(), + asn: key.asn, + as_org: counts.as_org.clone(), + } + } else { + let known = day.located.get(&key.network); + if known.is_some_and(|l| !l.country.is_empty() || l.asn != 0) { + known.cloned().unwrap_or_default() + } else { + let at = counts + .address + .parse() + .map(|ip| geo.locate(ip)) + .unwrap_or_default(); + day.located.insert(key.network.clone(), at.clone()); + at + } + }; + rows.push(DayRow { + day: start, + repository: key.repository.clone(), + network: key.network.clone(), + address: counts.address.clone(), + country: at.country, + asn: i64::from(at.asn), + as_org: at.as_org, + requests: signed(counts.requests), + bytes: signed(counts.bytes), + }); + } + rows +} + +/// Delete the day files past `settings`' retention, whoever wrote them. +/// Idempotent and safe on every instance at once. Returns how many files it +/// deleted. +pub async fn prune( + objects: &dyn Objects, + settings: &TrafficSettings, + now: DateTime, +) -> anyhow::Result { + let today = now.timestamp().div_euclid(DAY) * DAY; + let oldest_kept = today - i64::from(settings.retention_days.max(1) - 1) * DAY; + let mut expired = Vec::new(); + for dir in objects.list_dirs(DAYS).await? { + let Some(day) = dir + .strip_prefix(DAYS) + .and_then(|d| d.strip_suffix('/')) + .and_then(day_of) + else { + continue; + }; + if day < oldest_kept { + expired.extend(objects.list(&dir).await?.into_iter().map(|o| o.key)); + } + } + let deleted = expired.len(); + let failures: Vec = stream::iter(expired) + .map(|key| async move { objects.delete(&key).await }) + .buffer_unordered(CONCURRENCY) + .filter_map(|result| async move { result.err() }) + .collect() + .await; + match failures.into_iter().next() { + Some(e) => Err(e.context("cannot delete expired traffic statistics")), + None => Ok(deleted), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::geo::GeoKind; + use crate::objects::MemoryObjects; + use crate::record::{Event, Recorder}; + use crate::rows::decode; + use crate::slot::Noted; + use crate::testing::{asn_db, country_db}; + + fn t(rfc3339: &str) -> DateTime { + DateTime::parse_from_rfc3339(rfc3339).unwrap().to_utc() + } + + #[test] + fn keys_name_what_they_hold() { + let day = t("2026-10-30T00:00:00Z").timestamp(); + let key = day_key(day, "web-1_x_0001"); + assert_eq!(key, "traffic/v1/days/2026-10-30/web-1_x_0001.parquet"); + assert_eq!(parse_key_ok(&key), (day, "web-1_x_0001".to_string())); + assert!(parse_day_key("traffic/v1/days/2026-13-30/a.parquet").is_none()); + assert!(parse_day_key("traffic/v1/days/2026-10-30/a.csv").is_none()); + assert!(parse_day_key("traffic/v1/geoip/country.mmdb").is_none()); + } + + fn parse_key_ok(key: &str) -> (i64, String) { + parse_day_key(key).unwrap() + } + + #[test] + fn an_instance_id_holds_its_host_and_start() { + let started = t("2026-10-30T14:05:01Z"); + let id = instance_id(Some("forge-7d9f.pod_x"), started); + assert!(id.starts_with("forge-7d9f-pod-x_20261030T140501Z_"), "{id}"); + assert_eq!( + instance_parts(&id), + ("forge-7d9f-pod-x".to_string(), Some(started)) + ); + assert!(instance_id(None, started).starts_with("forge_")); + } + + fn event(at: &str, client: &str, repository: Option<&str>) -> Event { + Event { + at: t(at), + bytes: 10, + noted: Noted { + client: client.parse().ok(), + network: Some(client.into()), + repository: repository.map(str::to_string), + }, + } + } + + fn geo() -> Geo { + let geo = Geo::default(); + geo.load( + GeoKind::Country, + country_db(&[("192.0.2.0/24", "DE")]), + "1".into(), + ) + .unwrap(); + geo.load( + GeoKind::Asn, + asn_db(&[("192.0.2.0/24", 64_500, "Example AS")]), + "1".into(), + ) + .unwrap(); + geo + } + + async fn rows(objects: &MemoryObjects, key: &str) -> Vec { + decode(objects.get(key).await.unwrap().unwrap()).unwrap() + } + + #[tokio::test] + async fn a_flush_writes_the_day_with_countries_and_networks() { + let objects = Arc::new(MemoryObjects::default()); + let mut flusher = Flusher::new(objects.clone(), "a".into()); + let settings = TrafficSettings::default(); + let geo = geo(); + let recorder = Recorder::new(); + recorder.record(event("2026-10-30T14:00:10Z", "192.0.2.1", Some("npm"))); + recorder.record(event("2026-10-30T14:00:20Z", "192.0.2.1", Some("npm"))); + recorder.record(event("2026-10-30T14:01:20Z", "198.51.100.2", None)); + let now = t("2026-10-30T14:02:00Z"); + let report = flusher.flush(recorder.drain(), &settings, &geo, now).await; + assert_eq!( + report, + FlushReport { + files: 1, + failures: 0 + } + ); + + recorder.record(event("2026-10-30T14:03:00Z", "192.0.2.1", Some("npm"))); + flusher.flush(recorder.drain(), &settings, &geo, now).await; + let day = rows(&objects, "traffic/v1/days/2026-10-30/a.parquet").await; + assert_eq!(day.len(), 2); + let npm = day.iter().find(|r| r.repository == "npm").unwrap(); + assert_eq!((npm.requests, npm.bytes), (3, 30), "added up over flushes"); + assert_eq!( + ( + npm.network.as_str(), + npm.country.as_str(), + npm.asn, + npm.as_org.as_str() + ), + ("192.0.2.1", "DE", 64_500, "Example AS") + ); + let other = day.iter().find(|r| r.repository.is_empty()).unwrap(); + assert_eq!( + (other.country.as_str(), other.asn), + ("", 0), + "not in the databases" + ); + + // Nothing new: nothing written. + let report = flusher.flush(recorder.drain(), &settings, &geo, now).await; + assert_eq!(report.files, 0); + // The next day: the day before is still kept, the one before that is not. + recorder.record(event("2026-10-31T00:00:05Z", "192.0.2.1", Some("npm"))); + flusher + .flush(recorder.drain(), &settings, &geo, t("2026-11-01T00:00:10Z")) + .await; + assert!(objects + .keys() + .contains(&"traffic/v1/days/2026-10-31/a.parquet".to_string())); + assert_eq!(flusher.days.len(), 1); + } + + #[tokio::test] + async fn without_addresses_rows_keep_the_country_alone() { + let objects = Arc::new(MemoryObjects::default()); + let mut flusher = Flusher::new(objects.clone(), "a".into()); + let settings = TrafficSettings { + client_addresses: false, + ..TrafficSettings::default() + }; + let recorder = Recorder::new(); + recorder.record(event("2026-10-30T14:00:10Z", "192.0.2.1", Some("npm"))); + recorder.record(event("2026-10-30T14:00:20Z", "192.0.2.2", Some("npm"))); + let now = t("2026-10-30T14:02:00Z"); + flusher + .flush(recorder.drain(), &settings, &geo(), now) + .await; + let day = rows(&objects, "traffic/v1/days/2026-10-30/a.parquet").await; + assert_eq!(day.len(), 1, "both addresses in one row"); + assert_eq!( + ( + day[0].network.as_str(), + day[0].address.as_str(), + day[0].country.as_str() + ), + ("", "", "DE") + ); + assert_eq!((day[0].requests, day[0].asn), (2, 64_500)); + } + + #[tokio::test] + async fn pruning_deletes_the_days_past_the_retention() { + let objects = MemoryObjects::default(); + let day = |d: &str| t(&format!("{d}T00:00:00Z")).timestamp(); + for key in [ + day_key(day("2026-07-31"), "a"), + day_key(day("2026-08-01"), "a"), + day_key(day("2026-08-01"), "b"), + day_key(day("2026-10-30"), "a"), + "traffic/v1/days/not-a-date/x.parquet".to_string(), + "traffic/v1/geoip/country.mmdb".to_string(), + ] { + objects.put(&key, vec![1]).await.unwrap(); + } + // 90 days up to 2026-10-29: from 2026-08-01 on. + let deleted = prune( + &objects, + &TrafficSettings::default(), + t("2026-10-29T12:00:00Z"), + ) + .await + .unwrap(); + assert_eq!(deleted, 1); + assert_eq!( + objects.keys(), + vec![ + "traffic/v1/days/2026-08-01/a.parquet", + "traffic/v1/days/2026-08-01/b.parquet", + "traffic/v1/days/2026-10-30/a.parquet", + "traffic/v1/days/not-a-date/x.parquet", + "traffic/v1/geoip/country.mmdb", + ] + ); + } +} diff --git a/crates/traffic/src/lib.rs b/crates/traffic/src/lib.rs new file mode 100644 index 00000000..cc09af48 --- /dev/null +++ b/crates/traffic/src/lib.rs @@ -0,0 +1,96 @@ +//! Forge's traffic statistics: how many requests each client network made to +//! each repository on each day, from which country and autonomous system, +//! kept as Parquet files in the object store, so that any instance can show +//! the traffic of all of them (Admin → Traffic). +//! +//! * **Recording** ([`Recorder`]) is a counter update in memory, behind one +//! short lock: no I/O on the request path. A request is counted under its +//! day (UTC), its repository and its client network, with the bytes sent. +//! * **What a request is** comes from the layers that know it: the server's +//! outermost middleware opens a [`Slot`] around the request, and the +//! principal layer ([`note_client`]) and the handlers that resolve a +//! repository ([`note_repository`]) fill it in. +//! * **Flushing** ([`Flusher`]) runs once a minute on every instance: it +//! locates the client networks in the geolocation databases ([`geo`]) and +//! rewrites the instance's file of the day ([`layout`]), named after the +//! process ([`instance_id`]), so instances never write the same object. +//! * **Reading** ([`query`]) lists the files of a range of days in the +//! object store, reads them (cached by ETag) and sums them: the view over +//! all instances needs no call between them. +//! * **Pruning** ([`layout::prune`]) deletes the days past their retention +//! ([`TrafficSettings`]); it is idempotent and runs on every instance. +//! +//! Client addresses are personal data: they can be switched off, and then +//! requests are still counted by repository, country and autonomous system, +//! the address only looked up in memory before it is dropped. + +pub mod geo; +pub mod layout; +pub mod objects; +pub mod query; +mod record; +mod rows; +mod settings; +mod slot; +#[cfg(any(test, feature = "testing"))] +pub mod testing; + +use std::sync::Arc; + +pub use geo::{DbInfo, Geo, GeoKind, Location}; +pub use layout::{instance_id, FlushReport, Flusher}; +pub use objects::{Listed, MemoryObjects, Objects}; +pub use record::{Drained, Event, PendingCounts, PendingKey, Recorder, OTHER}; +pub use settings::{TrafficSettings, MAX_RETENTION_DAYS}; +pub use slot::{note_client, note_repository, Noted, Slot}; + +/// What the server holds of the traffic statistics: this instance's +/// [`Recorder`] and [`Flusher`], the geolocation databases, the object store +/// the files go to, and the cache of the files read back. +#[derive(Clone)] +pub struct Traffic { + /// Counts this instance's requests. + pub recorder: Arc, + /// The geolocation databases this instance loaded. + pub geo: Arc, + /// Where the files are written and read. + pub objects: Arc, + /// This process's name in the files ([`instance_id`]). + pub instance: Arc, + /// Files read back, decoded ([`query`]). + pub cache: query::FileCache, + /// Writes this instance's files. Shared, so that it outlives a restart + /// of the worker and the last flush on shutdown adds to what the worker + /// wrote instead of replacing it. + flusher: Arc>, +} + +impl Traffic { + /// The statistics of the instance `instance`, kept in `objects`. + pub fn new(objects: Arc, instance: String) -> Traffic { + let instance: Arc = instance.into(); + Traffic { + recorder: Arc::new(Recorder::new()), + geo: Arc::new(Geo::default()), + flusher: Arc::new(tokio::sync::Mutex::new(Flusher::new( + objects.clone(), + instance.clone(), + ))), + objects, + instance, + cache: query::FileCache::new(), + } + } + + /// Write what was counted since the last flush. + pub async fn flush( + &self, + settings: &TrafficSettings, + now: chrono::DateTime, + ) -> FlushReport { + let mut flusher = self.flusher.lock().await; + flusher + .flush(self.recorder.drain(), settings, &self.geo, now) + .await + } +} diff --git a/crates/traffic/src/objects.rs b/crates/traffic/src/objects.rs new file mode 100644 index 00000000..b7ab713b --- /dev/null +++ b/crates/traffic/src/objects.rs @@ -0,0 +1,127 @@ +//! The object store the files live in: Forge's S3 bucket +//! ([`forge_storage::BlobStore`]), or memory in tests ([`MemoryObjects`]). + +use std::collections::BTreeMap; +use std::sync::{Mutex, PoisonError}; + +use bytes::Bytes; +use forge_storage::BlobStore; + +/// An object a listing found. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Listed { + pub key: String, + pub size: u64, + /// Changes whenever the object is replaced (the cache key of a file). + pub etag: Option, +} + +/// What the statistics need of an object store. +#[async_trait::async_trait] +pub trait Objects: Send + Sync + 'static { + /// Store `body` at `key`, replacing what is there. + async fn put(&self, key: &str, body: Vec) -> anyhow::Result<()>; + /// The object at `key`, or `None` when there is none (any more). + async fn get(&self, key: &str) -> anyhow::Result>; + /// Every object whose key starts with `prefix`. + async fn list(&self, prefix: &str) -> anyhow::Result>; + /// The "directories" right under `prefix` (which ends with `/`): the + /// distinct key prefixes up to the next `/`, each ending with it. + async fn list_dirs(&self, prefix: &str) -> anyhow::Result>; + /// Delete the object at `key`; a missing one is no error. + async fn delete(&self, key: &str) -> anyhow::Result<()>; +} + +#[async_trait::async_trait] +impl Objects for BlobStore { + async fn put(&self, key: &str, body: Vec) -> anyhow::Result<()> { + Ok(self.put_bytes(key, body).await?) + } + + async fn get(&self, key: &str) -> anyhow::Result> { + Ok(self.get_bytes(key).await?) + } + + async fn list(&self, prefix: &str) -> anyhow::Result> { + Ok(BlobStore::list(self, prefix, None) + .await? + .objects + .into_iter() + .map(|o| Listed { + key: o.key, + size: o.size, + etag: o.etag, + }) + .collect()) + } + + async fn list_dirs(&self, prefix: &str) -> anyhow::Result> { + Ok(BlobStore::list(self, prefix, Some("/")).await?.prefixes) + } + + async fn delete(&self, key: &str) -> anyhow::Result<()> { + Ok(BlobStore::delete(self, key).await?) + } +} + +/// An object store in memory, for tests. +#[derive(Debug, Default)] +pub struct MemoryObjects { + objects: Mutex>, +} + +impl MemoryObjects { + fn lock(&self) -> std::sync::MutexGuard<'_, BTreeMap> { + self.objects.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// The keys stored, in order. + pub fn keys(&self) -> Vec { + self.lock().keys().cloned().collect() + } +} + +#[async_trait::async_trait] +impl Objects for MemoryObjects { + async fn put(&self, key: &str, body: Vec) -> anyhow::Result<()> { + let mut objects = self.lock(); + let version = objects.get(key).map_or(1, |(_, v)| v + 1); + objects.insert(key.to_string(), (body.into(), version)); + Ok(()) + } + + async fn get(&self, key: &str) -> anyhow::Result> { + Ok(self.lock().get(key).map(|(body, _)| body.clone())) + } + + async fn list(&self, prefix: &str) -> anyhow::Result> { + Ok(self + .lock() + .range(prefix.to_string()..) + .take_while(|(key, _)| key.starts_with(prefix)) + .map(|(key, (body, version))| Listed { + key: key.clone(), + size: body.len() as u64, + etag: Some(version.to_string()), + }) + .collect()) + } + + async fn list_dirs(&self, prefix: &str) -> anyhow::Result> { + let mut dirs: Vec = Vec::new(); + for listed in self.list(prefix).await? { + if let Some(end) = listed.key[prefix.len()..].find('/') { + let dir = &listed.key[..prefix.len() + end + 1]; + if dirs.last().map(String::as_str) != Some(dir) { + dirs.push(dir.to_string()); + } + } + } + Ok(dirs) + } + + async fn delete(&self, key: &str) -> anyhow::Result<()> { + self.lock().remove(key); + Ok(()) + } +} diff --git a/crates/traffic/src/query.rs b/crates/traffic/src/query.rs new file mode 100644 index 00000000..ad3ea771 --- /dev/null +++ b/crates/traffic/src/query.rs @@ -0,0 +1,651 @@ +//! Reading the day files back: the requests of a range of days, over all +//! instances or one, and where they came from. +//! +//! The files are found by listing the days of the range, read in parallel +//! and summed. They are cached decoded, keyed by their ETag, so a file +//! rewritten since (today's) is read again. + +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::sync::Arc; + +use chrono::{NaiveDate, TimeZone, Utc}; +use futures_util::{stream, StreamExt, TryStreamExt}; +use serde::Serialize; + +use crate::layout::{day_prefix, instance_parts, parse_day_key, CONCURRENCY}; +use crate::objects::{Listed, Objects}; +use crate::record::OTHER; +use crate::rows::{decode, DayRow}; + +/// The lists' lengths: repositories, autonomous systems, clients. +pub const TOP: usize = 20; +/// The rows the cache of decoded files holds. +const CACHE_ROWS: u64 = 500_000; +/// The longest range, in days. +pub const MAX_DAYS: i64 = 400; + +const DAY: i64 = 86_400; + +/// Day files read back, decoded. +#[derive(Clone)] +pub struct FileCache(moka::future::Cache>>); + +impl FileCache { + pub fn new() -> FileCache { + FileCache( + moka::future::Cache::builder() + .max_capacity(CACHE_ROWS) + .weigher(|_, rows: &Arc>| { + u32::try_from(rows.len().max(1)).unwrap_or(u32::MAX) + }) + .build(), + ) + } +} + +impl Default for FileCache { + fn default() -> Self { + FileCache::new() + } +} + +/// An instance, as the views name it. +#[derive(Debug, Clone, Serialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct InstanceView { + /// Its name in the files ([`crate::instance_id`]). + pub id: String, + pub host: String, + pub started_at: Option>, +} + +impl InstanceView { + fn of(id: &str) -> InstanceView { + let (host, started_at) = instance_parts(id); + InstanceView { + id: id.to_string(), + host, + started_at, + } + } +} + +/// What to read: the days `from` to `to`, both included, and what to keep. +#[derive(Debug, Clone, Default)] +pub struct Filter { + /// Only the files of this instance ([`crate::instance_id`]). + pub instance: Option, + /// Only the requests to this repository; `""` for the requests that + /// addressed none. + pub repository: Option, + /// Only the requests of this client network. + pub network: Option, + /// Only the requests from this country (ISO code; `""` unknown). + pub country: Option, + /// Only the requests from this autonomous system (`0` unknown). + pub asn: Option, +} + +impl Filter { + fn keeps(&self, row: &DayRow) -> bool { + self.repository + .as_ref() + .is_none_or(|r| *r == row.repository) + && self.network.as_ref().is_none_or(|n| *n == row.network) + && self.country.as_ref().is_none_or(|c| *c == row.country) + && self.asn.is_none_or(|a| i64::from(a) == row.asn) + } +} + +/// A day of the range. +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct DayPoint { + pub day: NaiveDate, + pub requests: u64, + pub bytes: u64, + /// The client networks that made requests that day. + pub clients: u64, +} + +/// The counts of the range. +#[derive(Debug, Clone, Default, Serialize, PartialEq, Eq)] +pub struct Totals { + pub requests: u64, + pub bytes: u64, + /// The client networks that made requests. + pub clients: u64, + /// The countries they came from (known ones). + pub countries: u64, + /// The autonomous systems they came from (known ones). + pub asns: u64, +} + +/// A share of the range: a repository's, a country's, an autonomous +/// system's. +#[derive(Debug, Clone, Default, Serialize, PartialEq, Eq)] +pub struct Share { + pub requests: u64, + pub bytes: u64, + /// The client networks among them. + pub clients: u64, +} + +#[derive(Default)] +struct Tally { + share: Share, + networks: HashSet, +} + +impl Tally { + fn add(&mut self, row: &DayRow) { + self.share.requests += u64::try_from(row.requests).unwrap_or(0); + self.share.bytes += u64::try_from(row.bytes).unwrap_or(0); + if counts_as_client(&row.network) { + self.networks.insert(row.network.clone()); + } + } + + fn finish(mut self) -> Share { + self.share.clients = self.networks.len() as u64; + self.share + } +} + +/// A client network being summed: its share, its requests by repository, +/// the days it was seen and the latest of them. +#[derive(Default)] +struct ClientTally<'a> { + client: ClientShare, + repositories: HashMap<&'a str, u64>, + days: BTreeSet, + latest: Option, +} + +fn counts_as_client(network: &str) -> bool { + !network.is_empty() && network != OTHER +} + +/// A repository's requests. +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct RepositoryShare { + /// `""` for the requests that addressed no repository. + pub name: String, + #[serde(flatten)] + pub share: Share, +} + +/// A country's requests. +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct CountryShare { + /// ISO 3166-1 alpha-2; `""` unknown. + pub country: String, + #[serde(flatten)] + pub share: Share, +} + +/// An autonomous system's requests. +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct AsnShare { + /// `0` unknown. + pub asn: u32, + pub org: String, + #[serde(flatten)] + pub share: Share, +} + +/// A client network's requests. +#[derive(Debug, Clone, Default, Serialize, PartialEq, Eq)] +pub struct ClientShare { + pub network: String, + /// The last address of the network seen. + pub address: String, + pub country: String, + pub asn: u32, + pub as_org: String, + pub requests: u64, + pub bytes: u64, + /// The repositories it made the most requests to, most first. + pub repositories: Vec, + /// The days it made requests on. + pub days: u64, + pub first_day: Option, + pub last_day: Option, +} + +/// The requests of a range of days: `GET /admin/traffic`. +#[derive(Debug, Clone, Serialize)] +pub struct Overview { + pub from: NaiveDate, + /// The last day, included. + pub to: NaiveDate, + /// The instances whose files the range holds (whatever the instance + /// asked for). + pub instances: Vec, + pub totals: Totals, + pub days: Vec, + /// The [`TOP`] repositories with the most requests. + pub repositories: Vec, + /// Every country, most requests first. + pub countries: Vec, + /// The [`TOP`] autonomous systems with the most requests. + pub asns: Vec, + /// The [`TOP`] client networks with the most requests. + pub clients: Vec, +} + +fn naive(day: i64) -> NaiveDate { + Utc.timestamp_opt(day, 0) + .single() + .map(|t| t.date_naive()) + .unwrap_or_default() +} + +/// The day files of the days `from` to `to` (Unix seconds, both included). +async fn files( + objects: &dyn Objects, + from: i64, + to: i64, +) -> anyhow::Result> { + let days: Vec = (0..) + .map(|i| from + i * DAY) + .take_while(|day| *day <= to) + .collect(); + let listings: Vec> = stream::iter(days) + .map(|day| async move { objects.list(&day_prefix(day)).await }) + .buffered(CONCURRENCY) + .try_collect() + .await?; + Ok(listings + .into_iter() + .flatten() + .filter_map(|listed| { + let (day, instance) = parse_day_key(&listed.key)?; + Some((day, instance, listed)) + }) + .collect()) +} + +/// The rows of the day file `listed`, from the cache when it holds this +/// version. +async fn read( + objects: &dyn Objects, + cache: &FileCache, + listed: &Listed, +) -> anyhow::Result>> { + let cache_key = format!("{}#{}", listed.key, listed.etag.as_deref().unwrap_or("")); + if let Some(hit) = cache.0.get(&cache_key).await { + return Ok(hit); + } + let Some(body) = objects.get(&listed.key).await? else { + // Deleted since the listing (pruned). + return Ok(Arc::new(Vec::new())); + }; + let rows = Arc::new(decode::(body)?); + if listed.etag.is_some() { + cache.0.insert(cache_key, rows.clone()).await; + } + Ok(rows) +} + +/// The requests of the days `from` to `to` (both included) over +/// `objects`' files, as `filter` narrows them. +pub async fn overview( + objects: &dyn Objects, + cache: &FileCache, + from: NaiveDate, + to: NaiveDate, + filter: &Filter, +) -> anyhow::Result { + let start = |d: NaiveDate| { + d.and_hms_opt(0, 0, 0) + .map(|t| t.and_utc().timestamp()) + .unwrap_or_default() + }; + let (from_s, to_s) = (start(from), start(to)); + let found = files(objects, from_s, to_s).await?; + let instances: BTreeSet = found.iter().map(|(_, i, _)| i.clone()).collect(); + let wanted: Vec = found + .into_iter() + .filter(|(_, i, _)| filter.instance.as_ref().is_none_or(|w| w == i)) + .map(|(_, _, listed)| listed) + .collect(); + let read: Vec>> = stream::iter(wanted) + .map(|listed| async move { read(objects, cache, &listed).await }) + .buffer_unordered(CONCURRENCY) + .try_collect() + .await?; + + let mut days: BTreeMap = (0..) + .map(|i| from_s + i * DAY) + .take_while(|d| *d <= to_s) + .map(|d| (d, Tally::default())) + .collect(); + let mut all = Tally::default(); + let mut repositories: HashMap<&str, Tally> = HashMap::new(); + let mut countries: HashMap<&str, Tally> = HashMap::new(); + let mut asns: HashMap = HashMap::new(); + let mut clients: HashMap<&str, ClientTally> = HashMap::new(); + for row in read.iter().flat_map(|rows| rows.iter()) { + if !(from_s..=to_s).contains(&row.day) || !filter.keeps(row) { + continue; + } + all.add(row); + if let Some(day) = days.get_mut(&row.day) { + day.add(row); + } + repositories.entry(&row.repository).or_default().add(row); + countries.entry(&row.country).or_default().add(row); + let asn = asns + .entry(row.asn) + .or_insert_with(|| (Tally::default(), "")); + asn.0.add(row); + if asn.1.is_empty() { + asn.1 = &row.as_org; + } + if counts_as_client(&row.network) { + let tally = clients.entry(&row.network).or_default(); + let client = &mut tally.client; + if client.network.is_empty() { + client.network.clone_from(&row.network); + } + let requests = u64::try_from(row.requests).unwrap_or(0); + client.requests += requests; + client.bytes += u64::try_from(row.bytes).unwrap_or(0); + *tally.repositories.entry(&row.repository).or_default() += requests; + tally.days.insert(row.day); + if tally.latest.is_none_or(|latest| row.day >= latest) { + tally.latest = Some(row.day); + if !row.address.is_empty() { + client.address.clone_from(&row.address); + } + if !row.country.is_empty() || row.asn != 0 { + client.country.clone_from(&row.country); + client.asn = u32::try_from(row.asn).unwrap_or(0); + client.as_org.clone_from(&row.as_org); + } + } + } + } + + let by_requests = |a: &Share, b: &Share| b.requests.cmp(&a.requests); + let mut repositories: Vec = repositories + .into_iter() + .map(|(name, t)| RepositoryShare { + name: name.to_string(), + share: t.finish(), + }) + .collect(); + repositories.sort_by(|a, b| by_requests(&a.share, &b.share).then(a.name.cmp(&b.name))); + repositories.truncate(TOP); + let mut countries: Vec = countries + .into_iter() + .map(|(country, t)| CountryShare { + country: country.to_string(), + share: t.finish(), + }) + .collect(); + countries.sort_by(|a, b| by_requests(&a.share, &b.share).then(a.country.cmp(&b.country))); + let known_countries = countries.iter().filter(|c| !c.country.is_empty()).count(); + let mut asns: Vec = asns + .into_iter() + .map(|(asn, (t, org))| AsnShare { + asn: u32::try_from(asn).unwrap_or(0), + org: org.to_string(), + share: t.finish(), + }) + .collect(); + asns.sort_by(|a, b| by_requests(&a.share, &b.share).then(a.asn.cmp(&b.asn))); + let known_asns = asns.iter().filter(|a| a.asn != 0).count(); + asns.truncate(TOP); + let mut clients: Vec = clients + .into_values() + .map(|tally| { + let ClientTally { + mut client, + repositories: repos, + days: seen, + .. + } = tally; + let mut repos: Vec<(&str, u64)> = repos.into_iter().collect(); + repos.sort_by(|a, b| b.1.cmp(&a.1).then(a.0.cmp(b.0))); + client.repositories = repos + .into_iter() + .take(5) + .map(|(r, _)| r.to_string()) + .collect(); + client.days = seen.len() as u64; + client.first_day = seen.first().map(|d| naive(*d)); + client.last_day = seen.last().map(|d| naive(*d)); + client + }) + .collect(); + clients.sort_by(|a, b| b.requests.cmp(&a.requests).then(a.network.cmp(&b.network))); + clients.truncate(TOP); + + let total = all.finish(); + Ok(Overview { + from, + to, + instances: instances.iter().map(|id| InstanceView::of(id)).collect(), + totals: Totals { + requests: total.requests, + bytes: total.bytes, + clients: total.clients, + countries: known_countries as u64, + asns: known_asns as u64, + }, + days: days + .into_iter() + .map(|(day, t)| { + let share = t.finish(); + DayPoint { + day: naive(day), + requests: share.requests, + bytes: share.bytes, + clients: share.clients, + } + }) + .collect(), + repositories, + countries, + asns, + clients, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::geo::{Geo, GeoKind}; + use crate::layout::Flusher; + use crate::objects::MemoryObjects; + use crate::record::{Event, Recorder}; + use crate::slot::Noted; + use crate::testing::{asn_db, country_db}; + use crate::TrafficSettings; + use chrono::DateTime; + + fn t(rfc3339: &str) -> DateTime { + DateTime::parse_from_rfc3339(rfc3339).unwrap().to_utc() + } + + fn d(date: &str) -> NaiveDate { + NaiveDate::parse_from_str(date, "%Y-%m-%d").unwrap() + } + + fn event(at: &str, client: &str, repository: &str) -> Event { + Event { + at: t(at), + bytes: 1_000, + noted: Noted { + client: client.parse().ok(), + network: Some(client.into()), + repository: (!repository.is_empty()).then(|| repository.to_string()), + }, + } + } + + /// Two instances, each with its own files, over two days. + async fn two_instances() -> Arc { + let objects = Arc::new(MemoryObjects::default()); + let settings = TrafficSettings::default(); + let geo = Geo::default(); + geo.load( + GeoKind::Country, + country_db(&[("192.0.2.0/24", "DE"), ("198.51.100.0/24", "US")]), + "1".into(), + ) + .unwrap(); + geo.load( + GeoKind::Asn, + asn_db(&[ + ("192.0.2.0/24", 64_500, "Example DE"), + ("198.51.100.0/24", 64_501, "Example US"), + ]), + "1".into(), + ) + .unwrap(); + for (instance, events) in [ + ( + "a_20261029T100000Z_0001", + vec![ + event("2026-10-29T10:00:00Z", "192.0.2.1", "npm"), + event("2026-10-30T10:00:00Z", "192.0.2.1", "npm"), + event("2026-10-30T10:00:01Z", "198.51.100.9", "docker"), + ], + ), + ( + "b_20261029T110000Z_0002", + vec![ + event("2026-10-30T11:00:00Z", "192.0.2.1", "npm"), + event("2026-10-30T11:00:00Z", "192.0.2.1", "npm"), + event("2026-10-30T11:00:01Z", "203.0.113.5", ""), + ], + ), + ] { + let recorder = Recorder::new(); + for e in events { + recorder.record(e); + } + let mut flusher = Flusher::new(objects.clone(), instance.into()); + flusher + .flush(recorder.drain(), &settings, &geo, t("2026-10-30T12:00:00Z")) + .await; + } + objects + } + + #[tokio::test] + async fn the_overview_sums_every_instance() { + let objects = two_instances().await; + let cache = FileCache::new(); + let all = overview( + &*objects, + &cache, + d("2026-10-24"), + d("2026-10-30"), + &Filter::default(), + ) + .await + .unwrap(); + assert_eq!(all.instances.len(), 2); + assert_eq!(all.instances[0].host, "a"); + assert_eq!(all.days.len(), 7); + assert_eq!(all.days[5].requests, 1, "the 29th"); + assert_eq!( + (all.days[6].requests, all.days[6].clients), + (5, 3), + "the 30th" + ); + assert_eq!( + all.totals, + Totals { + requests: 6, + bytes: 6_000, + clients: 3, + countries: 2, + asns: 2 + } + ); + assert_eq!(all.repositories[0].name, "npm"); + assert_eq!( + ( + all.repositories[0].share.requests, + all.repositories[0].share.clients + ), + (4, 1) + ); + assert_eq!(all.countries[0].country, "DE"); + let unknown = all.countries.iter().find(|c| c.country.is_empty()).unwrap(); + assert_eq!(unknown.share.requests, 1, "203.0.113.5 is in no database"); + assert_eq!( + (all.asns[0].asn, all.asns[0].org.as_str()), + (64_500, "Example DE") + ); + let top = &all.clients[0]; + assert_eq!( + ( + top.network.as_str(), + top.requests, + top.days, + top.country.as_str() + ), + ("192.0.2.1", 4, 2, "DE") + ); + assert_eq!(top.repositories, vec!["npm"]); + assert_eq!(top.first_day, Some(d("2026-10-29"))); + } + + #[tokio::test] + async fn filters_narrow_every_list() { + let objects = two_instances().await; + let cache = FileCache::new(); + let range = (d("2026-10-24"), d("2026-10-30")); + let only = |f: Filter| { + let objects = objects.clone(); + let cache = cache.clone(); + async move { + overview(&*objects, &cache, range.0, range.1, &f) + .await + .unwrap() + } + }; + let b = only(Filter { + instance: Some("b_20261029T110000Z_0002".into()), + ..Filter::default() + }) + .await; + assert_eq!(b.totals.requests, 3); + assert_eq!(b.instances.len(), 2, "the selector still lists both"); + let docker = only(Filter { + repository: Some("docker".into()), + ..Filter::default() + }) + .await; + assert_eq!(docker.totals.requests, 1); + assert_eq!(docker.countries[0].country, "US"); + let none = only(Filter { + repository: Some(String::new()), + ..Filter::default() + }) + .await; + assert_eq!(none.totals.requests, 1, "the requests to no repository"); + let de = only(Filter { + country: Some("DE".into()), + ..Filter::default() + }) + .await; + assert_eq!(de.totals.requests, 4); + let asn = only(Filter { + asn: Some(64_501), + ..Filter::default() + }) + .await; + assert_eq!(asn.clients[0].network, "198.51.100.9"); + let client = only(Filter { + network: Some("192.0.2.1".into()), + ..Filter::default() + }) + .await; + assert_eq!((client.totals.requests, client.totals.clients), (4, 1)); + } +} diff --git a/crates/traffic/src/record.rs b/crates/traffic/src/record.rs new file mode 100644 index 00000000..7cc421a6 --- /dev/null +++ b/crates/traffic/src/record.rs @@ -0,0 +1,175 @@ +//! Counting requests in memory until the next flush. + +use std::collections::HashMap; +use std::net::IpAddr; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Mutex, PoisonError}; + +use chrono::{DateTime, Utc}; + +use crate::slot::Noted; + +/// The keys counted between two flushes before new client networks are +/// counted as [`OTHER`]: a bound whatever the number of addresses. +const MAX_PENDING: usize = 100_000; + +/// The network requests are counted under past a bound. +pub const OTHER: &str = "other"; + +const DAY: i64 = 86_400; + +/// One answered request. +#[derive(Debug, Clone)] +pub struct Event { + /// When the response was sent (its body done or cut off). + pub at: DateTime, + /// Response body bytes sent. + pub bytes: u64, + /// What the layers of the request noted ([`crate::Slot`]). + pub noted: Noted, +} + +/// What requests are counted under until the next flush. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct PendingKey { + /// The start of the day (UTC), in Unix seconds. + pub day: i64, + /// `""` for none. + pub repository: String, + /// `""` for none. + pub network: String, +} + +/// The counts of a [`PendingKey`]. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct PendingCounts { + pub requests: u64, + pub bytes: u64, + /// The last address of the network seen. Kept in memory until the + /// flush looks up its country and autonomous system, even with client + /// addresses switched off. + pub address: Option, +} + +/// What a flush takes from the [`Recorder`]. +#[derive(Debug, Default)] +pub struct Drained { + pub counts: Vec<(PendingKey, PendingCounts)>, + /// Requests counted under [`OTHER`] for want of room. + pub overflowed: u64, +} + +/// Counts this instance's requests until the next flush. +#[derive(Debug)] +pub struct Recorder { + enabled: AtomicBool, + overflowed: AtomicU64, + pending: Mutex>, +} + +impl Default for Recorder { + fn default() -> Self { + Recorder::new() + } +} + +impl Recorder { + /// A recorder that counts until [`Self::set_enabled`] says otherwise. + pub fn new() -> Recorder { + Recorder { + enabled: AtomicBool::new(true), + overflowed: AtomicU64::new(0), + pending: Mutex::new(HashMap::new()), + } + } + + /// Count requests from the next one on, or not. + pub fn set_enabled(&self, enabled: bool) { + self.enabled.store(enabled, Ordering::Relaxed); + } + + /// Whether requests are counted. + pub fn enabled(&self) -> bool { + self.enabled.load(Ordering::Relaxed) + } + + /// Count `event`. + pub fn record(&self, event: Event) { + if !self.enabled() { + return; + } + let Event { at, bytes, noted } = event; + let mut key = PendingKey { + day: at.timestamp().div_euclid(DAY) * DAY, + repository: noted.repository.unwrap_or_default(), + network: noted.network.unwrap_or_default(), + }; + let mut address = noted.client; + let mut pending = self.pending.lock().unwrap_or_else(PoisonError::into_inner); + if pending.len() >= MAX_PENDING && !pending.contains_key(&key) { + key.network = OTHER.to_string(); + address = None; + self.overflowed.fetch_add(1, Ordering::Relaxed); + } + let counts = pending.entry(key).or_default(); + counts.requests += 1; + counts.bytes += bytes; + if address.is_some() { + counts.address = address; + } + } + + /// Take what was counted since the last call. + pub fn drain(&self) -> Drained { + let pending = + std::mem::take(&mut *self.pending.lock().unwrap_or_else(PoisonError::into_inner)); + Drained { + counts: pending.into_iter().collect(), + overflowed: self.overflowed.swap(0, Ordering::Relaxed), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::TimeZone; + + fn event(secs: i64, network: &str, repository: Option<&str>) -> Event { + Event { + at: Utc.timestamp_opt(1_793_318_400 + secs, 0).unwrap(), + bytes: 100, + noted: Noted { + client: network.parse().ok(), + network: Some(network.into()), + repository: repository.map(str::to_string), + }, + } + } + + #[test] + fn requests_are_counted_by_day_repository_and_network() { + let r = Recorder::new(); + r.record(event(10, "192.0.2.1", Some("npm"))); + r.record(event(20, "192.0.2.1", Some("npm"))); + r.record(event(30, "192.0.2.1", None)); + r.record(event(DAY + 1, "192.0.2.1", Some("npm"))); + let mut counts = r.drain().counts; + counts.sort_by_key(|(k, _)| (k.day, k.repository.clone())); + assert_eq!(counts.len(), 3); + assert_eq!(counts[0].0.repository, ""); + assert_eq!(counts[1].1.requests, 2); + assert_eq!(counts[1].1.bytes, 200); + assert_eq!(counts[1].1.address, "192.0.2.1".parse().ok()); + assert_eq!(counts[2].0.day, 1_793_318_400 + DAY); + assert!(r.drain().counts.is_empty(), "drained"); + } + + #[test] + fn switched_off_it_counts_nothing() { + let r = Recorder::new(); + r.set_enabled(false); + r.record(event(0, "192.0.2.1", None)); + assert!(r.drain().counts.is_empty()); + } +} diff --git a/crates/traffic/src/rows.rs b/crates/traffic/src/rows.rs new file mode 100644 index 00000000..2d91faab --- /dev/null +++ b/crates/traffic/src/rows.rs @@ -0,0 +1,101 @@ +//! The rows of the Parquet files. +//! +//! The files are plain Parquet, readable by any tool (DuckDB, pandas). A +//! column the record API cannot leave empty holds `""` or `0` for none: the +//! derived reader takes no `Option` field. + +use std::sync::Arc; + +use bytes::Bytes; +use parquet::basic::Compression; +use parquet::file::properties::WriterProperties; +use parquet::file::reader::{FileReader, SerializedFileReader}; +use parquet::file::writer::SerializedFileWriter; +use parquet::record::{RecordReader, RecordWriter}; +use parquet_derive::{ParquetRecordReader, ParquetRecordWriter}; + +/// A row of a day file: the requests one client network made to one +/// repository on one day, as one instance answered them. +#[derive(Debug, Clone, Default, PartialEq, Eq, ParquetRecordWriter, ParquetRecordReader)] +pub(crate) struct DayRow { + /// The start of the day (UTC), in Unix seconds. + pub day: i64, + /// The repository the requests addressed; `""` for requests that + /// addressed none (the web UI, the APIs, sign-in). + pub repository: String, + /// The client network: the address, or an IPv6 address's /64; `""` with + /// client addresses switched off, `other` past the day's bound. + pub network: String, + /// The last address of the network seen; `""` with addresses off. + pub address: String, + /// The ISO 3166-1 code of the country the network is in; `""` unknown. + pub country: String, + /// The autonomous system the network belongs to; `0` unknown. + pub asn: i64, + /// The organisation of the autonomous system; `""` unknown. + pub as_org: String, + pub requests: i64, + /// Response body bytes sent. + pub bytes: i64, +} + +/// `rows` as a Parquet file, in one row group. +pub(crate) fn encode(rows: &[T]) -> anyhow::Result> +where + for<'a> &'a [T]: RecordWriter, +{ + let schema = rows.schema()?; + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .set_created_by(format!("forge {}", env!("CARGO_PKG_VERSION"))) + .build(); + let mut writer = SerializedFileWriter::new(Vec::new(), schema, Arc::new(props))?; + let mut group = writer.next_row_group()?; + rows.write_to_row_group(&mut group)?; + group.close()?; + Ok(writer.into_inner()?) +} + +/// The rows of the Parquet file `bytes`. +pub(crate) fn decode(bytes: Bytes) -> anyhow::Result> +where + Vec: RecordReader, +{ + let reader = SerializedFileReader::new(bytes)?; + let mut rows = Vec::new(); + for i in 0..reader.num_row_groups() { + let mut group = reader.get_row_group(i)?; + let n = usize::try_from(group.metadata().num_rows()).unwrap_or(0); + if n > 0 { + rows.read_from_row_group(&mut *group, n)?; + } + } + Ok(rows) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rows_round_trip_through_parquet() { + let rows = vec![ + DayRow { + day: 1_793_318_400, + repository: "npm-proxy".into(), + network: "2001:db8::/64".into(), + address: "2001:db8::7".into(), + country: "DE".into(), + asn: 64_500, + as_org: "Example AS".into(), + requests: 12, + bytes: 3_400, + }, + DayRow::default(), + ]; + let back: Vec = decode(encode(&rows).unwrap().into()).unwrap(); + assert_eq!(back, rows); + let none: Vec = decode(encode::(&[]).unwrap().into()).unwrap(); + assert!(none.is_empty()); + } +} diff --git a/crates/traffic/src/settings.rs b/crates/traffic/src/settings.rs new file mode 100644 index 00000000..1ece4299 --- /dev/null +++ b/crates/traffic/src/settings.rs @@ -0,0 +1,46 @@ +//! What the statistics record and how long they are kept. + +/// The longest retention accepted, in days (100 years). +pub const MAX_RETENTION_DAYS: u32 = 36_500; + +/// The traffic settings of every instance, set in the web UI (Admin → +/// Traffic → Settings) or through `/admin/settings/traffic` and kept in the +/// database. An install that never saved them uses +/// [`TrafficSettings::default`]. +/// +/// Read leniently, like the other stored settings: a key left out takes its +/// default and one this version does not know is ignored. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(default)] +pub struct TrafficSettings { + /// Whether requests are counted at all. Off, nothing is recorded or + /// written; what was written stays until it expires. + pub enabled: bool, + /// Whether client addresses are recorded. Off, requests are still + /// counted by repository, country and autonomous system. + pub client_addresses: bool, + /// How long the day files are kept, in days. + pub retention_days: u32, +} + +impl Default for TrafficSettings { + fn default() -> Self { + TrafficSettings { + enabled: true, + client_addresses: true, + retention_days: 90, + } + } +} + +impl TrafficSettings { + /// What is wrong with these settings, if anything. + pub fn check(&self) -> Result<(), String> { + if !(1..=MAX_RETENTION_DAYS).contains(&self.retention_days) { + return Err(format!( + "retention_days must be between 1 and {MAX_RETENTION_DAYS}" + )); + } + Ok(()) + } +} diff --git a/crates/traffic/src/slot.rs b/crates/traffic/src/slot.rs new file mode 100644 index 00000000..b8d0dba6 --- /dev/null +++ b/crates/traffic/src/slot.rs @@ -0,0 +1,88 @@ +//! What the layers of a request learn about it, for its count: the client +//! address and the repository it addressed. +//! +//! The outermost middleware opens a [`Slot`] around the request +//! ([`Slot::scope`]); code running in the request's task fills it in with +//! [`note_client`] and [`note_repository`] without the values being threaded +//! through every handler. Outside a slot (a worker, a test calling a handler +//! directly) noting does nothing. + +use std::future::Future; +use std::net::IpAddr; +use std::sync::{Arc, Mutex, PoisonError}; + +tokio::task_local! { + static SLOT: Arc>; +} + +/// What was noted about a request. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Noted { + /// The client address: the socket peer, or the one a trusted proxy + /// forwarded. + pub client: Option, + /// The network the address is counted under (the address itself, an + /// IPv6 address's /64), as text. + pub network: Option, + /// The repository the request addressed, once it was found. + pub repository: Option, +} + +/// The slot of one request. +#[derive(Debug, Clone, Default)] +pub struct Slot(Arc>); + +impl Slot { + /// An empty slot. + pub fn new() -> Slot { + Slot::default() + } + + /// Run `fut` with this slot open: what it notes lands here. + pub async fn scope(&self, fut: F) -> F::Output { + SLOT.scope(self.0.clone(), fut).await + } + + /// What was noted so far. + pub fn take(&self) -> Noted { + std::mem::take(&mut *self.0.lock().unwrap_or_else(PoisonError::into_inner)) + } +} + +fn note(f: impl FnOnce(&mut Noted)) { + let _ = SLOT.try_with(|slot| f(&mut slot.lock().unwrap_or_else(PoisonError::into_inner))); +} + +/// Note the request's client `address`, counted under `network`. +pub fn note_client(address: IpAddr, network: String) { + note(|n| { + n.client = Some(address); + n.network = Some(network); + }); +} + +/// Note the repository the request addresses, once it is known to exist. +pub fn note_repository(name: &str) { + note(|n| n.repository = Some(name.to_string())); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn notes_land_in_the_open_slot_only() { + note_repository("outside"); + let slot = Slot::new(); + slot.scope(async { + note_client("192.0.2.7".parse().unwrap(), "192.0.2.7".into()); + tokio::task::yield_now().await; + note_repository("nuget"); + }) + .await; + let noted = slot.take(); + assert_eq!(noted.repository.as_deref(), Some("nuget")); + assert_eq!(noted.network.as_deref(), Some("192.0.2.7")); + assert_eq!(slot.take(), Noted::default(), "taken once"); + } +} diff --git a/crates/traffic/src/testing.rs b/crates/traffic/src/testing.rs new file mode 100644 index 00000000..d42a5861 --- /dev/null +++ b/crates/traffic/src/testing.rs @@ -0,0 +1,174 @@ +//! Small MaxMind databases for tests: an IPv4 search tree over a few +//! networks, their records and the metadata, in the MaxMind DB format +//! (), as GeoLite2 editions name +//! them. + +use std::net::Ipv4Addr; + +/// A value of the MaxMind DB data section. +enum Value<'a> { + Str(&'a str), + U16(u16), + U32(u32), + U64(u64), + Map(Vec<(&'a str, Value<'a>)>), + Array(Vec>), +} + +/// The control byte(s) of a value of `kind` (1–15) and `size`. +fn control(out: &mut Vec, kind: u8, size: usize) { + let (first, extra): (u8, Vec) = match size { + 0..=28 => (size as u8, vec![]), + 29..=284 => (29, vec![(size - 29) as u8]), + _ => { + let n = size - 285; + (30, vec![(n >> 8) as u8, n as u8]) + } + }; + if kind <= 7 { + out.push((kind << 5) | first); + } else { + out.push(first); + out.push(kind - 7); + } + out.extend(extra); +} + +fn uint(out: &mut Vec, kind: u8, n: u64) { + let bytes = n.to_be_bytes(); + let skip = bytes.iter().take_while(|b| **b == 0).count(); + control(out, kind, bytes.len() - skip); + out.extend(&bytes[skip..]); +} + +fn encode(out: &mut Vec, value: &Value<'_>) { + match value { + Value::Str(s) => { + control(out, 2, s.len()); + out.extend(s.as_bytes()); + } + Value::U16(n) => uint(out, 5, u64::from(*n)), + Value::U32(n) => uint(out, 6, u64::from(*n)), + Value::U64(n) => uint(out, 9, *n), + Value::Map(entries) => { + control(out, 7, entries.len()); + for (k, v) in entries { + encode(out, &Value::Str(k)); + encode(out, v); + } + } + Value::Array(items) => { + control(out, 11, items.len()); + for item in items { + encode(out, item); + } + } + } +} + +#[derive(Clone, Copy)] +enum Record { + Empty, + Node(usize), + Data(usize), +} + +/// A database of `database_type` mapping each network (`a.b.c.d/len`) to +/// its record. +fn database(database_type: &str, networks: Vec<(&str, Value<'_>)>) -> Vec { + let mut nodes: Vec<[Record; 2]> = vec![[Record::Empty; 2]]; + let mut data = Vec::new(); + for (network, record) in networks { + let (addr, len) = network.split_once('/').expect("a.b.c.d/len"); + let addr = u32::from(addr.parse::().expect("an IPv4 network")); + let len: usize = len.parse().expect("a prefix length"); + assert!((1..=32).contains(&len), "prefix length"); + let offset = data.len(); + encode(&mut data, &record); + let mut node = 0; + for depth in 0..len { + let bit = ((addr >> (31 - depth)) & 1) as usize; + if depth == len - 1 { + nodes[node][bit] = Record::Data(offset); + } else { + node = match nodes[node][bit] { + Record::Node(next) => next, + _ => { + nodes.push([Record::Empty; 2]); + let next = nodes.len() - 1; + nodes[node][bit] = Record::Node(next); + next + } + }; + } + } + } + let node_count = nodes.len(); + let mut out = Vec::new(); + for node in &nodes { + for record in node { + let value = match *record { + Record::Empty => node_count, + Record::Node(n) => n, + Record::Data(offset) => node_count + 16 + offset, + } as u32; + out.extend(&value.to_be_bytes()[1..]); + } + } + out.extend([0u8; 16]); + out.extend(data); + out.extend(b"\xAB\xCD\xEFMaxMind.com"); + encode( + &mut out, + &Value::Map(vec![ + ("binary_format_major_version", Value::U16(2)), + ("binary_format_minor_version", Value::U16(0)), + ("build_epoch", Value::U64(1_793_318_400)), + ("database_type", Value::Str(database_type)), + ("description", Value::Map(vec![("en", Value::Str("Test"))])), + ("ip_version", Value::U16(4)), + ("languages", Value::Array(vec![Value::Str("en")])), + ("node_count", Value::U32(node_count as u32)), + ("record_size", Value::U16(24)), + ]), + ); + out +} + +/// A `GeoLite2-Country` database of `(network, ISO code)`. +pub fn country_db(networks: &[(&str, &str)]) -> Vec { + database( + "GeoLite2-Country", + networks + .iter() + .map(|(net, code)| { + ( + *net, + Value::Map(vec![( + "country", + Value::Map(vec![("iso_code", Value::Str(code))]), + )]), + ) + }) + .collect(), + ) +} + +/// A `GeoLite2-ASN` database of `(network, number, organisation)`. +pub fn asn_db(networks: &[(&str, u32, &str)]) -> Vec { + database( + "GeoLite2-ASN", + networks + .iter() + .map(|(net, asn, org)| { + ( + *net, + Value::Map(vec![ + ("autonomous_system_number", Value::U32(*asn)), + ("autonomous_system_organization", Value::Str(org)), + ]), + ) + }) + .collect(), + ) +} diff --git a/docs/changes/2.2.0/709.feature.md b/docs/changes/2.2.0/709.feature.md new file mode 100644 index 00000000..ab279c17 --- /dev/null +++ b/docs/changes/2.2.0/709.feature.md @@ -0,0 +1,11 @@ +**Admin → Traffic** shows how many requests each client made to each +repository, day by day, over every instance, and the countries and +autonomous systems they came from. Each list narrows the view: a repository +shows where its requests came from, a country, network or client what it +used. Every instance writes its daily counts to Forge's bucket once a +minute, as Parquet files under `traffic/v1/` that DuckDB or pandas read too, +so the page covers all instances without Redis. Countries and networks come +from MaxMind's GeoLite2 databases, which Forge downloads every three days +with a MaxMind account ID and license key set under **Admin → Traffic → +Settings**, or which can be put in the bucket by hand. Recording, client +addresses and the retention (90 days by default) are set there too. diff --git a/docs/deployment.md b/docs/deployment.md index 5fef1d37..3ff1cb59 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -69,7 +69,9 @@ for the bucket `forge`; MinIO and Ceph RGW use the same action names): These are the S3 calls behind the policy: `HeadBucket`, `CreateBucket` (only when the bucket is missing), `HeadObject`, `GetObject`, `PutObject`, `DeleteObject`, `CreateMultipartUpload`, `UploadPart`, -`CompleteMultipartUpload` and `AbortMultipartUpload`. The policy is the same +`CompleteMultipartUpload`, `AbortMultipartUpload` and `ListObjectsV2` (the +[traffic statistics](#traffic-statistics) list their files, under +`traffic/`). The policy is the same for static keys and for the role behind short-lived credentials. ### Short-lived S3 credentials @@ -1313,6 +1315,75 @@ console log is) is exported too, each record with the trace and span it was written in, so a backend links a request's log lines to its trace. The console log stays as it is. +## Traffic statistics + +**Admin → Traffic** (`/manage/traffic`) shows how many requests each client +made to each repository, day by day, over every instance, and where the +requests came from: the countries and autonomous systems (the provider or +company a network belongs to) of the clients. Every list narrows the view: +open a repository to see where its requests came from, or a country, a +network or a client to see what it used. Health probes are not counted. + +Each instance counts its requests in memory, by day (UTC), repository and +client network, and writes them to Forge's bucket once a minute, as one +Parquet file per instance and day: +`traffic/v1/days//.parquet`, a row per repository and +client with its country, autonomous system, requests and bytes sent. Every +instance writes files of its own, so the page reads the files of all of +them from the bucket and sums them: it needs no Redis and no call between +instances. The files are plain Parquet, so DuckDB, pandas or Spark read them +too. + +What is recorded and how long it is kept (90 days by default) is set under +**Admin → Traffic → Settings** (`/admin/settings/traffic`), for every +instance at once. Each instance deletes the days past the retention about +once an hour, whichever instance wrote them. + +- **Client addresses are personal data.** The address is the one Forge + resolves for the login lockout: the socket peer, or the `X-Forwarded-For` + of a [trusted proxy](#reverse-proxy). An IPv6 client is counted by its + /64. With **Client addresses** off, requests are still counted by + repository, country and autonomous system: the address is looked up in + memory and dropped before anything is written. +- **Accuracy.** Today fills in as the instances write it, once a minute. An + instance that stops writes what it counted on a graceful shutdown; one + that is killed loses its last minute. Past 500,000 clients a day on one + instance, new ones are counted together as `other`. + +The statistics are separate from the [OpenTelemetry metrics](#telemetry), +which carry no address or repository per client and need a backend of their +own. + +### Countries and networks (MaxMind) + +The countries and autonomous systems come from MaxMind databases: the free +GeoLite2-Country and GeoLite2-ASN, or the commercial GeoIP2 editions. Forge +keeps them in its bucket, as `traffic/v1/geoip/country.mmdb` and +`traffic/v1/geoip/asn.mmdb`, and every instance loads them from there, +again within ten minutes of a change. A client is located when its row of +the day is first written; days written before a database was there stay +unknown. + +- **Downloaded by Forge.** Create a free MaxMind account and a license key + at [maxmind.com](https://www.maxmind.com/en/geolite2/signup) and enter + them under **Admin → Traffic → Settings → Geolocation**. The license key + is sealed like Forge's other secrets. One instance downloads the + databases at once with **Download now**, then every three days, and + stores them in the bucket; a failed download is retried six hours later, + and the settings show its error. Forge needs outbound HTTPS to + `download.maxmind.com` and the storage it redirects to; a mirror that + answers the same download API can be set as + `traffic.maxmind_url` (`FORGE_TRAFFIC__MAXMIND_URL`). **Turn off** forgets + the account and deletes the databases. +- **Put there yourself.** Without internet access, or for a commercial + edition, upload the `.mmdb` files to the bucket under those two keys (for + example with `geoipupdate` and `aws s3 cp`). A Country or City edition + fills `country.mmdb`, an ASN or ISP edition `asn.mmdb`; a file of another + edition is refused and logged. + +MaxMind's GeoLite license asks for its databases to be kept current and for +the credit the traffic screen shows next to the countries and networks. + ## Running more than one instance Instances coordinate through PostgreSQL (row locks and leases), so any number @@ -1594,7 +1665,8 @@ incomplete multipart uploads, for example after one day: (`aws s3api put-bucket-lifecycle-configuration --bucket forge --lifecycle-configuration file://lifecycle.json`; MinIO and Ceph RGW accept the same rule.) Do not add expiration rules for current objects: Forge's own -garbage collection decides when a blob may go. +garbage collection decides when a blob may go, and Forge deletes the +[traffic statistics](#traffic-statistics) past their retention itself. To restore: stop every Forge instance, restore PostgreSQL, restore the bucket as described above, then start Forge. Practise this before you need it. diff --git a/forge.toml.example b/forge.toml.example index bb769146..03db509c 100644 --- a/forge.toml.example +++ b/forge.toml.example @@ -284,6 +284,13 @@ processed_retention_seconds = 604800 # (default: 604800 = 7 days) keep processe # bytes, and the outbox backlog for the metrics (database queries). # inventory_interval_seconds = 300 # (default: 300, at least 10) +[traffic] +# The traffic statistics (Admin → Traffic; docs/deployment.md → Traffic +# statistics). What they record and the MaxMind account their country and +# ASN databases come from are set in the web UI. Where Forge downloads those +# databases: MaxMind, or a mirror answering its download API. +# maxmind_url = "https://download.maxmind.com" # (default) + # Optional: OpenID Connect / Microsoft Entra interactive login. # This block is a one-shot SEED — on first boot it is imported into an # `auth_realm` row and authentication is managed in the admin UI from then on diff --git a/web/src/api/traffic.ts b/web/src/api/traffic.ts new file mode 100644 index 00000000..f541a123 --- /dev/null +++ b/web/src/api/traffic.ts @@ -0,0 +1,203 @@ +import { httpClient } from '~/lib/httpClient' + +/** + * Admin traffic API: the traffic statistics every instance writes to the + * object store (`crates/server/src/admin/traffic.rs`, `crates/traffic`). + * + * - `GET /admin/traffic` answers the requests of a range of days over every + * instance (or the one `instance` names), by day, repository, country, + * autonomous system and client network, narrowed by the filters given. + * - `GET` / `PUT /admin/settings/traffic`: what is recorded and how long it + * is kept; `PUT` changes the fields it names. + * - `/admin/traffic/geoip`: the geolocation databases and the MaxMind + * download (`PUT` the account, `DELETE` it and the databases, `POST + * …/update` to download now). + */ + +/** The ranges the screen offers, in days up to today. */ +export type TrafficRange = 'today' | '7d' | '30d' | '90d' | '365d' + +export const TRAFFIC_RANGES: { id: TrafficRange; label: string }[] = [ + { id: 'today', label: 'Today' }, + { id: '7d', label: '7 days' }, + { id: '30d', label: '30 days' }, + { id: '90d', label: '90 days' }, + { id: '365d', label: '1 year' }, +] + +export function isTrafficRange(value: unknown): value is TrafficRange { + return TRAFFIC_RANGES.some((r) => r.id === value) +} + +/** What a filter value says for "none": no repository, no known country. */ +export const NONE = '-' + +/** An instance whose files the range holds. */ +export interface TrafficInstance { + id: string + host: string + started_at: string | null +} + +/** A day of the range. */ +export interface TrafficDay { + /** `YYYY-MM-DD`, UTC. */ + day: string + requests: number + bytes: number + /** The client networks that made requests that day. */ + clients: number +} + +/** A share of the range. */ +export interface TrafficShare { + requests: number + bytes: number + clients: number +} + +/** A client network's requests. */ +export interface TrafficClient { + network: string + address: string + /** ISO 3166-1 alpha-2; `""` unknown. */ + country: string + /** `0` unknown. */ + asn: number + as_org: string + requests: number + bytes: number + /** The repositories it made the most requests to (`""`: none). */ + repositories: string[] + days: number + first_day: string | null + last_day: string | null +} + +/** `GET /admin/traffic`. */ +export interface TrafficOverview { + from: string + to: string + instances: TrafficInstance[] + totals: { + requests: number + bytes: number + clients: number + countries: number + asns: number + } + days: TrafficDay[] + /** `""` names the requests that addressed no repository. */ + repositories: ({ name: string } & TrafficShare)[] + countries: ({ country: string } & TrafficShare)[] + asns: ({ asn: number; org: string } & TrafficShare)[] + clients: TrafficClient[] + recording: { + enabled: boolean + client_addresses: boolean + /** Whether a country database is loaded. */ + countries: boolean + /** Whether an autonomous system database is loaded. */ + asns: boolean + } +} + +/** `GET /admin/settings/traffic`. */ +export interface TrafficSettings { + enabled: boolean + client_addresses: boolean + retention_days: number +} + +/** `PUT /admin/settings/traffic` body: the fields to change. */ +export type TrafficSettingsUpdate = Partial + +/** A loaded geolocation database. */ +export interface GeoDatabase { + database_type: string + built_at: string | null + size: number +} + +/** `GET /admin/traffic/geoip`. */ +export interface GeoipStatus { + country: GeoDatabase | null + asn: GeoDatabase | null + maxmind: { + account_id: string | null + license_key_set: boolean + last_attempt: string | null + last_success: string | null + last_error: string | null + url: string + editions: string[] + } +} + +/** What the screen asks for: a range and the filters. */ +export interface TrafficQuery { + range: TrafficRange + instance?: string + repository?: string + network?: string + country?: string + asn?: string +} + +export const trafficKeys = { + all: ['traffic'] as const, + overview: (q: TrafficQuery) => ['traffic', 'overview', q] as const, + settings: () => ['settings', 'traffic'] as const, + geoip: () => ['traffic', 'geoip'] as const, +} + +/** `GET /admin/traffic`. */ +export function fetchTrafficOverview( + q: TrafficQuery, + signal?: AbortSignal, +): Promise { + return httpClient.get('/admin/traffic', { + signal, + query: { + range: q.range, + instance: q.instance, + repository: q.repository, + network: q.network, + country: q.country, + asn: q.asn, + }, + }) +} + +/** `GET /admin/settings/traffic`. */ +export function fetchTrafficSettings(signal?: AbortSignal): Promise { + return httpClient.get('/admin/settings/traffic', { signal }) +} + +/** `PUT /admin/settings/traffic`. */ +export function updateTrafficSettings(body: TrafficSettingsUpdate): Promise { + return httpClient.put('/admin/settings/traffic', body) +} + +/** `GET /admin/traffic/geoip`. */ +export function fetchGeoip(signal?: AbortSignal): Promise { + return httpClient.get('/admin/traffic/geoip', { signal }) +} + +/** `PUT /admin/traffic/geoip`: the license key left out keeps the stored one. */ +export function updateGeoip(body: { + account_id: string + license_key?: string +}): Promise { + return httpClient.put('/admin/traffic/geoip', body) +} + +/** `POST /admin/traffic/geoip/update`. */ +export function downloadGeoip(): Promise { + return httpClient.post('/admin/traffic/geoip/update') +} + +/** `DELETE /admin/traffic/geoip`. */ +export function removeGeoip(): Promise { + return httpClient.delete('/admin/traffic/geoip') +} diff --git a/web/src/components/AppRail.tsx b/web/src/components/AppRail.tsx index 6c417049..5c1c002f 100644 --- a/web/src/components/AppRail.tsx +++ b/web/src/components/AppRail.tsx @@ -28,6 +28,7 @@ const ADMINISTRATION: RailLink[] = [ { label: 'Settings', to: '/manage/settings', icon: 'Sliders' }, { label: 'Email', to: '/manage/settings/email', icon: 'Mail' }, { label: 'Retention', to: '/manage/settings/retention', icon: 'Clock' }, + { label: 'Traffic', to: '/manage/traffic', icon: 'Activity' }, { label: 'System', to: '/manage/system', icon: 'Database' }, ] diff --git a/web/src/components/TrafficChart.tsx b/web/src/components/TrafficChart.tsx new file mode 100644 index 00000000..f39a6439 --- /dev/null +++ b/web/src/components/TrafficChart.tsx @@ -0,0 +1,215 @@ +import { useEffect, useId, useRef, useState, type KeyboardEvent, type PointerEvent } from 'react' + +import type { TrafficDay } from '~/api/traffic' +import { formatBytes } from '~/lib/oci' +import { dayLabel } from '~/lib/traffic' +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '~/components/ui' + +/** What the chart plots of each day. */ +export type TrafficMetric = 'requests' | 'bytes' | 'clients' + +export const TRAFFIC_METRICS: { id: TrafficMetric; label: string }[] = [ + { id: 'requests', label: 'Requests' }, + { id: 'bytes', label: 'Data sent' }, + { id: 'clients', label: 'Clients' }, +] + +const METRIC_NAMES: Record = { + requests: 'Requests', + bytes: 'Data sent', + clients: 'Client networks', +} + +const compact = new Intl.NumberFormat(undefined, { notation: 'compact', maximumFractionDigits: 1 }) +const full = new Intl.NumberFormat() + +/** `n` as a metric's value: bytes in binary units, counts in full. */ +export function formatMetric(metric: TrafficMetric, n: number): string { + return metric === 'bytes' ? formatBytes(n) : full.format(n) +} + +/** An axis tick: compact once the axis reaches 10,000. */ +function formatTick(metric: TrafficMetric, n: number, max: number): string { + if (metric === 'bytes') return formatBytes(n) + return max >= 10_000 ? compact.format(n) : full.format(n) +} + +/** About `count` round ticks from 0 to at least `max`. */ +function niceTicks(max: number, count = 4): number[] { + if (max <= 0) return [0, 1] + const raw = max / count + const power = 10 ** Math.floor(Math.log10(raw)) + const step = [1, 2, 2.5, 5, 10].map((m) => m * power).find((s) => s >= raw) ?? raw + const ticks: number[] = [] + for (let t = 0; t <= max + step * 0.999; t += step) ticks.push(Math.round(t * 1e6) / 1e6) + return ticks +} + +/** A day on the axis: `Oct 30`. */ +function shortDay(day: string): string { + const d = new Date(`${day}T00:00:00Z`) + return Number.isNaN(d.getTime()) + ? day + : d.toLocaleDateString(undefined, { month: 'short', day: 'numeric', timeZone: 'UTC' }) +} + +/** How many days apart the axis labels may be. */ +const LABEL_EVERY = [1, 2, 7, 14, 30, 61, 91] + +const HEIGHT = 220 +const MARGIN = { top: 12, right: 16, bottom: 28, left: 64 } + +/** + * One measure of the traffic by day: an area with a 2px line, a crosshair + * that snaps to the nearest day and a tooltip with its value (the arrow keys + * step through the days), or the same days as a table. Drawn at the width it + * is given, so its text keeps its size. + */ +export function TrafficChart({ + days, + metric, + view, +}: { + days: TrafficDay[] + metric: TrafficMetric + view: 'chart' | 'table' +}) { + if (view === 'table') { + return + } + return +} + +function AreaChart({ days, metric }: { days: TrafficDay[]; metric: TrafficMetric }) { + const box = useRef(null) + const [width, setWidth] = useState(720) + const [active, setActive] = useState(null) + const tipId = useId() + + useEffect(() => { + const el = box.current + if (!el || typeof ResizeObserver === 'undefined') return + const observer = new ResizeObserver(([entry]) => { + const w = Math.round(entry.contentRect.width) + if (w > 0) setWidth(w) + }) + observer.observe(el) + return () => observer.disconnect() + }, []) + + const values = days.map((d) => d[metric]) + const max = Math.max(0, ...values) + const ticks = niceTicks(max) + const top = ticks[ticks.length - 1] || 1 + const plotW = Math.max(width - MARGIN.left - MARGIN.right, 1) + const plotH = HEIGHT - MARGIN.top - MARGIN.bottom + const x = (i: number) => + MARGIN.left + (days.length <= 1 ? plotW / 2 : (i / (days.length - 1)) * plotW) + const y = (v: number) => MARGIN.top + plotH - (v / top) * plotH + const line = days.map((_, i) => `${i === 0 ? 'M' : 'L'}${x(i)},${y(values[i])}`).join('') + const area = + days.length > 0 ? `${line}L${x(days.length - 1)},${y(0)}L${x(0)},${y(0)}Z` : '' + // Room for a label: about 6.5px a character, and a gap; counted back from + // the last day, so today always has one. + const perDay = days.length <= 1 ? plotW : plotW / (days.length - 1) + const wanted = Math.ceil((shortDay(days[0]?.day ?? '').length * 6.5 + 24) / perDay) + const every = LABEL_EVERY.find((n) => n >= wanted) ?? wanted + const labelled = (i: number) => (days.length - 1 - i) % every === 0 + const total = values.reduce((a, b) => a + b, 0) + + function nearest(e: PointerEvent): number | null { + if (days.length === 0) return null + const rect = e.currentTarget.getBoundingClientRect() + const px = e.clientX - rect.left - MARGIN.left + const i = days.length <= 1 ? 0 : Math.round((px / plotW) * (days.length - 1)) + return Math.min(days.length - 1, Math.max(0, i)) + } + + function onKey(e: KeyboardEvent) { + if (days.length === 0) return + const step = { ArrowLeft: -1, ArrowRight: 1, Home: -Infinity, End: Infinity }[e.key] + if (step === undefined) return + e.preventDefault() + const from = active ?? (step > 0 ? -1 : days.length) + setActive(Math.min(days.length - 1, Math.max(0, from + step))) + } + + const tip = active !== null ? days[active] : null + const tipLeft = active !== null ? Math.min(Math.max(x(active), 80), width - 80) : 0 + + return ( +
+ setActive(nearest(e))} + onPointerLeave={() => setActive(null)} + onBlur={() => setActive(null)} + onKeyDown={onKey} + > + {ticks.map((t) => ( + + + + {formatTick(metric, t, top)} + + + ))} + {days.map((d, i) => + labelled(i) ? ( + + {shortDay(d.day)} + + ) : null, + )} + + + {active !== null && tip && ( + + + + + )} + + {tip && active !== null && ( +
+
+
+
+ {METRIC_NAMES[metric]}, {dayLabel(tip.day)} +
+
+ )} +
+ ) +} + +/** The days of the chart as rows, newest first. */ +function DayTable({ days, metric }: { days: TrafficDay[]; metric: TrafficMetric }) { + return ( + + + Day + Requests + Data sent + Clients + + + {[...days].reverse().map((d) => ( + + {dayLabel(d.day)} + {full.format(d.requests)} + {formatBytes(d.bytes)} + {full.format(d.clients)} + + ))} + +
+ ) +} diff --git a/web/src/components/TrafficFilters.tsx b/web/src/components/TrafficFilters.tsx new file mode 100644 index 00000000..72f576ac --- /dev/null +++ b/web/src/components/TrafficFilters.tsx @@ -0,0 +1,50 @@ +import { TRAFFIC_RANGES, type TrafficInstance, type TrafficRange } from '~/api/traffic' +import { formatDateTime } from '~/lib/oci' +import { SegmentedControl, Select } from '~/components/ui' + +/** + * The filters of a traffic view, in one row above what they scope: the + * range first, then the instance (all of them, or one). + */ +export function TrafficFilters({ + range, + onRange, + instance, + instances, + onInstance, +}: { + range: TrafficRange + onRange: (range: TrafficRange) => void + instance: string | undefined + instances: TrafficInstance[] + onInstance: (instance: string | undefined) => void +}) { + // An instance named in the URL stays selectable when the range holds no + // file of it. + const known = + instance && !instances.some((i) => i.id === instance) + ? [...instances, { id: instance, host: instance, started_at: null }] + : instances + return ( +
+ ({ id: r.id, label: r.label }))} + /> + +
+ ) +} diff --git a/web/src/components/ui/icons.tsx b/web/src/components/ui/icons.tsx index f72fbbeb..0272eb0c 100644 --- a/web/src/components/ui/icons.tsx +++ b/web/src/components/ui/icons.tsx @@ -81,6 +81,7 @@ const ICONS = { Eye: [EYE, circ(10, 10, 2.5)], EyeOff: [EYE, circ(10, 10, 2.5), 'M3 3l14 14'], Clock: [circ(10, 10, 7), 'M10 6v4l2.5 2.5'], + Activity: ['M2.5 10.5h3l2.2-5.5 4.6 10 2.2-4.5h3'], Alert: ['M10 3l8 14H2z', 'M10 8.5V12', 'M10 14.6v.1'], Info: [circ(10, 10, 7), 'M10 9v5', 'M10 6.4v.1'], X: ['M5 5l10 10', 'M15 5L5 15'], diff --git a/web/src/lib/traffic.ts b/web/src/lib/traffic.ts new file mode 100644 index 00000000..7c7d4686 --- /dev/null +++ b/web/src/lib/traffic.ts @@ -0,0 +1,49 @@ +/** + * How the traffic views name what the statistics count: countries, + * autonomous systems, repositories, shares. + */ + +const regions = (() => { + try { + return new Intl.DisplayNames(undefined, { type: 'region' }) + } catch { + return null + } +})() + +/** A country's name from its ISO code (`DE` → Germany); `Unknown` for none. */ +export function countryName(code: string): string { + if (!code) return 'Unknown' + try { + return regions?.of(code) ?? code + } catch { + return code + } +} + +/** An autonomous system: `AS3320 Deutsche Telekom AG`; `Unknown` for none. */ +export function asnLabel(asn: number, org: string): string { + if (!asn) return 'Unknown' + return org ? `AS${asn} ${org}` : `AS${asn}` +} + +/** A repository's name; what `""` stands for: requests to no repository. */ +export function repositoryLabel(name: string): string { + return name || 'No repository (web UI, APIs, sign-in)' +} + +const percent = new Intl.NumberFormat(undefined, { style: 'percent', maximumFractionDigits: 1 }) + +/** `part` of `whole` as a percentage; an em dash without a whole. */ +export function share(part: number, whole: number): string { + return whole > 0 ? percent.format(part / whole) : '—' +} + +/** A `YYYY-MM-DD` day as the locale writes a date (`Oct 30, 2026`). */ +export function dayLabel(day: string | null | undefined): string { + if (!day) return '—' + const d = new Date(`${day}T00:00:00Z`) + return Number.isNaN(d.getTime()) + ? day + : d.toLocaleDateString(undefined, { dateStyle: 'medium', timeZone: 'UTC' }) +} diff --git a/web/src/queries/useTraffic.ts b/web/src/queries/useTraffic.ts new file mode 100644 index 00000000..80fa3744 --- /dev/null +++ b/web/src/queries/useTraffic.ts @@ -0,0 +1,94 @@ +import { keepPreviousData, useMutation, useQuery, useQueryClient } from '@tanstack/react-query' + +import type { ApiError } from '~/lib/httpClient' +import { + downloadGeoip, + fetchGeoip, + fetchTrafficOverview, + fetchTrafficSettings, + removeGeoip, + trafficKeys, + updateGeoip, + updateTrafficSettings, + type GeoipStatus, + type TrafficOverview, + type TrafficQuery, + type TrafficSettings, + type TrafficSettingsUpdate, +} from '~/api/traffic' + +/** How often an open traffic view is read again: today's files change every minute. */ +const REFRESH_MS = 60_000 + +/** + * The requests of a range (`GET /admin/traffic`). While another range or + * filter loads, the previous answer stays on screen. + */ +export function useTrafficOverview(q: TrafficQuery) { + return useQuery({ + queryKey: trafficKeys.overview(q), + queryFn: ({ signal }) => fetchTrafficOverview(q, signal), + placeholderData: keepPreviousData, + refetchInterval: REFRESH_MS, + }) +} + +/** What is recorded and how long it is kept (`GET /admin/settings/traffic`). */ +export function useTrafficSettings() { + return useQuery({ + queryKey: trafficKeys.settings(), + queryFn: ({ signal }) => fetchTrafficSettings(signal), + }) +} + +/** + * Change the traffic settings; the answer replaces the cached ones, and the + * traffic views (which say what is recorded) are read again. + */ +export function useUpdateTrafficSettings() { + const queryClient = useQueryClient() + return useMutation({ + mutationFn: (body) => updateTrafficSettings(body), + onSuccess: (saved) => { + queryClient.setQueryData(trafficKeys.settings(), saved) + void queryClient.invalidateQueries({ queryKey: trafficKeys.all }) + }, + }) +} + +/** The geolocation databases and the MaxMind download (`GET /admin/traffic/geoip`). */ +export function useGeoip() { + return useQuery({ + queryKey: trafficKeys.geoip(), + queryFn: ({ signal }) => fetchGeoip(signal), + }) +} + +/** Answers that replace the geolocation status, and make the traffic views be read again. */ +function useGeoipMutation(fn: (v: V) => Promise) { + const queryClient = useQueryClient() + return useMutation({ + mutationFn: fn, + onSuccess: (status) => { + if (status) queryClient.setQueryData(trafficKeys.geoip(), status) + void queryClient.invalidateQueries({ queryKey: trafficKeys.all }) + }, + }) +} + +/** Set the MaxMind account ID and license key. */ +export function useUpdateGeoip() { + return useGeoipMutation((body: { account_id: string; license_key?: string }) => + updateGeoip(body), + ) +} + +/** Download the databases now. */ +export function useDownloadGeoip() { + return useGeoipMutation((_: void) => downloadGeoip()) +} + +/** Forget the MaxMind account and delete the databases. */ +export function useRemoveGeoip() { + return useGeoipMutation((_: void) => removeGeoip()) +} diff --git a/web/src/routeTree.gen.ts b/web/src/routeTree.gen.ts index a87b8c8a..c9cfcb1f 100644 --- a/web/src/routeTree.gen.ts +++ b/web/src/routeTree.gen.ts @@ -24,6 +24,7 @@ import { Route as adminRolesListRouteImport } from './routes/admin/roles/List' import { Route as adminServiceAccountsListRouteImport } from './routes/admin/service-accounts/List' import { Route as adminSettingsSettingsRouteImport } from './routes/admin/settings/Settings' import { Route as adminSystemSystemRouteImport } from './routes/admin/system/System' +import { Route as adminTrafficTrafficRouteImport } from './routes/admin/traffic/Traffic' import { Route as adminUsersListRouteImport } from './routes/admin/users/List' import { Route as RepoBrowseRouteImport } from './routes/RepoBrowse' import { Route as ImageDetailRouteImport } from './routes/ImageDetail' @@ -38,6 +39,7 @@ import { Route as adminServiceAccountsEditRouteImport } from './routes/admin/ser import { Route as adminServiceAccountsCreateRouteImport } from './routes/admin/service-accounts/Create' import { Route as adminSettingsEmailRouteImport } from './routes/admin/settings/Email' import { Route as adminSettingsRetentionSettingsRouteImport } from './routes/admin/settings/RetentionSettings' +import { Route as adminTrafficSettingsRouteImport } from './routes/admin/traffic/Settings' import { Route as adminUsersEditRouteImport } from './routes/admin/users/Edit' import { Route as adminUsersCreateRouteImport } from './routes/admin/users/Create' import { Route as PackageDetailRouteImport } from './routes/PackageDetail' @@ -120,6 +122,11 @@ const adminSystemSystemRoute = adminSystemSystemRouteImport.update({ path: '/system', getParentRoute: () => AdminLayoutRoute, } as any) +const adminTrafficTrafficRoute = adminTrafficTrafficRouteImport.update({ + id: '/traffic', + path: '/traffic', + getParentRoute: () => AdminLayoutRoute, +} as any) const adminUsersListRoute = adminUsersListRouteImport.update({ id: '/users', path: '/users', @@ -193,6 +200,11 @@ const adminSettingsRetentionSettingsRoute = path: '/settings/retention', getParentRoute: () => AdminLayoutRoute, } as any) +const adminTrafficSettingsRoute = adminTrafficSettingsRouteImport.update({ + id: '/traffic/settings', + path: '/traffic/settings', + getParentRoute: () => AdminLayoutRoute, +} as any) const adminUsersEditRoute = adminUsersEditRouteImport.update({ id: '/users/$userId', path: '/users/$userId', @@ -241,6 +253,7 @@ export interface FileRoutesByFullPath { '/manage/service-accounts': typeof adminServiceAccountsListRoute '/manage/settings': typeof adminSettingsSettingsRoute '/manage/system': typeof adminSystemSystemRoute + '/manage/traffic': typeof adminTrafficTrafficRoute '/manage/users': typeof adminUsersListRoute '/repos/$repo': typeof RepoBrowseRoute '/images/$repo/$': typeof ImageDetailRoute @@ -255,6 +268,7 @@ export interface FileRoutesByFullPath { '/manage/service-accounts/new': typeof adminServiceAccountsCreateRoute '/manage/settings/email': typeof adminSettingsEmailRoute '/manage/settings/retention': typeof adminSettingsRetentionSettingsRoute + '/manage/traffic/settings': typeof adminTrafficSettingsRoute '/manage/users/$userId': typeof adminUsersEditRoute '/manage/users/new': typeof adminUsersCreateRoute '/packages/$repo/$id': typeof PackageDetailRoute @@ -276,6 +290,7 @@ export interface FileRoutesByTo { '/manage/service-accounts': typeof adminServiceAccountsListRoute '/manage/settings': typeof adminSettingsSettingsRoute '/manage/system': typeof adminSystemSystemRoute + '/manage/traffic': typeof adminTrafficTrafficRoute '/manage/users': typeof adminUsersListRoute '/repos/$repo': typeof RepoBrowseRoute '/images/$repo/$': typeof ImageDetailRoute @@ -290,6 +305,7 @@ export interface FileRoutesByTo { '/manage/service-accounts/new': typeof adminServiceAccountsCreateRoute '/manage/settings/email': typeof adminSettingsEmailRoute '/manage/settings/retention': typeof adminSettingsRetentionSettingsRoute + '/manage/traffic/settings': typeof adminTrafficSettingsRoute '/manage/users/$userId': typeof adminUsersEditRoute '/manage/users/new': typeof adminUsersCreateRoute '/packages/$repo/$id': typeof PackageDetailRoute @@ -314,6 +330,7 @@ export interface FileRoutesById { '/_app/manage/service-accounts': typeof adminServiceAccountsListRoute '/_app/manage/settings': typeof adminSettingsSettingsRoute '/_app/manage/system': typeof adminSystemSystemRoute + '/_app/manage/traffic': typeof adminTrafficTrafficRoute '/_app/manage/users': typeof adminUsersListRoute '/_app/repos/$repo': typeof RepoBrowseRoute '/_app/images/$repo/$': typeof ImageDetailRoute @@ -328,6 +345,7 @@ export interface FileRoutesById { '/_app/manage/service-accounts/new': typeof adminServiceAccountsCreateRoute '/_app/manage/settings/email': typeof adminSettingsEmailRoute '/_app/manage/settings/retention': typeof adminSettingsRetentionSettingsRoute + '/_app/manage/traffic/settings': typeof adminTrafficSettingsRoute '/_app/manage/users/$userId': typeof adminUsersEditRoute '/_app/manage/users/new': typeof adminUsersCreateRoute '/_app/packages/$repo/$id': typeof PackageDetailRoute @@ -352,6 +370,7 @@ export interface FileRouteTypes { | '/manage/service-accounts' | '/manage/settings' | '/manage/system' + | '/manage/traffic' | '/manage/users' | '/repos/$repo' | '/images/$repo/$' @@ -366,6 +385,7 @@ export interface FileRouteTypes { | '/manage/service-accounts/new' | '/manage/settings/email' | '/manage/settings/retention' + | '/manage/traffic/settings' | '/manage/users/$userId' | '/manage/users/new' | '/packages/$repo/$id' @@ -387,6 +407,7 @@ export interface FileRouteTypes { | '/manage/service-accounts' | '/manage/settings' | '/manage/system' + | '/manage/traffic' | '/manage/users' | '/repos/$repo' | '/images/$repo/$' @@ -401,6 +422,7 @@ export interface FileRouteTypes { | '/manage/service-accounts/new' | '/manage/settings/email' | '/manage/settings/retention' + | '/manage/traffic/settings' | '/manage/users/$userId' | '/manage/users/new' | '/packages/$repo/$id' @@ -424,6 +446,7 @@ export interface FileRouteTypes { | '/_app/manage/service-accounts' | '/_app/manage/settings' | '/_app/manage/system' + | '/_app/manage/traffic' | '/_app/manage/users' | '/_app/repos/$repo' | '/_app/images/$repo/$' @@ -438,6 +461,7 @@ export interface FileRouteTypes { | '/_app/manage/service-accounts/new' | '/_app/manage/settings/email' | '/_app/manage/settings/retention' + | '/_app/manage/traffic/settings' | '/_app/manage/users/$userId' | '/_app/manage/users/new' | '/_app/packages/$repo/$id' @@ -559,6 +583,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof adminSystemSystemRouteImport parentRoute: typeof AdminLayoutRoute } + '/_app/manage/traffic': { + id: '/_app/manage/traffic' + path: '/traffic' + fullPath: '/manage/traffic' + preLoaderRoute: typeof adminTrafficTrafficRouteImport + parentRoute: typeof AdminLayoutRoute + } '/_app/manage/users': { id: '/_app/manage/users' path: '/users' @@ -657,6 +688,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof adminSettingsRetentionSettingsRouteImport parentRoute: typeof AdminLayoutRoute } + '/_app/manage/traffic/settings': { + id: '/_app/manage/traffic/settings' + path: '/traffic/settings' + fullPath: '/manage/traffic/settings' + preLoaderRoute: typeof adminTrafficSettingsRouteImport + parentRoute: typeof AdminLayoutRoute + } '/_app/manage/users/$userId': { id: '/_app/manage/users/$userId' path: '/users/$userId' @@ -711,6 +749,7 @@ interface AdminLayoutRouteChildren { adminServiceAccountsListRoute: typeof adminServiceAccountsListRoute adminSettingsSettingsRoute: typeof adminSettingsSettingsRoute adminSystemSystemRoute: typeof adminSystemSystemRoute + adminTrafficTrafficRoute: typeof adminTrafficTrafficRoute adminUsersListRoute: typeof adminUsersListRoute adminSelectorsEditRoute: typeof adminSelectorsEditRoute adminSelectorsCreateRoute: typeof adminSelectorsCreateRoute @@ -723,6 +762,7 @@ interface AdminLayoutRouteChildren { adminServiceAccountsCreateRoute: typeof adminServiceAccountsCreateRoute adminSettingsEmailRoute: typeof adminSettingsEmailRoute adminSettingsRetentionSettingsRoute: typeof adminSettingsRetentionSettingsRoute + adminTrafficSettingsRoute: typeof adminTrafficSettingsRoute adminUsersEditRoute: typeof adminUsersEditRoute adminUsersCreateRoute: typeof adminUsersCreateRoute adminRepositoriesImportRoute: typeof adminRepositoriesImportRoute @@ -739,6 +779,7 @@ const AdminLayoutRouteChildren: AdminLayoutRouteChildren = { adminServiceAccountsListRoute: adminServiceAccountsListRoute, adminSettingsSettingsRoute: adminSettingsSettingsRoute, adminSystemSystemRoute: adminSystemSystemRoute, + adminTrafficTrafficRoute: adminTrafficTrafficRoute, adminUsersListRoute: adminUsersListRoute, adminSelectorsEditRoute: adminSelectorsEditRoute, adminSelectorsCreateRoute: adminSelectorsCreateRoute, @@ -751,6 +792,7 @@ const AdminLayoutRouteChildren: AdminLayoutRouteChildren = { adminServiceAccountsCreateRoute: adminServiceAccountsCreateRoute, adminSettingsEmailRoute: adminSettingsEmailRoute, adminSettingsRetentionSettingsRoute: adminSettingsRetentionSettingsRoute, + adminTrafficSettingsRoute: adminTrafficSettingsRoute, adminUsersEditRoute: adminUsersEditRoute, adminUsersCreateRoute: adminUsersCreateRoute, adminRepositoriesImportRoute: adminRepositoriesImportRoute, diff --git a/web/src/routes.ts b/web/src/routes.ts index 0e56848e..116d860b 100644 --- a/web/src/routes.ts +++ b/web/src/routes.ts @@ -23,7 +23,12 @@ import { index, layout, rootRoute, route } from '@tanstack/virtual-file-routes' * /tokens -> Tokens.tsx (self-service registry tokens) * /cli-login/$secret -> CliLogin.tsx (approve a command-line client's * web login: `npm login`) - * /manage -> AdminLayout.tsx (beforeLoad: require is_admin) + * /manage -> AdminLayout.tsx (beforeLoad: require is_admin; + * /manage/traffic: every instance's + * requests by day, repository and + * client; /traffic/settings: what + * is recorded and kept, and the + * geolocation databases) * * The admin screens live under `/manage/*`, NOT `/admin/*`: `/admin/*` is the * admin REST API, which the server routes before the SPA fallback, so a UI @@ -72,6 +77,8 @@ export const routes = rootRoute('root.tsx', [ route('/settings', 'admin/settings/Settings.tsx'), route('/settings/email', 'admin/settings/Email.tsx'), route('/settings/retention', 'admin/settings/RetentionSettings.tsx'), + route('/traffic', 'admin/traffic/Traffic.tsx'), + route('/traffic/settings', 'admin/traffic/Settings.tsx'), route('/system', 'admin/system/System.tsx'), ]), ]), diff --git a/web/src/routes/admin/AdminHome.tsx b/web/src/routes/admin/AdminHome.tsx index bc703b03..e1d4427b 100644 --- a/web/src/routes/admin/AdminHome.tsx +++ b/web/src/routes/admin/AdminHome.tsx @@ -67,6 +67,13 @@ const SECTIONS = [ description: 'Whether retention runs, for every repository, and how long its audit log is kept.', }, + { + to: '/manage/traffic', + title: 'Traffic', + icon: 'Activity', + description: + 'Requests of every instance over time, by repository and status; the top clients and their request logs.', + }, { to: '/manage/system', title: 'System', @@ -84,8 +91,8 @@ const SECTIONS = [ * Admin landing. Only reachable once the `AdminLayout` guard has confirmed * `is_admin`. A card per admin screen, with the rail's icon for it: the * repositories, users (incl. per-user tokens and roles), service accounts, roles, content - * selectors, authentication-realm, settings, email-service, retention and - * system screens. + * selectors, authentication-realm, settings, email-service, retention, + * traffic and system screens. */ function AdminHome() { return ( @@ -97,8 +104,8 @@ function AdminHome() { Manage repositories and their retention policies, users and service accounts and their API tokens, roles and their grants, content selectors, authentication realms, the sign-in and - anonymous-access settings, the email service, and the secret - encryption and blob cache. + anonymous-access settings, the email service, the traffic of every + instance, and the secret encryption and blob cache. } /> diff --git a/web/src/routes/admin/traffic/Settings.tsx b/web/src/routes/admin/traffic/Settings.tsx new file mode 100644 index 00000000..26870f64 --- /dev/null +++ b/web/src/routes/admin/traffic/Settings.tsx @@ -0,0 +1,312 @@ +import { useState, type FormEvent } from 'react' +import { createFileRoute } from '@tanstack/react-router' + +import type { GeoDatabase, GeoipStatus, TrafficSettings } from '~/api/traffic' +import { formatBytes, formatDateTime } from '~/lib/oci' +import { + useDownloadGeoip, + useGeoip, + useRemoveGeoip, + useTrafficSettings, + useUpdateGeoip, + useUpdateTrafficSettings, +} from '~/queries/useTraffic' +import { + Button, + Code, + Dialog, + FactList, + FormActions, + Notice, + Page, + PageHeader, + Panel, + Skeleton, + Switch, + TextField, +} from '~/components/ui' +import { ErrorState } from '~/components/ErrorState' + +export const Route = createFileRoute('/_app/manage/traffic/settings')({ + component: TrafficSettingsScreen, +}) + +const DESCRIPTION = + 'What the traffic statistics record, how long they are kept, and where the countries and networks of the clients come from, for every instance at once. Each instance follows a change within a minute.' + +/** The longest retention the server accepts, in days. */ +const MAX_DAYS = 36_500 + +/** + * Traffic settings (`/admin/settings/traffic`, `/admin/traffic/geoip`): + * what is recorded and how long it is kept, and the MaxMind account the + * geolocation databases are downloaded with. Each panel saves its own + * fields (`PUT` changes only what it names), so saving one never undoes a + * change another administrator made to the other. + */ +function TrafficSettingsScreen() { + const settings = useTrafficSettings() + + return ( + + + {settings.isError ? ( + void settings.refetch()} + /> + ) : settings.isLoading || !settings.data ? ( + + ) : ( + <> + + + + )} + + ) +} + +/** A whole number of days, or the error to show. */ +function days(value: string): number | string { + const n = value.trim() + if (!/^\d+$/.test(n) || Number(n) < 1 || Number(n) > MAX_DAYS) { + return `Enter a whole number of days from 1 to ${MAX_DAYS.toLocaleString()}.` + } + return Number(n) +} + +/** What is recorded and how long it is kept. */ +function RecordingForm({ settings }: { settings: TrafficSettings }) { + const update = useUpdateTrafficSettings() + const [enabled, setEnabled] = useState(settings.enabled) + const [addresses, setAddresses] = useState(settings.client_addresses) + const [retention, setRetention] = useState(String(settings.retention_days)) + const [error, setError] = useState(null) + + function handleSubmit(e: FormEvent) { + e.preventDefault() + const kept = days(retention) + if (typeof kept === 'string') { + setError(kept) + return + } + setError(null) + update.mutate({ enabled, client_addresses: addresses, retention_days: kept }) + } + + return ( + + {update.isPending ? 'Saving…' : 'Save recording'} + + } + > +
+ + + setRetention(e.target.value)} + error={error} + description="Each instance deletes older days about once an hour." + /> + {update.isError && ( + {update.error.body || update.error.message} + )} + {update.isSuccess && Saved.} +
+ + ) +} + +function database(db: GeoDatabase | null): string { + if (!db) return 'Not loaded' + const built = db.built_at ? `, built ${formatDateTime(db.built_at)}` : '' + return `${db.database_type}${built} (${formatBytes(db.size)})` +} + +/** The geolocation databases and the MaxMind account they come from. */ +function GeolocationPanel() { + const status = useGeoip() + return ( + + The country and autonomous system (the provider or company) of each + client come from MaxMind's free GeoLite2 databases. With a + MaxMind account ID and license key, Forge downloads them every three + days and keeps them in its bucket for every instance. Without + internet access, put the .mmdb files there yourself as{' '} + traffic/v1/geoip/country.mmdb and{' '} + traffic/v1/geoip/asn.mmdb. + + } + > + {status.isError ? ( + void status.refetch()} + /> + ) : !status.data ? ( + + ) : ( + + )} + + ) +} + +function GeolocationBody({ status }: { status: GeoipStatus }) { + const update = useUpdateGeoip() + const download = useDownloadGeoip() + const remove = useRemoveGeoip() + const [accountId, setAccountId] = useState(status.maxmind.account_id ?? '') + const [licenseKey, setLicenseKey] = useState('') + const [errors, setErrors] = useState<{ account?: string; key?: string }>({}) + const [confirming, setConfirming] = useState(false) + const m = status.maxmind + const configured = m.account_id !== null && m.license_key_set + + function handleSubmit(e: FormEvent) { + e.preventDefault() + const found: { account?: string; key?: string } = {} + if (!/^\d+$/.test(accountId.trim())) found.account = 'Enter your MaxMind account ID, a number.' + if (!licenseKey.trim() && !m.license_key_set) found.key = 'Enter a license key.' + setErrors(found) + if (Object.keys(found).length > 0) return + update.mutate( + { + account_id: accountId.trim(), + ...(licenseKey.trim() ? { license_key: licenseKey.trim() } : {}), + }, + { onSuccess: () => setLicenseKey('') }, + ) + } + + return ( +
+ + {m.last_error && ( + + {m.last_error} + + )} +
+
+ setAccountId(e.target.value)} + error={errors.account} + /> + setLicenseKey(e.target.value)} + error={errors.key} + placeholder={m.license_key_set ? '••••••••' : undefined} + description={m.license_key_set ? 'Stored. Enter a new one to replace it.' : undefined} + /> +
+

+ Create a free account and a license key at{' '} + + maxmind.com + + . The key is stored encrypted. Forge downloads {m.editions.join(' and ')} from{' '} + {m.url}. +

+ {update.isError && ( + {update.error.body || update.error.message} + )} + {download.isError && ( + {download.error.body || download.error.message} + )} + {download.isSuccess && !m.last_error && ( + Downloaded: every instance uses the databases within ten minutes. + )} + + {configured && ( + + )} + {configured && ( + + )} + + +
+ setConfirming(false)} + icon="Globe" + tone="danger" + title="Turn geolocation off?" + description="Forge forgets the MaxMind account and deletes the databases from its bucket. Requests are no longer located; the countries already counted stay." + confirmLabel="Turn off" + onConfirm={() => { + remove.mutate(undefined, { + onSuccess: () => { + setAccountId('') + setLicenseKey('') + }, + }) + setConfirming(false) + }} + /> +
+ ) +} diff --git a/web/src/routes/admin/traffic/Traffic.tsx b/web/src/routes/admin/traffic/Traffic.tsx new file mode 100644 index 00000000..91420511 --- /dev/null +++ b/web/src/routes/admin/traffic/Traffic.tsx @@ -0,0 +1,416 @@ +import { useState, type ReactNode } from 'react' +import { createFileRoute, Link, useNavigate } from '@tanstack/react-router' + +import { isTrafficRange, NONE, type TrafficOverview, type TrafficRange } from '~/api/traffic' +import { formatBytes } from '~/lib/oci' +import { asnLabel, countryName, dayLabel, repositoryLabel, share } from '~/lib/traffic' +import { useTrafficOverview } from '~/queries/useTraffic' +import { TRAFFIC_METRICS, TrafficChart, type TrafficMetric } from '~/components/TrafficChart' +import { TrafficFilters } from '~/components/TrafficFilters' +import { ErrorState } from '~/components/ErrorState' +import { + Button, + EmptyState, + Notice, + Page, + PageHeader, + Panel, + SegmentedControl, + Skeleton, + Stat, + StatRow, + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from '~/components/ui' + +/** The screen's URL state: the range, an instance, and what to narrow to. */ +export interface TrafficSearch { + range?: TrafficRange + instance?: string + /** `-` for the requests to no repository. */ + repository?: string + network?: string + /** ISO code; `-` for unknown. */ + country?: string + /** `0` for unknown. */ + asn?: string +} + +function text(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined +} + +export const Route = createFileRoute('/_app/manage/traffic')({ + validateSearch: (search: Record): TrafficSearch => ({ + range: isTrafficRange(search.range) ? search.range : undefined, + instance: text(search.instance), + repository: text(search.repository), + network: text(search.network), + country: text(search.country), + asn: /^\d+$/.test(String(search.asn ?? '')) ? String(search.asn) : undefined, + }), + component: TrafficScreen, +}) + +const count = new Intl.NumberFormat() + +/** The filters a list row sets: what it is, in the URL's words. */ +type Narrow = Pick + +/** + * The traffic screen (`/admin/traffic`): how many requests each client + * network made to each repository, by day, and where they came from + * (countries and autonomous systems, with a geolocation database). Every + * list narrows the view: open a repository to see where its requests came + * from, a country or a network to see what it asked for. The range, the + * instance and the filters are in the URL, so a view can be shared. + */ +function TrafficScreen() { + const search = Route.useSearch() + const navigate = useNavigate({ from: Route.fullPath }) + const range = search.range ?? '30d' + const overview = useTrafficOverview({ + range, + instance: search.instance, + repository: search.repository, + network: search.network, + country: search.country, + asn: search.asn, + }) + const set = (next: Partial) => + void navigate({ search: (prev) => ({ ...prev, ...next }) }) + + return ( + + + Settings + + } + /> + set({ range: r })} + instance={search.instance} + instances={overview.data?.instances ?? []} + onInstance={(i) => set({ instance: i })} + /> + + {overview.isError ? ( + void overview.refetch()} + /> + ) : !overview.data ? ( + + ) : ( +
+ +
+ )} + + ) +} + +/** The filters in force, each with a button that removes it. */ +function ActiveFilters({ + search, + onClear, +}: { + search: TrafficSearch + onClear: (next: Partial) => void +}) { + const active: { key: keyof Narrow; label: string }[] = [] + if (search.repository) { + active.push({ + key: 'repository', + label: `Repository: ${search.repository === NONE ? repositoryLabel('') : search.repository}`, + }) + } + if (search.country) { + active.push({ + key: 'country', + label: `Country: ${countryName(search.country === NONE ? '' : search.country)}`, + }) + } + if (search.asn) { + active.push({ key: 'asn', label: `Network: ${search.asn === '0' ? 'Unknown' : `AS${search.asn}`}` }) + } + if (search.network) active.push({ key: 'network', label: `Client: ${search.network}` }) + if (active.length === 0) return null + return ( +
+ {active.map((f) => ( + + ))} + {active.length > 1 && ( + + )} +
+ ) +} + +/** A link that narrows the view to `narrow`, keeping the range and instance. */ +function NarrowLink({ narrow, children, mono }: { narrow: Narrow; children: ReactNode; mono?: boolean }) { + return ( + ({ ...prev, ...narrow })} + className={mono ? 'fg-link fg-mono' : 'fg-link'} + > + {children} + + ) +} + +function Body({ data }: { data: TrafficOverview }) { + const [metric, setMetric] = useState('requests') + const [view, setView] = useState<'chart' | 'table'>('chart') + const { totals, recording } = data + const located = recording.countries || recording.asns || totals.countries > 0 || totals.asns > 0 + return ( + <> + {!recording.enabled && ( + + Recording is switched off in the traffic settings: no instance counts + requests until it is switched on again. What was written before stays + until it expires. + + )} + + + + + + + + m.id === metric)?.label}, from ${dayLabel(data.from)} to ${dayLabel(data.to)} (UTC). Today fills in as the instances write it, once a minute.`} + actions={ +
+ + +
+ } + > + {totals.requests === 0 ? ( + + Nothing was counted here yet. Each instance writes what it counted + once a minute. + + ) : ( + + )} + + +
+ + {data.repositories.length === 0 ? ( + + ) : ( + + + Repository + Requests + Data sent + Clients + + + {data.repositories.map((r) => ( + + + {repositoryLabel(r.name)} + + {count.format(r.requests)} + {formatBytes(r.bytes)} + {count.format(r.clients)} + + ))} + +
+ )} + + c.country).length} + flush + footer={located ? : undefined} + > + {!located ? ( + + Set a MaxMind account in the{' '} + + traffic settings + {' '} + to see where the requests come from. + + ) : ( + + + Country + Requests + Share + Clients + + + {data.countries.map((c) => ( + + + {countryName(c.country)} + {c.country && {c.country}} + + {count.format(c.requests)} + {share(c.requests, totals.requests)} + {count.format(c.clients)} + + ))} + +
+ )} + +
+ + {located && ( + } + > + + + Autonomous system + Requests + Share + Data sent + Clients + + + {data.asns.map((a) => ( + + + {asnLabel(a.asn, a.org)} + + {count.format(a.requests)} + {share(a.requests, totals.requests)} + {formatBytes(a.bytes)} + {count.format(a.clients)} + + ))} + +
+ + )} + + + {!recording.client_addresses && ( + + Client addresses are not recorded: the traffic settings switch them + off. Requests are still counted by repository and country. + + )} + {data.clients.length === 0 ? ( + + ) : ( + + + Client + Requests + Data sent + {located && Country} + {located && Network} + Repositories + Days + Last seen + + + {data.clients.map((c) => ( + + + + {c.network} + + {c.address && c.address !== c.network && ( +
{c.address}
+ )} + + {count.format(c.requests)} + {formatBytes(c.bytes)} + {located && {countryName(c.country)}} + {located && {asnLabel(c.asn, c.as_org)}} + {c.repositories.map(repositoryLabel).join(', ')} + {count.format(c.days)} + {dayLabel(c.last_day)} + + ))} + +
+ )} + + + ) +} + +/** The credit MaxMind's GeoLite license asks for wherever its data is shown. */ +function Attribution() { + return ( +

+ Countries and networks: GeoLite Data created by MaxMind, available from{' '} + + maxmind.com + + . +

+ ) +} diff --git a/web/src/routes/admin/traffic/traffic.test.tsx b/web/src/routes/admin/traffic/traffic.test.tsx new file mode 100644 index 00000000..99ec4167 --- /dev/null +++ b/web/src/routes/admin/traffic/traffic.test.tsx @@ -0,0 +1,298 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { screen, waitFor, within } from '@testing-library/react' +import userEvent from '@testing-library/user-event' + +import { clearSession, setSession } from '~/lib/auth' +import type { Me } from '~/api/me' +import type { GeoipStatus, TrafficOverview, TrafficSettings } from '~/api/traffic' +import { renderApp } from '~/test/renderApp' + +/** + * The traffic screens (`/manage/traffic` and its settings), driven against + * the real route tree and query hooks: + * - the totals and the lists by repository, country, network and client; + * each row narrows the view, the filters are sent and kept in the URL and + * removed one by one; + * - without a geolocation database, the countries say how to get one; + * - the settings, the MaxMind account (the key never shown) and its + * download, and turning geolocation off after confirming. + */ + +const adminMe: Me = { + user: { id: 1, username: 'root', is_admin: true, active: true }, + is_admin: true, + capabilities: ['repo.read', 'repo.admin'], +} + +const recording = { enabled: true, client_addresses: true, countries: true, asns: true } + +const overview: TrafficOverview = { + from: '2026-10-24', + to: '2026-10-30', + instances: [ + { id: 'web-1_20261030T100000Z_00a1', host: 'web-1', started_at: '2026-10-30T10:00:00Z' }, + { id: 'web-2_20261030T110000Z_00b2', host: 'web-2', started_at: '2026-10-30T11:00:00Z' }, + ], + totals: { requests: 1234, bytes: 5 * 1024 ** 2, clients: 3, countries: 2, asns: 2 }, + days: [ + { day: '2026-10-29', requests: 600, bytes: 1024, clients: 2 }, + { day: '2026-10-30', requests: 634, bytes: 2048, clients: 3 }, + ], + repositories: [ + { name: 'npm-proxy', requests: 1000, bytes: 4 * 1024 ** 2, clients: 2 }, + { name: '', requests: 234, bytes: 1024 ** 2, clients: 3 }, + ], + countries: [ + { country: 'DE', requests: 900, bytes: 3 * 1024 ** 2, clients: 1 }, + { country: '', requests: 334, bytes: 2 * 1024 ** 2, clients: 2 }, + ], + asns: [{ asn: 3320, org: 'Deutsche Telekom AG', requests: 900, bytes: 3 * 1024 ** 2, clients: 1 }], + clients: [ + { + network: '2001:db8:1:2::/64', + address: '2001:db8:1:2::7', + country: 'DE', + asn: 3320, + as_org: 'Deutsche Telekom AG', + requests: 900, + bytes: 3 * 1024 ** 2, + repositories: ['npm-proxy', ''], + days: 2, + first_day: '2026-10-29', + last_day: '2026-10-30', + }, + ], + recording, +} + +const settings: TrafficSettings = { enabled: true, client_addresses: true, retention_days: 90 } + +const geoip = (over: Partial = {}): GeoipStatus => ({ + country: { database_type: 'GeoLite2-Country', built_at: '2026-10-28T00:00:00Z', size: 9_000_000 }, + asn: { database_type: 'GeoLite2-ASN', built_at: '2026-10-28T00:00:00Z', size: 8_000_000 }, + maxmind: { + account_id: '123456', + license_key_set: true, + last_attempt: '2026-10-30T10:00:00Z', + last_success: '2026-10-30T10:00:00Z', + last_error: null, + url: 'https://download.maxmind.com', + editions: ['GeoLite2-Country', 'GeoLite2-ASN'], + ...over, + }, +}) + +function json(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }) +} + +const fetchMock = vi.fn() + +function mockApi(h: { overview?: TrafficOverview; geoip?: GeoipStatus } = {}) { + // The geolocation status, as the server keeps it between requests. + let geo = h.geoip ?? geoip() + fetchMock.mockImplementation((input: RequestInfo | URL, init?: RequestInit) => { + const url = new URL(String(input), 'http://localhost') + const method = (init?.method ?? 'GET').toUpperCase() + const body = init?.body ? (JSON.parse(String(init.body)) as Record) : {} + if (url.pathname.endsWith('/admin/me')) return Promise.resolve(json(adminMe)) + if (url.pathname.endsWith('/admin/traffic')) return Promise.resolve(json(h.overview ?? overview)) + if (url.pathname.endsWith('/admin/settings/traffic')) { + return Promise.resolve(json(method === 'PUT' ? { ...settings, ...body } : settings)) + } + if (url.pathname.endsWith('/admin/traffic/geoip/update')) { + geo = { ...geo, maxmind: { ...geo.maxmind, last_error: 'MaxMind refused the account ID or the license key (401)' } } + return Promise.resolve(json(geo)) + } + if (url.pathname.endsWith('/admin/traffic/geoip')) { + if (method === 'DELETE') { + geo = { country: null, asn: null, maxmind: { ...geo.maxmind, account_id: null, license_key_set: false } } + return Promise.resolve(new Response(null, { status: 204 })) + } + if (method === 'PUT') { + geo = { + ...geo, + maxmind: { ...geo.maxmind, account_id: String(body.account_id), license_key_set: true }, + } + } + return Promise.resolve(json(geo)) + } + return Promise.resolve(new Response('not found', { status: 404 })) + }) +} + +/** The query strings of the requests to `path` sent so far. */ +function sent(path: string, method = 'GET'): URLSearchParams[] { + return fetchMock.mock.calls + .filter(([, init]) => ((init as RequestInit | undefined)?.method ?? 'GET') === method) + .map(([input]) => new URL(String(input), 'http://localhost')) + .filter((u) => u.pathname === path) + .map((u) => u.searchParams) +} + +/** The bodies of the requests to `path` sent with `method`. */ +function bodies(path: string, method: string): unknown[] { + return fetchMock.mock.calls + .filter(([input, init]) => { + const i = init as RequestInit | undefined + return i?.method === method && new URL(String(input), 'http://localhost').pathname === path + }) + .map(([, init]) => { + const raw = (init as RequestInit).body + return raw ? (JSON.parse(String(raw)) as unknown) : null + }) +} + +describe('admin traffic screens', () => { + beforeEach(() => { + vi.stubGlobal('fetch', fetchMock) + fetchMock.mockReset() + setSession('admin-csrf') + }) + afterEach(() => { + vi.unstubAllGlobals() + clearSession() + }) + + it('shows the totals and where the requests came from', async () => { + mockApi() + await renderApp('/manage/traffic') + + expect(await screen.findByText('1,234')).toBeInTheDocument() + expect(sent('/admin/traffic')[0].get('range')).toBe('30d') + + const repos = screen.getByRole('table', { name: 'Repositories by requests' }) + expect(within(repos).getByRole('link', { name: 'npm-proxy' })).toBeInTheDocument() + expect( + within(repos).getByRole('link', { name: 'No repository (web UI, APIs, sign-in)' }), + ).toBeInTheDocument() + const countries = screen.getByRole('table', { name: 'Countries by requests' }) + expect(within(countries).getByRole('link', { name: 'Germany' })).toBeInTheDocument() + expect(within(countries).getByRole('link', { name: 'Unknown' })).toBeInTheDocument() + const asns = screen.getByRole('table', { name: 'Autonomous systems by requests' }) + expect(within(asns).getByRole('link', { name: 'AS3320 Deutsche Telekom AG' })).toBeInTheDocument() + const clients = screen.getByRole('table', { name: 'Top clients' }) + const row = within(clients).getAllByRole('row')[1] + expect(within(row).getByRole('link', { name: '2001:db8:1:2::/64' })).toBeInTheDocument() + expect(within(row).getByText('Germany')).toBeInTheDocument() + expect(within(row).getByText('npm-proxy, No repository (web UI, APIs, sign-in)')).toBeInTheDocument() + expect(screen.getAllByText(/GeoLite Data created by MaxMind/).length).toBeGreaterThan(0) + expect(screen.getByRole('img', { name: /Requests by day: 1,234 in all/ })).toBeInTheDocument() + }) + + it('narrows the view from its lists, and removes the filters one by one', async () => { + mockApi() + const user = userEvent.setup() + const router = await renderApp('/manage/traffic') + await screen.findByText('1,234') + + await user.click(screen.getByRole('radio', { name: '7 days' })) + await user.click(screen.getByRole('link', { name: 'npm-proxy' })) + await user.click(await screen.findByRole('link', { name: 'Germany' })) + await waitFor(() => + expect( + sent('/admin/traffic').some( + (q) => q.get('range') === '7d' && q.get('repository') === 'npm-proxy' && q.get('country') === 'DE', + ), + ).toBe(true), + ) + expect(router.state.location.search).toMatchObject({ range: '7d', repository: 'npm-proxy', country: 'DE' }) + + await user.click(screen.getByRole('button', { name: 'Remove the filter Repository: npm-proxy' })) + await waitFor(() => expect(router.state.location.search).not.toHaveProperty('repository')) + expect(router.state.location.search).toMatchObject({ country: 'DE' }) + + // The requests to no repository and the unknown country are filters too. + await user.click(screen.getByRole('link', { name: 'No repository (web UI, APIs, sign-in)' })) + await waitFor(() => expect(sent('/admin/traffic').some((q) => q.get('repository') === '-')).toBe(true)) + await user.click(screen.getByRole('button', { name: 'Clear all' })) + await waitFor(() => expect(router.state.location.search).not.toHaveProperty('country')) + }) + + it('shows the days as a table on request', async () => { + mockApi() + const user = userEvent.setup() + await renderApp('/manage/traffic') + await screen.findByText('1,234') + + await user.click(screen.getByRole('radio', { name: 'Table' })) + const table = screen.getByRole('table', { name: 'Requests by day' }) + const rows = within(table).getAllByRole('row').slice(1) + expect(rows.map((r) => within(r).getAllByRole('cell')[1].textContent)).toEqual(['634', '600']) + }) + + it('says how to look countries up without a database, and when nothing is recorded', async () => { + mockApi({ + overview: { + ...overview, + totals: { ...overview.totals, countries: 0, asns: 0 }, + countries: [{ country: '', requests: 1234, bytes: 0, clients: 3 }], + asns: [{ asn: 0, org: '', requests: 1234, bytes: 0, clients: 3 }], + recording: { ...recording, enabled: false, countries: false, asns: false }, + }, + }) + await renderApp('/manage/traffic') + expect(await screen.findByText('Traffic is not recorded')).toBeInTheDocument() + expect(screen.getByText('Countries are not looked up')).toBeInTheDocument() + expect(screen.queryByRole('table', { name: 'Autonomous systems by requests' })).toBeNull() + }) + + it('saves the recording settings', async () => { + mockApi() + const user = userEvent.setup() + await renderApp('/manage/traffic/settings') + + await user.click(await screen.findByRole('switch', { name: /Client addresses/ })) + const retention = screen.getByRole('textbox', { name: 'Keep the statistics for (days)' }) + await user.clear(retention) + await user.type(retention, '0') + await user.click(screen.getByRole('button', { name: 'Save recording' })) + expect(await screen.findByText(/Enter a whole number of days from 1/)).toBeInTheDocument() + expect(bodies('/admin/settings/traffic', 'PUT')).toEqual([]) + + await user.clear(retention) + await user.type(retention, '30') + await user.click(screen.getByRole('button', { name: 'Save recording' })) + await waitFor(() => + expect(bodies('/admin/settings/traffic', 'PUT')).toEqual([ + { enabled: true, client_addresses: false, retention_days: 30 }, + ]), + ) + }) + + it('sets the MaxMind account, downloads, and turns geolocation off', async () => { + mockApi({ geoip: { country: null, asn: null, maxmind: { ...geoip().maxmind, account_id: null, license_key_set: false, last_success: null } } }) + const user = userEvent.setup() + await renderApp('/manage/traffic/settings') + + expect(await screen.findAllByText('Not loaded')).toHaveLength(2) + await user.click(screen.getByRole('button', { name: 'Save account' })) + expect(screen.getByText('Enter your MaxMind account ID, a number.')).toBeInTheDocument() + expect(screen.getByText('Enter a license key.')).toBeInTheDocument() + + await user.type(screen.getByRole('textbox', { name: 'MaxMind account ID' }), '123456') + await user.type(screen.getByLabelText('License key'), 'abc_secret') + await user.click(screen.getByRole('button', { name: 'Save account' })) + await waitFor(() => + expect(bodies('/admin/traffic/geoip', 'PUT')).toEqual([ + { account_id: '123456', license_key: 'abc_secret' }, + ]), + ) + // The key is not kept on screen once stored. + await waitFor(() => expect(screen.getByLabelText('License key')).toHaveValue('')) + + await user.click(await screen.findByRole('button', { name: 'Download now' })) + expect(await screen.findByText('The last download failed')).toBeInTheDocument() + expect(screen.getByText(/refused the account ID or the license key/)).toBeInTheDocument() + + await user.click(screen.getByRole('button', { name: 'Turn off' })) + const dialog = screen.getByRole('dialog', { name: 'Turn geolocation off?' }) + expect(bodies('/admin/traffic/geoip', 'DELETE')).toEqual([]) + await user.click(within(dialog).getByRole('button', { name: 'Turn off' })) + await waitFor(() => expect(bodies('/admin/traffic/geoip', 'DELETE')).toHaveLength(1)) + expect(await screen.findByText('Not set')).toBeInTheDocument() + }) +}) diff --git a/web/src/styles/forge-ui.css b/web/src/styles/forge-ui.css index 3f405268..0bae4df0 100644 --- a/web/src/styles/forge-ui.css +++ b/web/src/styles/forge-ui.css @@ -455,6 +455,25 @@ } .fg-dialog { max-height: calc(100dvh - 2 * var(--space-4)); overflow-y: auto; } + /* ---------- App additions: charts (Admin → Traffic) ---------- */ + /* One series in categorical slot 1, checked against both surfaces. */ + .fg-chart { --chart-series: #2a78d6; position: relative; min-width: 0; } + [data-theme='dark'] .fg-chart { --chart-series: #3987e5; } + .fg-chart svg { display: block; overflow: visible; } + .fg-chart svg:focus-visible { outline: 2px solid var(--focus); outline-offset: 2px; border-radius: var(--radius-xs); } + .fg-chart-grid { stroke: var(--line); stroke-width: 1; shape-rendering: crispEdges; } + .fg-chart-axis { fill: var(--ink-muted); font-size: 11px; font-variant-numeric: tabular-nums; } + .fg-chart-line { fill: none; stroke: var(--chart-series); stroke-width: 2; stroke-linejoin: round; stroke-linecap: round; } + .fg-chart-area { fill: var(--chart-series); opacity: 0.1; } + .fg-chart-cross { stroke: var(--line-strong); stroke-width: 1; } + .fg-chart-dot { fill: var(--chart-series); stroke: var(--surface); stroke-width: 2; } + .fg-chart-tip { position: absolute; top: 0; transform: translateX(-50%); pointer-events: none; padding: 6px 10px; border: 1px solid var(--line); border-radius: var(--radius-sm); background: var(--surface); box-shadow: var(--shadow-md); font-size: 12.5px; line-height: 18px; white-space: nowrap; } + .fg-chart-tip-value { color: var(--ink); font-weight: 600; font-variant-numeric: tabular-nums; } + .fg-chart-tip-label { color: var(--ink-muted); } + .fg-chart-key { display: inline-block; width: 12px; height: 2px; margin-right: 6px; border-radius: 1px; background: var(--chart-series); vertical-align: middle; } + .fg-filters { display: flex; flex-wrap: wrap; align-items: center; gap: var(--space-3); min-width: 0; } + .fg-filters > .fg-seg { max-width: 100%; overflow-x: auto; } + /* ---------- App additions: remembered accounts (login screen) ---------- */ .fg-accounts { display: grid; gap: var(--space-2); margin: 0; padding: 0; list-style: none; } .fg-account { display: flex; align-items: center; gap: var(--space-1); padding-right: var(--space-1); border: 1px solid var(--line); border-radius: var(--radius-md); background: var(--surface); transition: background-color 120ms ease-out, border-color 120ms ease-out; }