diff --git a/crates/engine/src/cmd/connection_browse.rs b/crates/engine/src/cmd/connection_browse.rs new file mode 100644 index 000000000..9d49dc822 --- /dev/null +++ b/crates/engine/src/cmd/connection_browse.rs @@ -0,0 +1,515 @@ +//! `connection.browse`: the datasets a saved connection offers, one level at a time, each with +//! the command that adds it. +//! +//! The app reads over the network in the background, so browsing is a probe/answer pair like +//! `connection.test`: `probe` names the address a level reads, the app fetches it and asks again +//! with the `answer`. Without an answer, the command line and the assistant read it themselves. + +use serde_json::{Value, json}; +use std::borrow::Cow; + +use super::super::{b, ps}; +use crate::connections::{Connection, DESKTOP_ONLY}; +use crate::render::rustgis_tiles::{address, s3, stac}; +use crate::{CmdResult, Session}; + +/// The most children of one level that are listed. +const MAX_CHILDREN: usize = 500; +/// The items of a STAC collection that are listed. +const ITEMS: usize = 20; +/// The longest node a caller may give. +const MAX_NODE: usize = 4096; + +/// What a node of a connection stands for. +enum Level { + Database, + Service { url: String, kind: Option }, + Catalogue, + StacRoot { api: String }, + StacCollection { api: String, id: String }, + StacCatalog { url: String }, + StacItem { url: String }, + S3Root, + S3Prefix { bucket: String, prefix: String }, +} + +/// The types a `service::` node names (`auto` leaves the type to be guessed). +const SERVICE_TYPES: [&str; 7] = ["auto", "wms", "wmts", "wfs", "wcs", "ogcapi", "arcgis"]; + +/// The level a connection's `node` names (its top level when empty). +fn level(c: &Connection, node: &str) -> Result { + let kind = c.kind.as_str(); + let not_a_node = || format!("'{node}' isn't a node of {}", c.name); + if node.is_empty() { + return Ok(match kind { + "postgres" | "mysql" | "sqlserver" => { + if cfg!(target_arch = "wasm32") { + return Err(DESKTOP_ONLY.into()); + } + Level::Database + } + "service" => Level::Service { url: c.address.clone(), kind: None }, + "wms" | "wmts" | "wfs" | "wcs" | "ogcapi" | "arcgis" => Level::Service { url: c.address.clone(), kind: Some(c.kind.clone()) }, + "csw" => Level::Catalogue, + "stac" => Level::StacRoot { api: stac::api(Some(&c.address))? }, + "s3" => Level::S3Root, + _ => return Err(format!("unknown kind '{}'", c.kind)), + }); + } + if let Some(rest) = node.strip_prefix("service:") { + if !matches!(kind, "service" | "wms" | "wmts" | "wfs" | "wcs" | "ogcapi" | "arcgis" | "csw") { + return Err(not_a_node()); + } + let (t, url) = rest.split_once(':').filter(|(t, _)| SERVICE_TYPES.contains(t)).ok_or_else(not_a_node)?; + // Judged as a document of the connection would be: a link may not lead into the local network. + address::check(url, Some(&c.address))?; + return Ok(Level::Service { url: url.to_string(), kind: (t != "auto").then(|| t.to_string()) }); + } + if let Some(id) = node.strip_prefix("collection:") { + if kind != "stac" || id.is_empty() { + return Err(not_a_node()); + } + let api = stac::api(Some(&c.address))?; + if stac::catalog::is_static(&api) { + return Err(not_a_node()); + } + return Ok(Level::StacCollection { api, id: id.to_string() }); + } + if let Some(url) = node.strip_prefix("catalog:") { + if kind != "stac" { + return Err(not_a_node()); + } + address::check(url, Some(&c.address))?; + return Ok(Level::StacCatalog { url: url.to_string() }); + } + if let Some(url) = node.strip_prefix("item:") { + // A static catalog's item: its own file, read when opened. + if kind != "stac" { + return Err(not_a_node()); + } + address::check(url, Some(&c.address))?; + return Ok(Level::StacItem { url: url.to_string() }); + } + if let Some(rest) = node.strip_prefix("s3:") { + let (bucket, prefix) = rest + .split_once('/') + .filter(|(bucket, prefix)| kind == "s3" && s3::valid_bucket(bucket) && prefix.len() <= 1024) + .ok_or_else(not_a_node)?; + return Ok(Level::S3Prefix { bucket: bucket.to_string(), prefix: prefix.to_string() }); + } + Err(not_a_node()) +} + +/// The session's S3 connection for an S3 level, logging in with what needs no typing. +/// (A probe logs in too, as `connection.test` does: the address it names, and the signature the +/// fetch needs, depend on the connection.) +fn s3_login(s: &mut Session, c: &Connection, level: &Level) -> Result, String> { + if !matches!(level, Level::S3Root | Level::S3Prefix { .. }) { + return Ok(None); + } + if s3::connection(&c.name).is_none() && c.options.get("credentials").is_none_or(|v| v != "keys") { + super::login(s, c, &Value::Null)?; + } + let conn = s3::connection(&c.name).ok_or("log in first: connection.login {id, accessKeyId, secretAccessKey}")?; + if let Level::S3Prefix { bucket, .. } = level { + s3::grant(&conn.name, bucket); + } + Ok(Some(conn)) +} + +/// The address a level reads (`None`: nothing is fetched, or the database driver reads it). +fn fetch_url(s: &Session, c: &Connection, level: &Level, conn: Option<&s3::Connection>) -> Result, String> { + Ok(match level { + Level::Database => None, + Level::Service { url, kind } => Some(rustgis_data::ogc::capabilities_url(url, kind.as_deref())), + Level::Catalogue => Some(super::super::service::records_request(s, &json!({"url": c.address}))?), + Level::StacRoot { api } => Some(if stac::catalog::is_static(api) { api.clone() } else { stac::collections_url(api) }), + Level::StacCollection { api, id } => { + Some(stac::search_url(api, &stac::Search { collections: vec![id.clone()], limit: ITEMS, ..stac::Search::default() })?) + } + Level::StacCatalog { url } | Level::StacItem { url } => Some(url.clone()), + Level::S3Root => match (super::first_bucket(c), conn) { + (Some(_), _) => None, + (None, Some(conn)) if conn.credentials.is_some() => Some(s3::buckets_url(conn)), + _ => return Err("name a bucket under Buckets to browse an anonymous S3 connection".into()), + }, + Level::S3Prefix { bucket, prefix } => Some(s3::list_url(bucket, prefix, None, conn)), + }) +} + +/// What a level reads, fetched here (waiting for it). +#[cfg(not(target_arch = "wasm32"))] +fn download(url: &str) -> Result { + let bytes = crate::render::rustgis_tiles::cache::download_blocking(url)?; + Ok(String::from_utf8(bytes).unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned())) +} + +#[cfg(target_arch = "wasm32")] +fn download(_: &str) -> Result { + Err("in the web app, expand the connection under Data Sources › Connections".into()) +} + +/// The children of a level, and what is known of the rest. +#[derive(Default)] +struct Found { + children: Vec, + truncated: bool, + matched: Option, +} + +impl Found { + /// Keeps the first [`MAX_CHILDREN`] of a list (collected with one more, to know it was cut). + fn new(mut children: Vec) -> Found { + let truncated = children.len() > MAX_CHILDREN; + children.truncate(MAX_CHILDREN); + Found { children, truncated, matched: None } + } +} + +/// A node that fits what [`browse`] accepts. +fn node_ok(node: &str) -> bool { + node.len() <= MAX_NODE && !node.chars().any(char::is_control) +} + +pub(super) fn browse(s: &mut Session, p: &Value) -> CmdResult { + let c = super::found(s, p)?.clone(); + let node = ps(p, "node").unwrap_or(""); + if !node_ok(node) { + return Err("node is a child's node, as connection.browse lists it".into()); + } + // The app's answers come from a fetch the cache already limits; a caller's may be any size. + let documents = p.get("documents").and_then(Value::as_array).into_iter().flatten().filter_map(|d| ps(d, "text")).map(str::len); + if ps(p, "answer").map(str::len).into_iter().chain(documents).sum::() > super::super::service::MAX_GROUP_BYTES { + return Err("answer and documents are too large to read".into()); + } + let level = level(&c, node)?; + let conn = s3_login(s, &c, &level)?; + let url = fetch_url(s, &c, &level, conn.as_ref())?; + let mut out = json!({"connection": c.id, "node": node}); + if b(p, "probe").unwrap_or(false) { + out["fetch"] = json!(url); + return Ok(out); + } + let found = match &level { + Level::Database => database(&c)?, + Level::Service { url, kind } => service(s, p, &mut out, url, kind.as_deref())?, + Level::S3Root if url.is_none() => { + buckets(c.options.get("buckets").map(String::as_str).unwrap_or("").split(',').map(str::to_string)) + } + _ => { + let at = url.as_deref().unwrap_or(""); + let text: Cow = match ps(p, "answer") { + Some(a) => Cow::Borrowed(a), + None => Cow::Owned(download(at)?), + }; + let bytes = text.as_bytes(); + match &level { + Level::Catalogue => catalogue(s, &c, &text, &mut out)?, + Level::StacRoot { api } if stac::catalog::is_static(api) => catalogs(bytes, api)?, + Level::StacRoot { .. } => collections(bytes)?, + Level::StacCatalog { url } => catalogs(bytes, url)?, + Level::StacItem { url } => item_file(bytes, url)?, + Level::StacCollection { .. } => items(bytes, at)?, + Level::S3Root => buckets(s3::parse_buckets(bytes)?.into_iter()), + Level::S3Prefix { bucket, prefix } => objects(bytes, bucket, prefix)?, + Level::Database | Level::Service { .. } => Found::default(), + } + } + }; + out["children"] = json!(found.children); + if found.truncated { + out["truncated"] = json!(true); + } + if let Some(m) = found.matched { + out["matched"] = json!(m); + } + Ok(out) +} + +/// A live database's geometry tables, under their schemas. +fn database(c: &Connection) -> Result { + #[cfg(target_arch = "wasm32")] + { + let _ = c; + Err(DESKTOP_ONLY.into()) + } + #[cfg(not(target_arch = "wasm32"))] + { + use std::collections::BTreeMap; + + let tables = super::super::layer::database_table_details(&json!({"database": c.address}))?; + let text = |t: &Value, k: &str| t.get(k).and_then(Value::as_str).unwrap_or("").to_string(); + let mut schemas: BTreeMap> = BTreeMap::new(); + let truncated = tables.len() > MAX_CHILDREN; + for t in tables.iter().take(MAX_CHILDREN) { + let (label, name) = (text(t, "label"), text(t, "name")); + let shown = if label.contains(':') { format!("{name}:{}", text(t, "column")) } else { name }; + schemas.entry(text(t, "schema")).or_default().push(json!({ + "key": label, + "name": shown, + "what": "table", + "detail": text(t, "kind"), + "add": {"command": "layer.addDatabase", "params": {"database": c.address, "layer": label}}, + "options": true, + })); + } + let children = schemas + .into_iter() + .map(|(s, tables)| json!({"key": format!("schema:{s}"), "name": s, "what": "schema", "children": tables})) + .collect(); + Ok(Found { children, truncated, matched: None }) + } +} + +/// A service's layers (capabilities read here, or the app's `answer` and `documents`). +fn service(s: &mut Session, p: &Value, out: &mut Value, url: &str, kind: Option<&str>) -> Result { + let mut q = json!({"url": url}); + if let Some(k) = kind { + q["type"] = json!(k); + } + if let Some(a) = ps(p, "answer") { + q["text"] = json!(a); + if let Some(d) = p.get("documents") { + q["documents"] = d.clone(); + } + } + let r = super::super::service::layers(s, &q)?; + out["service"] = json!({"url": url, "type": kind.unwrap_or("auto")}); + if let Some(n) = r.get("notes").filter(|n| n.as_array().is_some_and(|a| !a.is_empty())) { + out["notes"] = n.clone(); + } + if let Some(next) = r.get("next").filter(|n| !n.is_null()) { + out["next"] = next.clone(); + } + // The dialog imports an OGC API's collections together: a features layer isn't added alone there. + let grouped = r.get("kind").and_then(Value::as_str).is_some_and(|k| k.starts_with("OGC API")); + let text = |l: &Value, k: &str| l.get(k).and_then(Value::as_str).unwrap_or("").to_string(); + let children = r + .get("layers") + .and_then(Value::as_array) + .into_iter() + .flatten() + .take(MAX_CHILDREN + 1) + .map(|l| { + let (name, title) = (text(l, "name"), text(l, "title")); + let offer = l.get("offer").cloned().unwrap_or(Value::Null); + let offer_type = offer.get("type").and_then(Value::as_str).unwrap_or(""); + if offer_type == "service" { + let node = format!("service:arcgis:{}", text(&offer, "url")); + if !node_ok(&node) { + return Value::Null; + } + let what = if name.ends_with('/') { "folder" } else { "service" }; + return json!({"key": node, "node": node, "name": title.trim_end_matches('/'), "what": what}); + } + let what = match offer_type { + "vectorTiles" => "vectorTiles", + "features" => "features", + "coverage" => "coverage", + "style" => "style", + _ => "map", + }; + let source = text(l, "source_url"); + let mut params = json!({"url": url, "layer": name, "source_url": source}); + if let Some(k) = kind { + params["type"] = json!(k); + } + let mut child = json!({ + "key": format!("{source}#{name}"), + "name": if title.is_empty() { name.as_str() } else { title.as_str() }, + "what": what, + "detail": name, + "layer": l, + "options": true, + }); + if !(grouped && what == "features") { + child["add"] = json!({"command": "service.add", "params": params}); + } + child + }) + .collect::>(); + // A folder or service whose address is too long to follow as a node is left out, and said so. + let listed = children.len(); + let children: Vec = children.into_iter().filter(|c| !c.is_null()).collect(); + let left_out = children.len() < listed; + let mut found = Found::new(children); + found.truncated |= left_out; + Ok(found) +} + +/// A catalogue's records, each with the services and files it links to. +fn catalogue(s: &mut Session, c: &Connection, text: &str, out: &mut Value) -> Result { + let r = super::super::service::catalog_search(s, &json!({"url": c.address, "text": text}))?; + if let Some(n) = r.get("notes") { + out["notes"] = n.clone(); + } + let str_of = |v: &Value, k: &str| v.get(k).and_then(Value::as_str).unwrap_or("").to_string(); + let mut dropped = false; + let children = r + .get("records") + .and_then(Value::as_array) + .into_iter() + .flatten() + .take(MAX_CHILDREN + 1) + .enumerate() + .map(|(i, rec)| { + let id = str_of(rec, "id"); + let title = str_of(rec, "title"); + let links: Vec = rec + .get("links") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|l| { + let (url, kind, name) = (str_of(l, "url"), str_of(l, "kind"), str_of(l, "name")); + match kind.as_str() { + "wms" | "wmts" | "wfs" | "wcs" | "ogcapi" | "arcgis" => { + let node = format!("service:{kind}:{url}"); + if !node_ok(&node) { + dropped = true; + return None; + } + Some(json!({"key": node, "node": node, "name": if name.is_empty() { url.as_str() } else { name.as_str() }, "what": "service", "detail": kind})) + } + "download" => { + let last = url.split(['?', '#']).next().unwrap_or("").trim_end_matches('/').rsplit('/').next().unwrap_or(""); + let shown: String = if name.is_empty() { last } else { name.as_str() }.chars().take(200).collect(); + let named: String = title.chars().take(200).collect(); + Some(json!({ + "key": url, + "name": shown, + "what": "download", + "add": {"command": "layer.addUrl", "params": {"url": url, "name": named}}, + })) + } + _ => None, + } + }) + .collect(); + let summary: String = str_of(rec, "abstract").chars().take(200).collect(); + json!({"key": format!("record:{}", if id.is_empty() { i.to_string() } else { id }), "name": title, "what": "record", "detail": summary, "children": links}) + }) + .collect(); + let mut found = Found::new(children); + found.matched = r.get("matched").and_then(Value::as_u64); + found.truncated |= dropped || r.get("next").is_some_and(|n| !n.is_null()); + Ok(found) +} + +/// A STAC API's collections. +fn collections(bytes: &[u8]) -> Result { + let list = stac::parse_collections(bytes)?; + Ok(Found::new( + list.into_iter() + .take(MAX_CHILDREN + 1) + .map(|c| { + let node = format!("collection:{}", c.id); + json!({"key": node, "node": node, "name": if c.title.is_empty() { c.id.as_str() } else { c.title.as_str() }, "what": "collection", "detail": c.id, "options": true}) + }) + .collect(), + )) +} + +/// What a static catalog's document lists below it: its catalogs and collections (`child` links) +/// and its items (`item` links), each a node read when it is opened (an item's file holds its +/// assets). +fn catalogs(bytes: &[u8], from: &str) -> Result { + let mut dropped = false; + let mut children = Vec::new(); + for (prefix, what, list) in [ + ("catalog", "catalog", stac::catalog::parse_children(bytes, from)?), + ("item", "item", stac::catalog::parse_item_links(bytes, from)?), + ] { + for c in list { + let node = format!("{prefix}:{}", c.url); + if node_ok(&node) { + children.push(json!({"key": node, "node": node, "name": c.title, "what": what})); + } else { + dropped = true; + } + } + } + let mut found = Found::new(children); + found.truncated |= dropped; + Ok(found) +} + +/// The assets of an item that can be added, each with the `stac.addAsset` that adds it. +fn assets_of(item: &stac::Item) -> Vec { + item.assets + .iter() + .take(MAX_CHILDREN) + .filter_map(|a| { + let (key, href) = item.asset_href(Some(&a.key)).ok()?; + let mut params = json!({"asset": key, "name": format!("{} {key}", item.id)}); + match &item.url { + Some(u) => params["url"] = json!(u), + None => params["item"] = stac::item_json_with(item, false), + } + Some(json!({ + "key": format!("{}#{key}", item.url.as_deref().unwrap_or(&item.id)), + "name": if a.title.is_empty() { key.as_str() } else { a.title.as_str() }, + "what": "asset", + "detail": a.media_type, + "href": href, + "add": {"command": "stac.addAsset", "params": params}, + })) + }) + .collect() +} + +/// A static catalog's item (its own file): its assets. +fn item_file(bytes: &[u8], from: &str) -> Result { + Ok(Found::new(assets_of(&stac::parse_item(bytes, from)?))) +} + +/// The items of a collection's search, each with the assets that can be added. +fn items(bytes: &[u8], from: &str) -> Result { + let (list, next) = stac::parse_items(bytes, from)?; + let matched = stac::parse_matched(bytes); + let truncated = next.is_some() || matched.is_some_and(|m| m > list.len() as u64); + let children = list + .iter() + .take(MAX_CHILDREN) + .map(|item| json!({"key": format!("item:{}", item.url.as_deref().unwrap_or(&item.id)), "name": item.id, "what": "item", "detail": item.datetime, "children": assets_of(item)})) + .collect(); + Ok(Found { children, truncated, matched }) +} + +/// Buckets to browse, each a node. +fn buckets(names: impl Iterator) -> Found { + Found::new( + names + .map(|n| n.trim().to_string()) + .filter(|n| s3::valid_bucket(n)) + .take(MAX_CHILDREN + 1) + .map(|n| { + let node = format!("s3:{n}/"); + json!({"key": node, "node": node, "name": n, "what": "bucket"}) + }) + .collect(), + ) +} + +/// One level of a bucket: its folders (each a node) and its files. +fn objects(bytes: &[u8], bucket: &str, prefix: &str) -> Result { + let list = s3::parse_list(bytes)?; + let last = |k: &str| k.trim_end_matches('/').rsplit('/').next().unwrap_or("").to_string(); + let folders = list.prefixes.iter().map(|p| { + let node = format!("s3:{bucket}/{p}"); + json!({"key": node, "node": node, "name": format!("{}/", last(p)), "what": "prefix"}) + }); + let files = list.objects.iter().filter(|o| o.key != prefix && !o.key.ends_with('/')).map(|o| { + let url = format!("s3://{bucket}/{}", o.key); + json!({ + "key": url, + "name": last(&o.key), + "what": "object", + "detail": o.size.to_string(), + "add": {"command": "layer.addUrl", "params": {"url": url}}, + }) + }); + let mut found = Found::new(folders.chain(files).take(MAX_CHILDREN + 1).collect()); + found.truncated |= list.next.is_some(); + Ok(found) +} diff --git a/crates/engine/src/cmd/connections.rs b/crates/engine/src/cmd/connections.rs index 833ffc6d4..b4f2e3c5a 100644 --- a/crates/engine/src/cmd/connections.rs +++ b/crates/engine/src/cmd/connections.rs @@ -10,6 +10,9 @@ use crate::connections::{Connection, DATABASE_KINDS, DESKTOP_ONLY, KINDS}; use crate::render::rustgis_tiles::{s3, stac}; use crate::{CmdResult, CommandSpec as C, Session}; +#[path = "connection_browse.rs"] +mod browse; + pub fn commands() -> Vec { vec![ C::new("connection.list", "List Connections", "Home › Add › Data Sources", list).view().params( @@ -29,6 +32,9 @@ pub fn commands() -> Vec { C::new("connection.test", "Test Connection", "Home › Add › Data Sources", test).view().params( r#"{"id"?: "a connection's id or name", "kind"?: "postgres"|"mysql"|"sqlserver"|"service"|"wms"|"wmts"|"wfs"|"wcs"|"ogcapi"|"arcgis"|"csw"|"stac"|"s3", "name"?: text, "group"?: text, "address"?: text, "options"?: "an unsaved connection, as connection.new (when no id)", "password"?: text, "accessKeyId"?: text, "secretAccessKey"?: text, "sessionToken"?: "as connection.login (logs in first)", "probe"?: "true: only the address the test reads, {fetch} (null for databases, which their driver tests)", "answer"?: "non-database kinds only: what that address returned, when the app fetched it (no network request then); databases ignore answer and test through their driver"}"#, ), + C::new("connection.browse", "Browse Connection", "Home › Add › Data Sources", browse::browse).view().params( + r#"{"id": "a connection's id or name", "node"?: "a child's node, as a listing gives it (the connection's top level when absent)", "probe"?: "true: only the address this level reads, {fetch} (null: read without a fetch, or by the database driver); an S3 connection logs in first, as connection.test does", "answer"?: "what that address returned, when the app fetched it", "documents"?: [{"url": "a further document a listing asked for (next)", "text": "its answer"}]} → {connection, node, children: [{key, name, what, detail?, node? (browse it), children? (listed already), add? {command, params}: the command that adds it}], truncated?, matched?, notes?, service? {url, type}, next? (fetch it, then ask again with answer and documents)}"#, + ), ] } @@ -53,16 +59,7 @@ fn shown(c: &Connection) -> Value { } else { false }; - let add_with = match c.kind.as_str() { - "postgres" | "mysql" | "sqlserver" => "info.databaseTables {database: address}, then layer.addDatabase {database: address, layer}", - "service" => "service.layers {url: address}, then service.add {url, layer}", - "wms" | "wmts" | "wfs" | "ogcapi" | "arcgis" => "service.layers {url: address, type: kind}, then service.add {url, type, layer}", - "wcs" => "service.layers {url: address, type: wcs}, then service.coverage {url, layer}", - "csw" => "catalog.search {url: address, query}", - "stac" => "stac.collections {url: address}, then stac.search {url, collections}", - "s3" => "connection.login {id} once, then s3.list {connection: name}, then layer.addUrl {url: s3://…}", - _ => "", - }; + let add_with = "connection.browse {id}, then {id, node} for a child's node; run a dataset's add {command, params}"; let mut out = serde_json::to_value(c).unwrap_or(Value::Null); out["available"] = json!(available); if !available { diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index d8b671a8a..9d7246ab2 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -1280,6 +1280,15 @@ pub(super) fn database_tables(_s: &mut Session, p: &Value) -> CmdResult { } } +/// The geometry tables of a live database server, with the schema, name, column and type of +/// each (`connection.browse` groups them by schema). +#[cfg(not(target_arch = "wasm32"))] +pub(super) fn database_table_details(p: &Value) -> Result, String> { + let db = database_param(p).ok_or("database is required")?; + let live = LiveDb::connect(&db).ok_or("not a database server")?.map_err(|e| format!("{}: {e}", db.short()))?; + Ok(live.tables().map_err(|e| format!("{}: {e}", db.short()))?.iter().map(LiveTable::detail).collect()) +} + /// Adds the tables of a database (a PostgreSQL connection or a DuckDB file) that have a geometry /// column, or the one `layer` names. fn add_database(s: &mut Session, p: &Value) -> CmdResult { @@ -1399,6 +1408,16 @@ impl LiveTable { LiveTable::Sqlserver(t) => t.label(), } } + + /// The table as a browser lists it: its label for `layer.addDatabase`, and what it is made of. + fn detail(&self) -> Value { + let (schema, name, column, kind) = match self { + LiveTable::Postgis(t) => (&t.schema, &t.name, &t.column, &t.kind), + LiveTable::Mysql(t) => (&t.schema, &t.name, &t.column, &t.kind), + LiveTable::Sqlserver(t) => (&t.schema, &t.name, &t.column, &t.kind), + }; + json!({"label": self.label(), "schema": schema, "name": name, "column": column, "kind": kind}) + } } #[cfg(not(target_arch = "wasm32"))] diff --git a/crates/engine/src/cmd/plugins.rs b/crates/engine/src/cmd/plugins.rs index f71652219..bc00ee10b 100644 --- a/crates/engine/src/cmd/plugins.rs +++ b/crates/engine/src/cmd/plugins.rs @@ -276,6 +276,7 @@ const NEVER: &[&str] = &[ "connection.remove", "connection.login", "connection.test", + "connection.browse", "s3.connect", "s3.disconnect", "earthdata.login", diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 11bd4e55a..9bfe9a1e1 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -1148,6 +1148,7 @@ fn hostile_params_never_panic() { json!({"id": "", "kind": "postgres", "address": "postgres:host=127.0.0.1 port=9 password=x", "options": "x"}), json!({"kind": "wms", "address": "http://127.0.0.1:9/wms", "probe": "x", "answer": 7}), json!({"kind": "stac", "address": "http://127.0.0.1:9/stac", "answer": "\u{0}{"}), + json!({"id": "c1", "node": "s3:/\u{7}", "probe": true, "answer": 7, "documents": "x"}), ]; let ids: Vec<&str> = Session::new(Project::default()).registry.commands.iter().map(|c| c.id).collect(); let streamed = grid_fgb("hostile", 20); @@ -6800,6 +6801,18 @@ fn wcs_coverages_and_catalogue_records() { ); let full = s.run("service.coverage", &json!({"url": url, "layer": "soc", "text": caps, "describe": describe, "bbox": "full"})).unwrap(); assert_eq!(full["bbox"], json!([5.0, 49.2, 5.8, 50.0])); + // The same coverage browsed from a saved connection: the add it lists runs as it is. + s.run("connection.new", &json!({"kind": "wcs", "address": url, "name": "Soils"})).unwrap(); + let browsed = s.run("connection.browse", &json!({"id": "Soils", "answer": caps})).unwrap(); + let child = &browsed["children"][0]; + assert_eq!((child["what"].as_str(), child["add"]["command"].as_str()), (Some("coverage"), Some("service.add")), "{browsed}"); + let mut listed = child["add"]["params"].clone(); + listed["text"] = json!(caps); + listed["describe"] = json!(describe); + listed["bbox"] = json!("full"); + let layers = s.map().layers.len(); + s.run("service.add", &listed).unwrap(); + assert_eq!(s.map().layers.len(), layers + 1); let (err, _) = serve_file(b"no".to_vec(), false); let p = json!({"url": format!("{err}/wcs"), "layer": "soc", "text": caps, "describe": describe, "bbox": "full"}); assert!(s.run("service.add", &p).unwrap_err().contains("error")); @@ -8856,7 +8869,15 @@ fn connections_commands() { assert_eq!(renamed["name"], "wms (2)"); assert_ne!(default["id"], renamed["id"]); assert!(crate::sources::holds_secret(&json!({"id":"x","password":"p"}))); - for id in ["connection.new", "connection.edit", "connection.duplicate", "connection.remove", "connection.login", "connection.test"] { + for id in [ + "connection.new", + "connection.edit", + "connection.duplicate", + "connection.remove", + "connection.login", + "connection.test", + "connection.browse", + ] { assert!(crate::cmd::plugins::user_only(id), "{id}"); } assert!(!crate::cmd::plugins::user_only("connection.list")); @@ -8967,6 +8988,195 @@ fn connection_test_probes_and_answers() { assert!(s.connections.connections.is_empty()); } +/// Browsing a saved connection lists what each kind offers, a level at a time, with the command +/// that adds each dataset; offline, with the answers supplied. A node that isn't the +/// connection's is refused, and nothing browsed is saved or undoable. +#[test] +fn connection_browse_levels_and_adds() { + use crate::render::rustgis_tiles::s3; + let mut s = Session::new(Project::default()); + let new = |s: &mut Session, p: Value| s.run("connection.new", &p).unwrap(); + let wms_caps = r#"Localimage/pngEPSG:3857aAbB"#; + + // WMS: the address read, its layers, and the add that adds one. + new(&mut s, json!({"kind": "wms", "address": "https://a.example/wms", "name": "W"})); + let v = s.run("connection.browse", &json!({"id": "W", "probe": true})).unwrap(); + assert!(v["fetch"].as_str().is_some_and(|f| f.contains("REQUEST=GetCapabilities")), "{v}"); + let v = s.run("connection.browse", &json!({"id": "W", "answer": wms_caps})).unwrap(); + assert_eq!(v["service"], json!({"url": "https://a.example/wms", "type": "wms"})); + let first = v["children"][0].clone(); + assert_eq!(first["what"], "map"); + assert_eq!(first["name"], "A"); + assert_eq!(first["add"]["command"], "service.add"); + assert_eq!(first["add"]["params"]["url"], "https://a.example/wms"); + assert_eq!(first["add"]["params"]["type"], "wms"); + assert_eq!(first["add"]["params"]["layer"], "a"); + assert!(first["add"]["params"].get("source_url").is_some()); + assert_eq!(v["children"].as_array().unwrap().len(), 2); + let before = s.map().tiles.len(); + let mut add = first["add"]["params"].clone(); + add["text"] = json!(wms_caps); + s.run("service.add", &add).unwrap(); + assert_eq!(s.map().tiles.len(), before + 1); + s.run("edit.undo", &Value::Null).unwrap(); + assert_eq!(s.map().tiles.len(), before); + + // ArcGIS: a directory's folder is a node of its own address. + new(&mut s, json!({"kind": "arcgis", "address": "https://x.example.org/arcgis/rest/services", "name": "Arc"})); + let dir = r#"{"currentVersion": 11, "folders": ["Hydro"], "services": [{"name": "Topo", "type": "MapServer"}]}"#; + let v = s.run("connection.browse", &json!({"id": "Arc", "answer": dir})).unwrap(); + let folder = "service:arcgis:https://x.example.org/arcgis/rest/services/Hydro"; + let child = v["children"].as_array().unwrap().iter().find(|c| c["node"] == folder).unwrap_or_else(|| panic!("{v}")); + assert_eq!(child["what"], "folder"); + assert_eq!(child["name"], "Hydro"); + let v = s.run("connection.browse", &json!({"id": "Arc", "node": folder, "probe": true})).unwrap(); + assert!(v["fetch"].as_str().is_some_and(|f| f.starts_with("https://x.example.org/arcgis/rest/services/Hydro")), "{v}"); + + // CSW: a record, the service it links to as a node, and more records than listed. + new(&mut s, json!({"kind": "csw", "address": "https://cat.example.org/csw", "name": "Cat"})); + let records = r#" + Forests + https://geo.example.org/wmshttp://192.168.1.2/wfs + "#; + let v = s.run("connection.browse", &json!({"id": "Cat", "answer": records})).unwrap(); + assert_eq!(v["children"].as_array().unwrap().len(), 1, "{v}"); + assert_eq!(v["children"][0]["what"], "record"); + assert_eq!(v["children"][0]["children"][0]["node"], "service:wms:https://geo.example.org/wms"); + assert_eq!((v["matched"].as_u64(), v["truncated"].as_bool()), (Some(3), Some(true))); + assert!(v.get("next").is_none()); + + // STAC: collections, a collection's search, and an item's asset with the add of it. + new(&mut s, json!({"kind": "stac", "address": "https://s.example/v1", "name": "Stac"})); + let v = s.run("connection.browse", &json!({"id": "Stac", "probe": true})).unwrap(); + assert_eq!(v["fetch"], "https://s.example/v1/collections"); + let v = s.run("connection.browse", &json!({"id": "Stac", "answer": r#"{"collections":[{"id":"c1","title":"C"}]}"#})).unwrap(); + assert_eq!(v["children"][0]["node"], "collection:c1"); + assert_eq!(v["children"][0]["options"], true); + let v = s.run("connection.browse", &json!({"id": "Stac", "node": "collection:c1", "probe": true})).unwrap(); + let fetch = v["fetch"].as_str().unwrap(); + assert!(fetch.contains("/search?") && fetch.contains("collections=c1") && fetch.contains("limit=20"), "{fetch}"); + let self_link = "https://s.example/v1/collections/c1/items/i1"; + let items = json!({"type": "FeatureCollection", "numberMatched": 1, "features": [{ + "type": "Feature", "id": "i1", "collection": "c1", "bbox": [0, 0, 1, 1], "geometry": null, + "properties": {"datetime": "2024-01-01T00:00:00Z"}, + "assets": {"visual": {"href": "https://s.example/a.tif", "type": "image/tiff; application=geotiff"}}, + "links": [{"rel": "self", "href": self_link}]}]}); + let v = s.run("connection.browse", &json!({"id": "Stac", "node": "collection:c1", "answer": items.to_string()})).unwrap(); + assert_eq!(v["children"][0]["what"], "item"); + assert_eq!( + v["children"][0]["children"][0]["add"], + json!({"command": "stac.addAsset", "params": {"url": self_link, "asset": "visual", "name": "i1 visual"}}) + ); + assert!(v.get("truncated").is_none()); + + // A static STAC catalog: catalogs and collections, then a collection's items, each opened + // for its assets. + new(&mut s, json!({"kind": "stac", "address": "https://s.example/static/catalog.json", "name": "Static"})); + let root = json!({"type": "Catalog", "id": "r", "links": [{"rel": "child", "href": "./c1/collection.json", "title": "C1"}]}); + let v = s.run("connection.browse", &json!({"id": "Static", "answer": root.to_string()})).unwrap(); + assert_eq!(v["children"][0]["node"], "catalog:https://s.example/static/c1/collection.json"); + let collection = json!({"type": "Collection", "id": "c1", "links": [ + {"rel": "item", "href": "./i1/i1.json", "title": "Scene 1"}, {"rel": "child", "href": "./sub/catalog.json", "title": "Sub"}]}); + let node = "catalog:https://s.example/static/c1/collection.json"; + let v = s.run("connection.browse", &json!({"id": "Static", "node": node, "answer": collection.to_string()})).unwrap(); + let kinds: Vec<_> = + v["children"].as_array().unwrap().iter().map(|c| (c["what"].as_str().unwrap(), c["node"].as_str().unwrap())).collect(); + assert_eq!( + kinds, + [("catalog", "catalog:https://s.example/static/c1/sub/catalog.json"), ("item", "item:https://s.example/static/c1/i1/i1.json")] + ); + let item_node = "item:https://s.example/static/c1/i1/i1.json"; + let v = s.run("connection.browse", &json!({"id": "Static", "node": item_node, "probe": true})).unwrap(); + assert_eq!(v["fetch"], "https://s.example/static/c1/i1/i1.json"); + let item = json!({"type": "Feature", "id": "i1", "collection": "c1", "bbox": [0, 0, 1, 1], "geometry": null, + "properties": {"datetime": "2024-01-01T00:00:00Z"}, + "assets": {"visual": {"href": "https://s.example/static/a.tif", "type": "image/tiff; application=geotiff"}}, + "links": [{"rel": "self", "href": "https://s.example/static/c1/i1/i1.json"}]}); + let v = s.run("connection.browse", &json!({"id": "Static", "node": item_node, "answer": item.to_string()})).unwrap(); + assert_eq!( + v["children"][0]["add"], + json!({"command": "stac.addAsset", "params": {"url": "https://s.example/static/c1/i1/i1.json", "asset": "visual", "name": "i1 visual"}}) + ); + // An asset's key names its item, so the same file under two items is two rows. + assert_eq!(v["children"][0]["key"], "https://s.example/static/c1/i1/i1.json#visual"); + // A relative asset address is resolved against the item's own file. + let relative = item.to_string().replace(r#""assets":{"#, r#""assets":{"data":{"href":"./data/b.tif","type":"image/tiff"},"#); + let v = s.run("connection.browse", &json!({"id": "Static", "node": item_node, "answer": relative})).unwrap(); + let hrefs: Vec<&str> = v["children"].as_array().unwrap().iter().filter_map(|c| c["href"].as_str()).collect(); + assert!(hrefs.contains(&"https://s.example/static/c1/i1/data/b.tif"), "{hrefs:?}"); + // An item node belongs to a STAC connection, and is judged as its documents are. + assert!(s.run("connection.browse", &json!({"id": "W", "node": item_node})).is_err()); + assert!(s.run("connection.browse", &json!({"id": "Static", "node": "item:http://127.0.0.1:9/i.json"})).is_err()); + // S3: buckets from the connection's options need no fetch; a bucket's level is read. + new(&mut s, json!({"kind": "s3", "name": "BrowseB", "options": {"credentials": "none", "buckets": "bkt1"}})); + let v = s.run("connection.browse", &json!({"id": "BrowseB", "probe": true})).unwrap(); + assert!(v["fetch"].is_null(), "{v}"); + let v = s.run("connection.browse", &json!({"id": "BrowseB"})).unwrap(); + assert_eq!(v["children"][0]["node"], "s3:bkt1/"); + let v = s.run("connection.browse", &json!({"id": "BrowseB", "node": "s3:bkt1/", "probe": true})).unwrap(); + assert!(v["fetch"].as_str().is_some_and(|f| f.contains("bkt1")), "{v}"); + let listing = "p/p.tif5q/0"; + let v = s.run("connection.browse", &json!({"id": "BrowseB", "node": "s3:bkt1/", "answer": listing})).unwrap(); + assert_eq!(v["children"][0]["node"], "s3:bkt1/p/"); + assert_eq!(v["children"][0]["name"], "p/"); + assert_eq!(v["children"][1]["add"]["params"]["url"], "s3://bkt1/p.tif"); + assert_eq!(v["children"].as_array().unwrap().len(), 2, "{v}"); + s3::disconnect("BrowseB"); + + // Nodes that aren't the connection's, and ones that are hostile. + let long = "x".repeat(5000); + for (id, node) in [ + ("W", "collection:x"), + ("BrowseB", "s3:BAD_/"), + ("W", long.as_str()), + ("W", "service:ftp:x"), + ("W", "service:wms:http://127.0.0.1:9/wms"), + ("Stac", "catalog:http://127.0.0.1:9/c.json"), + ("Stac", "s3:bkt1/"), + ("W", "nope"), + ("W", "a\u{7}b"), + ] { + let e = s.run("connection.browse", &json!({"id": id, "node": node})).unwrap_err(); + assert!(!e.is_empty()); + } + assert!(s.run("connection.browse", &json!({"id": "missing"})).unwrap_err().contains("no connection")); + #[cfg(not(target_arch = "wasm32"))] + { + new(&mut s, json!({"kind": "postgres", "address": "postgres:host=127.0.0.1 port=9 dbname=x user=u", "name": "Pg"})); + assert!(!s.run("connection.browse", &json!({"id": "Pg"})).unwrap_err().is_empty()); + } + // A catalogue's download link too long to be one (the parser keeps addresses to 2000 bytes) is not listed. + let long_link = format!("https://files.example.org/{}.tif", "a".repeat(5000)); + let records = format!( + r#"Files + https://files.example.org/a.tif{long_link}"# + ); + let v = s.run("connection.browse", &json!({"id": "Cat", "answer": records})).unwrap(); + let links = v["children"][0]["children"].as_array().unwrap(); + assert_eq!(links.len(), 1, "{v}"); + assert_eq!(links[0]["add"]["params"]["url"], "https://files.example.org/a.tif"); + assert!(v.get("truncated").is_none(), "{v}"); + // An answer larger than a service document may be is refused before it is parsed. + let huge = "x".repeat(65 << 20); + assert!(s.run("connection.browse", &json!({"id": "W", "answer": huge})).unwrap_err().contains("too large")); + assert!( + s.run("connection.browse", &json!({"id": "W", "answer": "", "documents": [{"url": "https://a.example/x", "text": huge}]})) + .unwrap_err() + .contains("too large") + ); + // An OGC API's feature collections are imported together: listed, with options, not added alone. + new(&mut s, json!({"kind": "ogcapi", "address": "https://o.example/collections", "name": "Ogc"})); + let collections = r#"{"collections":[{"id":"lakes","title":"Lakes","links":[{"rel":"items","type":"application/geo+json","href":"https://o.example/collections/lakes/items"}]}]}"#; + let v = s.run("connection.browse", &json!({"id": "Ogc", "answer": collections})).unwrap(); + assert_eq!((v["children"][0]["what"].as_str(), v["children"][0]["options"].as_bool()), (Some("features"), Some(true)), "{v}"); + assert!(v["children"][0].get("add").is_none(), "{v}"); + let saved = s.connections.connections.len(); + assert!(saved >= 5); + s.run("connection.browse", &json!({"id": "W", "answer": wms_caps})).unwrap(); + assert_eq!(s.connections.connections.len(), saved); + assert!(!s.can_undo()); +} + fn rustgis_engine_sources_roundtrip(text: &str) -> crate::sources::Sources { crate::sources::Sources::from_json(text) } diff --git a/crates/tiles/src/stac/catalog.rs b/crates/tiles/src/stac/catalog.rs index 1bd505334..c7a11a441 100644 --- a/crates/tiles/src/stac/catalog.rs +++ b/crates/tiles/src/stac/catalog.rs @@ -69,6 +69,22 @@ pub fn parse_children(bytes: &[u8], from: &str) -> Result, String> { Ok(out) } +/// The items (`item` links) of a catalog's or collection's document fetched from `from`: web +/// addresses only, in the order the document gives them. +pub fn parse_item_links(bytes: &[u8], from: &str) -> Result, String> { + let v = super::json(bytes)?; + if !v.is_object() || !matches!(v.get("type").and_then(Value::as_str), Some("Catalog" | "Collection")) { + return Err("the answer isn't a STAC catalog or collection".into()); + } + let mut seen = HashSet::new(); + Ok(links(&v, "item", from) + .into_iter() + .filter(|(url, _)| is_web(url) && crate::address::check(url, Some(from)).is_ok() && seen.insert(url.clone())) + .take(MAX_CHILDREN) + .map(|(url, title)| Child { title: if title.is_empty() { label_of(&url) } else { title }, url }) + .collect()) +} + /// The links of a relation, resolved against `from`: (address, title), the first /// [`MAX_LINKS`] of them (counted among that relation's alone). fn links(v: &Value, rel: &str, from: &str) -> Vec<(String, String)> { @@ -499,6 +515,22 @@ mod tests { assert_eq!(parse_children(br#"{"type": "Catalog", "links": 7}"#, "https://x.org/c.json"), Ok(vec![])); assert_eq!(label_of("https://x.org/events/catalog.json"), "events"); assert_eq!(label_of("https://x.org/events/a.json"), "a"); + // A collection lists its items by `item` links, in the order it gives them. + let coll = json!({"type": "Collection", "id": "c", "links": [ + {"rel": "item", "href": "./i2/i2.json", "title": "Second"}, + {"rel": "item", "href": "./i1/i1.json"}, + {"rel": "item", "href": "./i2/i2.json"}, + {"rel": "item", "href": "file:///etc/passwd"}, + {"rel": "child", "href": "./sub/catalog.json"}]}); + let i = parse_item_links(coll.to_string().as_bytes(), "https://x.org/c/collection.json").unwrap(); + assert_eq!( + i, + vec![ + Child { title: "Second".into(), url: "https://x.org/c/i2/i2.json".into() }, + Child { title: "i1".into(), url: "https://x.org/c/i1/i1.json".into() }, + ] + ); + assert!(parse_item_links(br#"{"type": "Feature"}"#, "https://x.org/c.json").is_err()); } #[test] diff --git a/crates/ui-egui/src/connectiontree.rs b/crates/ui-egui/src/connectiontree.rs new file mode 100644 index 000000000..f952a2885 --- /dev/null +++ b/crates/ui-egui/src/connectiontree.rs @@ -0,0 +1,525 @@ +//! The datasets a saved connection offers, listed under it in Data Sources › Connections: one +//! level at a time as it is expanded (`connection.browse` names what each level reads, the app +//! fetches it off the frame and asks again with the answer), with the selection that +//! Add Selected (in a dataset's menu) adds. Each dataset added is its own undo step. + +use std::collections::HashMap; +use std::sync::Arc; + +use egui::collapsing_header::CollapsingState; +use egui::{Label, RichText, Sense, Ui}; +use rustgis_engine::connections::{Connection, DATABASE_KINDS}; +use rustgis_engine::render::rustgis_tiles::stac; +use rustgis_engine::sources::Source; +use serde_json::{Value, json}; + +use crate::serviceui::{self, ServiceSource}; +use crate::theme::Tokens; +use crate::{RustGisApp, Slot, i18n}; + +type Key = (String, String); + +#[derive(Default)] +pub struct State { + /// What each level of each connection lists: (connection id, node; empty for the top). + entries: HashMap, + /// The definition each connection's entries were made for: an edit makes them stale. + seen: HashMap, + pub picks: Vec, + /// Signed addresses of STAC assets waiting on a token: (token address, asset, name, answer). + tokens: Vec<(String, String, String, Slot, String>>)>, +} + +pub enum Entry { + Loading(Fetch), + Ready(Arc), + Failed(String), +} + +/// A level whose answer is on its way. +pub struct Fetch { + node: String, + /// The first answer, and the further documents a service's discovery asked for. + first: Option, + documents: Vec, + /// The address being fetched. + pending: String, + slot: Slot, String>>, +} + +pub struct Listing { + pub children: Vec, + pub truncated: bool, + pub matched: Option, + pub notes: Vec, + /// The service its layers belong to, to add them from the documents already read. + pub source: Option>, +} + +/// A dataset selected for Add Selected. +pub struct Pick { + pub conn: String, + pub key: String, + pub name: String, + pub child: Value, + pub source: Option>, +} + +fn params(id: &str, node: &str) -> Value { + let mut p = json!({"id": id}); + if !node.is_empty() { + p["node"] = json!(node); + } + p +} + +/// Starts reading a level: what it needs fetched is asked for, and a level read without a fetch +/// (a database's tables, an S3 connection's own buckets) is read now. Errors go into the entry. +fn start(app: &mut RustGisApp, c: &Connection, node: &str) { + let mut probe = params(&c.id, node); + probe["probe"] = json!(true); + let entry = match app.session.run("connection.browse", &probe) { + Err(e) => Entry::Failed(e), + Ok(v) => match v["fetch"].as_str() { + None => match app.session.run("connection.browse", ¶ms(&c.id, node)) { + Ok(v) => Entry::Ready(Arc::new(listing(&v, None, Vec::new()))), + Err(e) => Entry::Failed(e), + }, + Some(url) => Entry::Loading(Fetch { + node: node.to_string(), + first: None, + documents: Vec::new(), + pending: url.to_string(), + slot: crate::fetch_into(url), + }), + }, + }; + app.ui.connection_tree.entries.insert((c.id.clone(), node.to_string()), entry); +} + +fn listing(v: &Value, first: Option, documents: Vec) -> Listing { + let source = v.get("service").filter(|s| s.is_object()).map(|s| { + Arc::new(ServiceSource { + url: s["url"].as_str().unwrap_or("").to_string(), + kind: serviceui::kind_of(s["type"].as_str().unwrap_or("")), + text: first.unwrap_or_default(), + documents, + }) + }); + Listing { + children: v["children"].as_array().cloned().unwrap_or_default(), + truncated: v["truncated"] == true, + matched: v["matched"].as_u64(), + notes: v["notes"].as_array().into_iter().flatten().filter_map(Value::as_str).map(str::to_string).collect(), + source, + } +} + +/// An answer arrived: asks again with it (and what came before). A service's discovery may need +/// further documents, which are fetched in turn. +fn advance(app: &mut RustGisApp, id: &str, mut f: Fetch, bytes: Vec) -> Entry { + let Ok(text) = crate::service_text(bytes) else { + return Entry::Failed(tl!("The answer isn't UTF-8 text").to_string()); + }; + if f.first.is_none() { + f.first = Some(text); + } else { + f.documents.push(json!({"url": f.pending, "text": text})); + } + let mut p = params(id, &f.node); + p["answer"] = json!(f.first); + if !f.documents.is_empty() { + p["documents"] = json!(f.documents); + } + match app.session.run("connection.browse", &p) { + Err(e) => Entry::Failed(e), + Ok(v) => match v["next"].as_str() { + Some(next) => { + f.pending = next.to_string(); + f.slot = crate::fetch_into(next); + Entry::Loading(f) + } + None => Entry::Ready(Arc::new(listing(&v, f.first, f.documents))), + }, + } +} + +pub fn poll(app: &mut RustGisApp, ctx: &egui::Context) { + let tree = &mut app.ui.connection_tree; + let mut landed = Vec::new(); + for (key, entry) in &tree.entries { + if let Entry::Loading(f) = entry + && let Some(r) = f.slot.lock().unwrap_or_else(|e| e.into_inner()).take() + { + landed.push((key.clone(), r)); + } + } + for (key, r) in landed { + let Some(Entry::Loading(f)) = app.ui.connection_tree.entries.remove(&key) else { continue }; + let entry = match r { + Ok(bytes) => advance(app, &key.0, f, bytes), + Err(e) => Entry::Failed(e), + }; + app.ui.connection_tree.entries.insert(key, entry); + } + // STAC assets that needed a token: signed and added when it lands. + for (token_url, href, name, slot) in std::mem::take(&mut app.ui.connection_tree.tokens) { + let Some(r) = slot.lock().unwrap_or_else(|e| e.into_inner()).take() else { + app.ui.connection_tree.tokens.push((token_url, href, name, slot)); + continue; + }; + match r.and_then(|b| stac::parse_sas(&b)) { + Ok(token) => { + stac::remember_token(&token_url, &token); + if let Err(e) = app.run("ui.addUrl", json!({"url": stac::sign(&href, &token), "name": name, "zoom": true})) { + app.session.notify(e, true); + } + app.map.invalidate(); + } + Err(e) => app.session.notify(e, true), + } + } + let tree = &app.ui.connection_tree; + if tree.entries.values().any(|e| matches!(e, Entry::Loading(_))) || !tree.tokens.is_empty() || !app.ui.service_steps.is_empty() { + ctx.request_repaint_after(std::time::Duration::from_millis(100)); + } +} + +/// Adds one dataset a listing offers. Each branch is one command, so each is one undo step. +pub fn add(app: &mut RustGisApp, child: &Value, source: Option<&ServiceSource>) -> Result<(), String> { + let add = child.get("add").filter(|a| a.is_object()).ok_or_else(|| tl!("Nothing to add here").to_string())?; + let command = add["command"].as_str().unwrap_or(""); + let params = add["params"].clone(); + match (source, child.get("layer").filter(|l| l.is_object())) { + (Some(source), Some(layer)) => { + serviceui::add_layer(app, source, layer, "", false)?; + } + _ => match command { + "layer.addUrl" => { + app.run("ui.addUrl", params)?; + } + "stac.addAsset" => add_asset(app, child["href"].as_str().unwrap_or(""), params["name"].as_str().unwrap_or(""))?, + _ => { + app.run(command, params)?; + } + }, + } + app.map.invalidate(); + Ok(()) +} + +/// A STAC asset by its address, signed first when it needs a token (one in hand is used; one to +/// fetch is fetched off the frame and the asset added when it lands). +fn add_asset(app: &mut RustGisApp, href: &str, name: &str) -> Result<(), String> { + if !stac::needs_signing(href) { + app.run("ui.addUrl", json!({"url": href, "name": name, "zoom": true}))?; + return Ok(()); + } + let token_url = stac::sas_token_url(href)?; + if let Some(token) = stac::cached_token(&token_url) { + app.run("ui.addUrl", json!({"url": stac::sign(href, &token), "name": name, "zoom": true}))?; + } else { + let slot = crate::fetch_into(&token_url); + app.ui.connection_tree.tokens.push((token_url, href.to_string(), name.to_string(), slot)); + } + Ok(()) +} + +/// Opens the kind's own add dialog on a dataset: a table in the database dialog, a service layer +/// in the service dialog, a STAC collection in the STAC dialog. +fn add_with_options(app: &mut RustGisApp, c: &Connection, child: &Value, source: Option<&ServiceSource>) -> Result<(), String> { + match child["what"].as_str().unwrap_or("") { + "table" => crate::sourcespane::open( + app, + &Source { kind: "database".into(), name: c.name.clone(), address: c.address.clone(), extra: json!({"layer": child["key"]}) }, + ), + "collection" => { + crate::sourcespane::open( + app, + &Source { kind: "stac".into(), name: c.name.clone(), address: c.address.clone(), extra: json!({"api": "true"}) }, + )?; + app.ui.stac.collection = child["detail"].as_str().unwrap_or("").to_string(); + Ok(()) + } + _ => match (source, child.get("layer").filter(|l| l.is_object())) { + (Some(source), Some(layer)) => { + app.run("ui.addService", json!({"url": source.url, "type": source.kind}))?; + app.ui.service_browse.filter = layer["name"].as_str().unwrap_or("").to_string(); + Ok(()) + } + _ => Err(tl!("Nothing to add here").to_string()), + }, + } +} + +/// What a listed child is, in words. +fn what_label(what: &str) -> &'static str { + match what { + "schema" => tl!("schema"), + "table" => tl!("table"), + "folder" => tl!("folder"), + "service" => tl!("service"), + "map" => tl!("map"), + "vectorTiles" => tl!("vector tiles"), + "features" => tl!("features"), + "coverage" => tl!("coverage"), + "style" => tl!("style"), + "record" => tl!("record"), + "download" => tl!("download"), + "collection" => tl!("collection"), + "catalog" => tl!("catalog"), + "item" => tl!("item"), + "asset" => tl!("asset"), + "bucket" => tl!("bucket"), + "prefix" => tl!("folder"), + "object" => tl!("file"), + _ => "", + } +} + +fn picked(app: &RustGisApp, conn: &str, key: &str) -> bool { + app.ui.connection_tree.picks.iter().any(|p| p.conn == conn && p.key == key) +} + +fn toggle(app: &mut RustGisApp, c: &Connection, child: &Value, source: Option<&Arc>) { + let key = child["key"].as_str().unwrap_or(""); + let picks = &mut app.ui.connection_tree.picks; + if let Some(i) = picks.iter().position(|p| p.conn == c.id && p.key == key) { + picks.remove(i); + } else { + picks.push(Pick { + conn: c.id.clone(), + key: key.to_string(), + name: child["name"].as_str().unwrap_or("").to_string(), + child: child.clone(), + source: source.cloned(), + }); + } +} + +/// Forgets what a connection (or, with `None`, every connection) listed and what was selected +/// from it: the next expansion reads it again. +pub fn refresh(app: &mut RustGisApp, id: Option<&str>) { + let tree = &mut app.ui.connection_tree; + match id { + Some(id) => { + tree.entries.retain(|(c, _), _| c != id); + tree.picks.retain(|p| p.conn != id); + } + None => { + tree.entries.clear(); + tree.picks.clear(); + } + } +} + +/// Drops the picks of a connection that was edited or removed since (even while its row is +/// folded, when its list isn't refreshed): they would add from the old definition. +pub fn prune(app: &mut RustGisApp) { + let (connections, tree) = (&app.session.connections, &mut app.ui.connection_tree); + let seen = &tree.seen; + tree.picks.retain(|p| connections.find(&p.conn).is_some_and(|c| seen.get(&p.conn) == Some(c))); +} + +/// Adds every selected dataset, each as its own undo step, and clears the selection. +fn add_picks(app: &mut RustGisApp) { + prune(app); + for pick in std::mem::take(&mut app.ui.connection_tree.picks) { + if let Err(e) = add(app, &pick.child, pick.source.as_deref()) { + app.session.notify(e, true); + } + } +} + +/// A connection's expanded body: its top level, listed (or its error). +pub fn show(app: &mut RustGisApp, ui: &mut Ui, t: &Tokens, c: &Connection) { + // An edited connection lists what it now offers. + if app.ui.connection_tree.seen.get(&c.id) != Some(c) { + refresh(app, Some(&c.id)); + app.ui.connection_tree.seen.insert(c.id.clone(), c.clone()); + } + level(app, ui, t, c, ""); +} + +/// One level: loading, its error with the ways to go on, or its children. +fn level(app: &mut RustGisApp, ui: &mut Ui, t: &Tokens, c: &Connection, node: &str) { + let key = (c.id.clone(), node.to_string()); + let shown = match app.ui.connection_tree.entries.get(&key) { + None => None, + Some(Entry::Loading(_)) => Some(Err(None)), + Some(Entry::Failed(e)) => Some(Err(Some(e.clone()))), + Some(Entry::Ready(l)) => Some(Ok(l.clone())), + }; + match shown { + None => { + start(app, c, node); + ui.ctx().request_repaint(); + } + Some(Err(None)) => { + ui.horizontal(|ui| { + ui.spinner(); + ui.label(tl!("Loading…")); + }); + } + Some(Err(Some(e))) => { + ui.label(RichText::new(e).small().color(t.red)); + ui.horizontal(|ui| { + if ui.small_button(tl!("Retry")).clicked() { + app.ui.connection_tree.entries.remove(&key); + } + let keys = c.kind == "s3" && c.options.get("credentials").is_some_and(|v| v == "keys"); + if (DATABASE_KINDS.contains(&c.kind.as_str()) || keys) && ui.small_button(tl!("Log In…")).clicked() { + match app.run("ui.connection", json!({"id": c.id})) { + Ok(_) => { + app.dialogs.info = Some(tl!("Type the password or keys to log in for this session, then Save.").to_string()) + } + Err(e) => app.session.notify(e, true), + } + } + }); + } + Some(Ok(l)) => { + children(app, ui, t, c, &l, &l.children); + if l.children.is_empty() { + ui.label(RichText::new(tl!("Nothing here")).small().weak()); + } + if l.truncated { + let n = l.children.len().to_string(); + let text = match l.matched { + Some(m) => i18n::fmt("{n} of {m} are listed.", &[("n", &n), ("m", &m.to_string())]), + None => i18n::fmt("The first {n} are listed.", &[("n", &n)]), + }; + ui.label(RichText::new(text).small().weak()); + } + for note in &l.notes { + ui.label(RichText::new(note).small().weak()); + } + } + } +} + +/// A folder in the tree: a header that opens on a click, and a body drawn when it is open. +/// `node` is set when the body is a level of its own (read when first opened, refreshed from +/// the header's menu). +fn branch( + app: &mut RustGisApp, + ui: &mut Ui, + c: &Connection, + child: &Value, + node: Option<&str>, + body: impl FnOnce(&mut RustGisApp, &mut Ui), +) { + let salt = node.or(child["key"].as_str()).unwrap_or(""); + let id = ui.make_persistent_id(("conntree", &c.id, salt)); + let state = CollapsingState::load_with_default_open(ui.ctx(), id, false); + let open = state.is_open(); + let (mut clicked, mut refreshed) = (false, false); + let mut header = state.show_header(ui, |ui| { + ui.label(RichText::new(what_label(child["what"].as_str().unwrap_or(""))).small().weak()); + let r = ui.add(Label::new(child["name"].as_str().unwrap_or("")).sense(Sense::click())); + clicked = r.clicked(); + if node.is_some() { + r.context_menu(|ui| { + if ui.button(tl!("Refresh")).clicked() { + refreshed = true; + ui.close(); + } + }); + } + }); + if clicked { + header.set_open(!open); + } + header.body(|ui| body(app, ui)); + if refreshed && let Some(node) = node { + app.ui.connection_tree.entries.remove(&(c.id.clone(), node.to_string())); + } +} + +fn children(app: &mut RustGisApp, ui: &mut Ui, t: &Tokens, c: &Connection, listing: &Listing, list: &[Value]) { + for child in list { + if let Some(node) = child["node"].as_str() { + branch(app, ui, c, child, Some(node), |app, ui| level(app, ui, t, c, node)); + continue; + } + if let Some(inline) = child["children"].as_array() { + branch(app, ui, c, child, None, |app, ui| { + children(app, ui, t, c, listing, inline); + if inline.is_empty() { + ui.label(RichText::new(tl!("Nothing here")).small().weak()); + } + }); + continue; + } + dataset(app, ui, c, listing, child); + } +} + +/// A dataset: click selects it, double-click adds it, the menu adds it (with options). +fn dataset(app: &mut RustGisApp, ui: &mut Ui, c: &Connection, listing: &Listing, child: &Value) { + let name = child["name"].as_str().unwrap_or(""); + let is_picked = picked(app, &c.id, child["key"].as_str().unwrap_or("")); + let r = ui + .horizontal(|ui| { + let r = ui.selectable_label(is_picked, name); + ui.label(RichText::new(what_label(child["what"].as_str().unwrap_or(""))).small().weak()); + r + }) + .inner; + // A dataset without an add (an OGC API feature collection, imported together) is added + // with options only. + let addable = child.get("add").is_some_and(Value::is_object); + let hint = if addable { tl!("Click to select, double-click to add, right-click for more") } else { tl!("Right-click for more") }; + let mut hover = hint.to_string(); + if let Some(detail) = child["detail"].as_str().filter(|d| !d.is_empty()) { + hover = format!("{hover}\n{detail}"); + } + let r = r.on_hover_text(hover); + let (mut add_now, mut with_options, mut add_selected, mut clear) = (false, false, false, false); + let selected = app.ui.connection_tree.picks.len(); + r.context_menu(|ui| { + if addable && ui.button(tl!("Add")).clicked() { + add_now = true; + ui.close(); + } + if child["options"] == true && ui.button(tl!("Add With Options…")).clicked() { + with_options = true; + ui.close(); + } + if selected > 0 { + ui.separator(); + if ui.button(i18n::fmt("Add Selected ({n})", &[("n", &selected.to_string())])).clicked() { + add_selected = true; + ui.close(); + } + if ui.button(tl!("Clear Selection")).clicked() { + clear = true; + ui.close(); + } + } + }); + if addable && r.double_clicked() { + // The first click selected it: a double-click adds it alone. + let key = child["key"].as_str().unwrap_or(""); + app.ui.connection_tree.picks.retain(|p| !(p.conn == c.id && p.key == key)); + add_now = true; + } else if addable && r.clicked() { + toggle(app, c, child, listing.source.as_ref()); + } + if clear { + app.ui.connection_tree.picks.clear(); + } + if add_selected { + add_picks(app); + } + let result = if with_options { + add_with_options(app, c, child, listing.source.as_deref()) + } else if add_now { + add(app, child, listing.source.as_deref()) + } else { + Ok(()) + }; + if let Err(e) = result { + app.session.notify(e, true); + } +} diff --git a/crates/ui-egui/src/dialogs.rs b/crates/ui-egui/src/dialogs.rs index c7041460f..06cfdb4ca 100644 --- a/crates/ui-egui/src/dialogs.rs +++ b/crates/ui-egui/src/dialogs.rs @@ -800,60 +800,17 @@ fn add_service(app: &mut RustGisApp, ui: &mut egui::Ui) -> bool { }); } if let (Some(l), Some(conn)) = (add, &app.ui.service) { - let (url, kind, text) = (conn.url.clone(), conn.kind, conn.text.clone().unwrap_or_default()); - let name = l.get("name").and_then(Value::as_str).unwrap_or("").to_string(); - let title = l.get("title").and_then(Value::as_str).unwrap_or(&name).to_string(); - let offer = l.get("offer").cloned().unwrap_or_default(); - // Features are downloaded in the background, as From URL… does; map layers are added now. - let r = if offer.get("type").and_then(Value::as_str) == Some("features") { - let items = offer.get("url").and_then(Value::as_str).unwrap_or("").to_string(); - app.run("ui.addUrl", json!({"url": items, "name": title, "service": true})).map(|_| tl!("Downloading…")) - } else if offer.get("type").and_then(Value::as_str) == Some("coverage") { - // A coverage: its description first, off the frame (poll_describe asks for it). - let describe = offer.get("describe").and_then(Value::as_str).unwrap_or("").to_string(); - let mut p = json!({"url": url, "layer": name, "text": text, "name": title, "documents": conn.documents}); - if kind != "auto" { - p["type"] = json!(kind); - } - let slot = crate::fetch_into(&describe); - if let Some(c) = &mut app.ui.service { - c.describing = Some((p, slot)); - } - Ok(tl!("Describing")) - } else { - let mut p = json!({"url": url, "layer": name, "text": text, "documents": conn.documents, "source_url": l.get("source_url")}); - if kind != "auto" { - p["type"] = json!(kind); - } - let time = app.ui.service_browse.time.trim(); - if l.get("time").is_some() && !time.is_empty() { - p["time"] = json!(time); - } - match l.get("tilesets").and_then(Value::as_str) { - // An OGC API layer: its tileset's advertised template is read first, off the - // frame (poll_tiles_resolve adds the layer). - Some(list) => { - p["name"] = json!(title); - let fallback = offer.get("url").and_then(Value::as_str).unwrap_or("").to_string(); - let vector = offer.get("type").and_then(Value::as_str) == Some("vectorTiles"); - let slot = crate::fetch_into(list); - if let Some(c) = &mut app.ui.service { - c.resolving = Some(crate::TilesResolve { - params: p, - list: list.to_string(), - doc: list.to_string(), - read: 0, - vector, - fallback, - slot, - }); - } - Ok(tl!("Reading the tileset of")) - } - None => app.run("service.add", p).map(|_| tl!("Added")), - } + let src = crate::serviceui::ServiceSource { + url: conn.url.clone(), + kind: conn.kind, + text: conn.text.clone().unwrap_or_default(), + documents: conn.documents.clone(), }; - match r { + let name = l.get("name").and_then(Value::as_str).unwrap_or(""); + let title = l.get("title").and_then(Value::as_str).unwrap_or(name).to_string(); + // Features are downloaded in the background, as From URL… does; map layers are added now. + let time = app.ui.service_browse.time.trim().to_string(); + match crate::serviceui::add_layer(app, &src, &l, &time, true) { Ok(m) => app.dialogs.info = Some(format!("{m} {title}")), Err(e) => app.dialogs.error = Some(e), } diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index e4040665c..100a0abbd 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -17,6 +17,7 @@ mod bookmarkpane; mod catalogsui; mod chrome; mod comets; +mod connectiontree; mod connectionui; #[cfg(not(target_arch = "wasm32"))] mod connectui; @@ -291,11 +292,6 @@ pub struct ServiceConn { pub error: Option, /// A capabilities request that failed, with what the server said (#509). pub failure: Option, - /// An OGC API layer whose advertised tile template is being read before it is added. - pub resolving: Option, - /// A WCS coverage whose description is being read before it is asked for: `service.coverage`'s - /// parameters and the description on its way. - pub describing: Option<(Value, Slot, String>>)>, } /// How Add From Service's list is browsed. @@ -324,6 +320,16 @@ pub struct TilesResolve { pub slot: Slot, String>>, } +/// A service layer waiting on a document before it is added. `dialog` tells whether the add came +/// from the add-service dialog (its result shows there) or from the Connections tree (a notice). +pub enum ServiceStep { + /// An OGC API layer whose advertised tile template is being read. + Tiles { resolve: TilesResolve, dialog: bool }, + /// A WCS coverage whose description is being read before it is asked for: `service.coverage`'s + /// parameters and the description on its way. + Coverage { params: Value, slot: Slot, String>>, dialog: bool }, +} + /// Fetches an address into a slot, off the frame. pub(crate) fn fetch_into(url: &str) -> Slot, String>> { fetch_into_limited(url, rustgis_engine::render::rustgis_tiles::cache::MAX_DOWNLOAD) @@ -365,7 +371,7 @@ pub enum LayersView { } /// Retains the GUI's legacy XML decoding while refusing malformed JSON text. -fn service_text(bytes: Vec) -> Result { +pub(crate) fn service_text(bytes: Vec) -> Result { String::from_utf8(bytes).or_else(|error| { let bytes = error.as_bytes(); let without_bom = bytes.strip_prefix(b"\xef\xbb\xbf").unwrap_or(bytes); @@ -515,6 +521,9 @@ pub struct UiState { /// Add From Service's list: its filter, a time-enabled layer's date, and the directories /// opened on the way to the service shown (Back returns to them). pub service_browse: ServiceBrowse, + /// Service layers added from the dialog or the Connections tree that wait on a document + /// (a tileset list, a coverage's description) before they are added. + pub service_steps: Vec, /// The Web Data panels of single services (NASA Earthdata, EnviroAtlas, FEMA NFHL, The /// National Map, Fields of the World): their documents, searches and choices. pub web_services: webdataui::State, @@ -528,6 +537,8 @@ pub struct UiState { pub s3: s3ui::State, /// New/Edit Connection: the form and the Test under way. pub connection: connectionui::State, + /// The datasets saved connections list under Data Sources › Connections, and the selection. + pub connection_tree: connectiontree::State, /// OpenStreetMap and Boundaries: their choices and the request under way. pub portals: portalsui::State, /// Overture Maps and Source Cooperative: choices, results and the request under way. @@ -649,6 +660,7 @@ impl Default for UiState { link_zoomed: false, service: None, service_browse: ServiceBrowse::default(), + service_steps: Vec::new(), web_services: Default::default(), service_import: None, service_group_name: "My OGC Layer".into(), @@ -656,6 +668,7 @@ impl Default for UiState { stac: Default::default(), s3: Default::default(), connection: Default::default(), + connection_tree: Default::default(), portals: Default::default(), catalogs: Default::default(), opendata: Default::default(), @@ -2113,8 +2126,6 @@ impl RustGisApp { listing: None, error: None, failure: None, - resolving: None, - describing: None, }); } "ui.addData" => { @@ -2993,60 +3004,71 @@ impl RustGisApp { } } - /// Moves an OGC API layer's template reading along: the list, then the tileset when the list - /// doesn't carry the template; the layer is added with the template read, or the usual path. - fn poll_tiles_resolve(&mut self) { - use rustgis_engine::data::ogc::{TilesStep, resolve_tiles}; - let Some(conn) = &mut self.ui.service else { return }; - let Some(res) = &mut conn.resolving else { return }; - let Some(r) = res.slot.lock().unwrap_or_else(|e| e.into_inner()).take() else { return }; - let step = r.ok().and_then(|body| resolve_tiles(&res.list, &res.doc, &body, res.vector)); - let template = match step { - Some(TilesStep::Next(next)) if res.read < 2 => { - res.read += 1; - res.doc = next.clone(); - res.slot = fetch_into(&next); - return; - } - Some(TilesStep::Template(t)) => t, - _ => res.fallback.clone(), - }; - let Some(res) = conn.resolving.take() else { return }; - let mut p = res.params; - p["tiles"] = json!(template); - let title = p["name"].as_str().unwrap_or("").to_string(); - match self.run("service.add", p) { - Ok(_) => self.dialogs.info = Some(format!("{} {title}", tl!("Added"))), - Err(e) => self.dialogs.error = Some(e), + /// Tells how a service layer added outside the add-service dialog (or in it) went: in the + /// dialog's own line, or as a notice for the Connections tree. + fn report_step(&mut self, dialog: bool, r: Result) { + match (dialog, r) { + (true, Ok(m)) => self.dialogs.info = Some(m), + (true, Err(e)) => self.dialogs.error = Some(e), + (false, Ok(m)) => self.session.notify(m, false), + (false, Err(e)) => self.session.notify(e, true), } - self.map.invalidate(); } - /// Asks for a WCS coverage once its description has arrived: its GetCoverage address for the - /// view, downloaded as From URL… does. - fn poll_describe(&mut self) { - let Some(conn) = &mut self.ui.service else { return }; - let Some((_, slot)) = &conn.describing else { return }; - let Some(r) = slot.lock().unwrap_or_else(|e| e.into_inner()).take() else { return }; - let Some((mut p, _)) = conn.describing.take() else { return }; - let title = p["name"].as_str().unwrap_or("").to_string(); - let r = r.and_then(|bytes| { - p["describe"] = json!(String::from_utf8_lossy(&bytes)); - // Not `self.run`: an error belongs in the dialog, not the notices. - let c = self.session.run("service.coverage", &p)?; - let get = c.get("url").and_then(Value::as_str).unwrap_or("").to_string(); - self.run("ui.addUrl", json!({"url": get, "name": title, "coverage": true})) - }); - match r { - Ok(_) => self.dialogs.info = Some(format!("{} {title}", tl!("Downloading…"))), - Err(e) => self.dialogs.error = Some(e), + /// Moves the service layers waiting on a document along: an OGC API layer's tile template + /// (read from its tileset list, then the tileset when the list doesn't carry it; the layer is + /// added with the template read, or the usual path) and a WCS coverage's description (the + /// coverage is then asked for the view and downloaded as From URL… does). + fn poll_service_steps(&mut self) { + use rustgis_engine::data::ogc::{TilesStep, resolve_tiles}; + for step in std::mem::take(&mut self.ui.service_steps) { + match step { + ServiceStep::Tiles { mut resolve, dialog } => { + let Some(r) = resolve.slot.lock().unwrap_or_else(|e| e.into_inner()).take() else { + self.ui.service_steps.push(ServiceStep::Tiles { resolve, dialog }); + continue; + }; + let next = r.ok().and_then(|body| resolve_tiles(&resolve.list, &resolve.doc, &body, resolve.vector)); + let template = match next { + Some(TilesStep::Next(next)) if resolve.read < 2 => { + resolve.read += 1; + resolve.doc = next.clone(); + resolve.slot = fetch_into(&next); + self.ui.service_steps.push(ServiceStep::Tiles { resolve, dialog }); + continue; + } + Some(TilesStep::Template(t)) => t, + _ => resolve.fallback.clone(), + }; + let mut p = resolve.params; + p["tiles"] = json!(template); + let title = p["name"].as_str().unwrap_or("").to_string(); + let r = self.run("service.add", p).map(|_| format!("{} {title}", tl!("Added"))); + self.report_step(dialog, r); + self.map.invalidate(); + } + ServiceStep::Coverage { mut params, slot, dialog } => { + let Some(r) = slot.lock().unwrap_or_else(|e| e.into_inner()).take() else { + self.ui.service_steps.push(ServiceStep::Coverage { params, slot, dialog }); + continue; + }; + let title = params["name"].as_str().unwrap_or("").to_string(); + let r = r.and_then(|bytes| { + params["describe"] = json!(String::from_utf8_lossy(&bytes)); + // Not `self.run`: an error belongs where the step reports, not the notices. + let c = self.session.run("service.coverage", ¶ms)?; + let get = c.get("url").and_then(Value::as_str).unwrap_or("").to_string(); + self.run("ui.addUrl", json!({"url": get, "name": title, "coverage": true})) + }); + self.report_step(dialog, r.map(|_| format!("{} {title}", tl!("Downloading…")))); + } + } } } /// Lists a service's layers once its capabilities have arrived. fn poll_service(&mut self) { - self.poll_tiles_resolve(); - self.poll_describe(); + self.poll_service_steps(); let Some(conn) = &mut self.ui.service else { return }; let Some(r) = conn.slot.lock().unwrap_or_else(|e| e.into_inner()).take() else { return }; let document = conn.pending.take().unwrap_or_else(|| conn.url.clone()); @@ -3170,6 +3192,7 @@ impl RustGisApp { stacui::poll(self, ctx); s3ui::poll(self, ctx); connectionui::poll(self, ctx); + connectiontree::poll(self, ctx); portalsui::poll(self, ctx); catalogsui::poll(self, ctx); opendataui::poll(self, ctx); diff --git a/crates/ui-egui/src/serviceui.rs b/crates/ui-egui/src/serviceui.rs index c000294fd..834c6e5a9 100644 --- a/crates/ui-egui/src/serviceui.rs +++ b/crates/ui-egui/src/serviceui.rs @@ -20,6 +20,74 @@ pub struct Import { bytes: usize, } +/// A service whose layers are listed: its address, type, capabilities document and the further +/// documents its discovery read. `service.add` reads layers from them without fetching again. +pub struct ServiceSource { + pub url: String, + /// `wms`, `wmts`, `wfs`, `wcs`, `ogcapi`, `arcgis`, or `auto`. + pub kind: &'static str, + pub text: String, + pub documents: Vec, +} + +/// A service type as the add dialog and `ui.addService` name it, `auto` for any other text. +pub(crate) fn kind_of(t: &str) -> &'static str { + match t { + "wms" => "wms", + "wmts" => "wmts", + "wfs" => "wfs", + "wcs" => "wcs", + "ogcapi" => "ogcapi", + "arcgis" => "arcgis", + _ => "auto", + } +} + +/// Adds one of a service's layers (as `service.layers` lists it): features are downloaded in the +/// background as From URL… does, map layers are added now, and an OGC API layer or a WCS coverage +/// waits on a document (a step the app moves along). `dialog` tells where the result is reported. +pub(crate) fn add_layer(app: &mut RustGisApp, src: &ServiceSource, l: &Value, time: &str, dialog: bool) -> Result<&'static str, String> { + let name = l.get("name").and_then(Value::as_str).unwrap_or("").to_string(); + let title = l.get("title").and_then(Value::as_str).unwrap_or(&name).to_string(); + let offer = l.get("offer").cloned().unwrap_or_default(); + if offer.get("type").and_then(Value::as_str) == Some("features") { + let items = offer.get("url").and_then(Value::as_str).unwrap_or("").to_string(); + return app.run("ui.addUrl", json!({"url": items, "name": title, "service": true})).map(|_| tl!("Downloading…")); + } + if offer.get("type").and_then(Value::as_str) == Some("coverage") { + // A coverage: its description first, off the frame (a step asks for it). + let describe = offer.get("describe").and_then(Value::as_str).unwrap_or("").to_string(); + let mut p = json!({"url": src.url, "layer": name, "text": src.text, "name": title, "documents": src.documents}); + if src.kind != "auto" { + p["type"] = json!(src.kind); + } + let slot = crate::fetch_into(&describe); + app.ui.service_steps.push(crate::ServiceStep::Coverage { params: p, slot, dialog }); + return Ok(tl!("Describing")); + } + let mut p = json!({"url": src.url, "layer": name, "text": src.text, "documents": src.documents, "source_url": l.get("source_url")}); + if src.kind != "auto" { + p["type"] = json!(src.kind); + } + if l.get("time").is_some() && !time.is_empty() { + p["time"] = json!(time); + } + match l.get("tilesets").and_then(Value::as_str) { + // An OGC API layer: its tileset's advertised template is read first, off the frame (a + // step adds the layer). + Some(list) => { + p["name"] = json!(title); + let fallback = offer.get("url").and_then(Value::as_str).unwrap_or("").to_string(); + let vector = offer.get("type").and_then(Value::as_str) == Some("vectorTiles"); + let slot = crate::fetch_into(list); + let resolve = crate::TilesResolve { params: p, list: list.to_string(), doc: list.to_string(), read: 0, vector, fallback, slot }; + app.ui.service_steps.push(crate::ServiceStep::Tiles { resolve, dialog }); + Ok(tl!("Reading the tileset of")) + } + None => app.run("service.add", p).map(|_| tl!("Added")), + } +} + pub fn start(app: &mut RustGisApp, p: &Value, refresh: bool) -> Result<(), String> { if !p.is_object() { return Err(tl!("Give the service's web address").into()); diff --git a/crates/ui-egui/src/sourcespane.rs b/crates/ui-egui/src/sourcespane.rs index 0d01ae54d..7ac0d7ae4 100644 --- a/crates/ui-egui/src/sourcespane.rs +++ b/crates/ui-egui/src/sourcespane.rs @@ -236,17 +236,23 @@ fn empty(ui: &mut Ui, what: &str) { } fn connections(app: &mut RustGisApp, ui: &mut Ui, t: &Tokens, list: &[Connection]) { - ui.menu_button(tl!("New Connection ▾"), |ui| { - for (kind, label) in crate::connectionui::KINDS { - if cfg!(target_arch = "wasm32") && DATABASE_KINDS.contains(&kind) { - continue; - } - if ui.button(tl!(label)).clicked() { - if let Err(e) = app.run("ui.connection", json!({"kind": kind})) { - app.session.notify(e, true); + crate::connectiontree::prune(app); + ui.horizontal(|ui| { + ui.menu_button(tl!("New Connection ▾"), |ui| { + for (kind, label) in crate::connectionui::KINDS { + if cfg!(target_arch = "wasm32") && DATABASE_KINDS.contains(&kind) { + continue; + } + if ui.button(tl!(label)).clicked() { + if let Err(e) = app.run("ui.connection", json!({"kind": kind})) { + app.session.notify(e, true); + } + ui.close(); } - ui.close(); } + }); + if ui.small_button("⟳").on_hover_text(tl!("Refresh every connection's lists")).clicked() { + crate::connectiontree::refresh(app, None); } }); if list.is_empty() { @@ -269,60 +275,86 @@ fn connections(app: &mut RustGisApp, ui: &mut Ui, t: &Tokens, list: &[Connection } fn connection_row(app: &mut RustGisApp, ui: &mut Ui, t: &Tokens, c: &Connection) { - let database = DATABASE_KINDS.contains(&c.kind.as_str()); - let unavailable = database && cfg!(target_arch = "wasm32"); - ui.horizontal(|ui| { - let icon = if database { - "table" - } else { - match c.kind.as_str() { - "stac" => "search", - "s3" => "package", - _ => "globe_view", - } - }; - let (r, _) = ui.allocate_exact_size(vec2(16.0, 16.0), Sense::hover()); - icons::paint(ui.painter(), r, icon, t.icon, t.accent); - let response = if unavailable { - ui.link(RichText::new(&c.name).weak()).on_hover_text(tl!("Database connections are used by the desktop app")) - } else { - ui.link(&c.name).on_hover_text(if c.kind == "s3" && c.address.is_empty() { tl!("Amazon S3") } else { &c.address }) - }; - if response.clicked() - && !unavailable - && let Err(e) = crate::connectionui::open_connection(app, c) - { - app.session.notify(e, true); + let unavailable = DATABASE_KINDS.contains(&c.kind.as_str()) && cfg!(target_arch = "wasm32"); + if unavailable { + ui.horizontal(|ui| { + connection_header(app, ui, t, c, true); + }); + return; + } + // A click on the name opens the row: the datasets the connection offers are listed under it. + let state = + egui::collapsing_header::CollapsingState::load_with_default_open(ui.ctx(), ui.make_persistent_id(("connection", &c.id)), false); + let open = state.is_open(); + let mut clicked = false; + let mut header = state.show_header(ui, |ui| clicked = connection_header(app, ui, t, c, false)); + if clicked { + header.set_open(!open); + } + header.body(|ui| crate::connectiontree::show(app, ui, t, c)); +} + +/// A connection's icon, name, kind and menu. True when the name was clicked. +fn connection_header(app: &mut RustGisApp, ui: &mut Ui, t: &Tokens, c: &Connection, unavailable: bool) -> bool { + let icon = if DATABASE_KINDS.contains(&c.kind.as_str()) { + "table" + } else { + match c.kind.as_str() { + "stac" => "search", + "s3" => "package", + _ => "globe_view", } - ui.label(RichText::new(tl!(crate::connectionui::label(&c.kind))).small().weak()); - response.context_menu(|ui| { - for (label, command, params) in [ - (tl!("Edit…"), "ui.connection", json!({"id":c.id})), - (tl!("Test"), "ui.connection", json!({"id":c.id,"test":true})), - (tl!("Duplicate"), "connection.duplicate", json!({"id":c.id})), - ] { - if params.get("test") == Some(&Value::Bool(true)) && unavailable { - continue; - } - if ui.button(label).clicked() { - if let Err(e) = app.run(command, params) { - app.session.notify(e, true); - } - ui.close(); + }; + let (r, _) = ui.allocate_exact_size(vec2(16.0, 16.0), Sense::hover()); + icons::paint(ui.painter(), r, icon, t.icon, t.accent); + let response = if unavailable { + ui.link(RichText::new(&c.name).weak()).on_hover_text(tl!("Database connections are used by the desktop app")) + } else { + ui.link(&c.name).on_hover_text(if c.kind == "s3" && c.address.is_empty() { tl!("Amazon S3") } else { &c.address }) + }; + ui.label(RichText::new(tl!(crate::connectionui::label(&c.kind))).small().weak()); + response.context_menu(|ui| { + let mut items = vec![ + (tl!("Edit…"), "ui.connection", json!({"id":c.id})), + (tl!("Test"), "ui.connection", json!({"id":c.id,"test":true})), + (tl!("Duplicate"), "connection.duplicate", json!({"id":c.id})), + ]; + if unavailable { + items.remove(1); + } + for (label, command, params) in items { + if ui.button(label).clicked() { + if let Err(e) = app.run(command, params) { + app.session.notify(e, true); } + ui.close(); } - if !c.address.is_empty() && ui.button(tl!("Copy Address")).clicked() { - ui.ctx().copy_text(c.address.clone()); + } + if !unavailable { + if ui.button(tl!("Refresh")).clicked() { + crate::connectiontree::refresh(app, Some(&c.id)); ui.close(); } - if ui.button(tl!("Remove")).clicked() { - if let Err(e) = app.run("connection.remove", json!({"id":c.id})) { + // The kind's own add dialog, as a click on the connection opened it before. + if ui.button(tl!("Add With Options…")).clicked() { + if let Err(e) = crate::connectionui::open_connection(app, c) { app.session.notify(e, true); } ui.close(); } - }); + } + if !c.address.is_empty() && ui.button(tl!("Copy Address")).clicked() { + ui.ctx().copy_text(c.address.clone()); + ui.close(); + } + if ui.button(tl!("Remove")).clicked() { + if let Err(e) = app.run("connection.remove", json!({"id":c.id})) { + app.session.notify(e, true); + } + ui.close(); + } }); + !unavailable && response.clicked() } pub fn show(app: &mut RustGisApp, ui: &mut Ui, t: &Tokens) { diff --git a/crates/ui-egui/tests/smoke.rs b/crates/ui-egui/tests/smoke.rs index 76cf33860..e488d14d4 100644 --- a/crates/ui-egui/tests/smoke.rs +++ b/crates/ui-egui/tests/smoke.rs @@ -1695,6 +1695,89 @@ fn connections_test_feedback_tracks_edited_inputs() { assert!(h.state().session.connections.connections.is_empty()); } +/// Expanding a connection lists its datasets, a click selects and Add Selected adds each as its +/// own undo step, and Refresh forgets the list. +#[test] +fn connections_tree_browses_and_adds() { + use std::io::{Read, Write}; + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + std::thread::spawn(move || { + let body = format!( + r#"Localimage/pngEPSG:3857aAbB"# + ); + for stream in listener.incoming().take(1) { + let Ok(mut s) = stream else { continue }; + let mut buf = [0u8; 2048]; + let _ = s.read(&mut buf); + let _ = write!(s, "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()); + let _ = s.write_all(body.as_bytes()); + } + }); + // Frames 0.05 s apart, so two clicks fall within egui's double-click delay. + let mut h = egui_kittest::Harness::builder().with_size(egui::vec2(1400.0, 860.0)).with_max_steps(64).with_step_dt(0.05).build_ui_state( + |ui, app: &mut RustGisApp| { + let ctx = ui.ctx().clone(); + app.logic(&ctx); + app.ui(ui); + }, + RustGisApp::with_sample(), + ); + h.state_mut().run("connection.new", json!({"kind":"wms","address":format!("http://{addr}/wms"),"name":"Local"})).unwrap(); + h.state_mut().run("ui.pane.sources", json!({})).unwrap(); + h.step(); + assert!(h.query_by_label("A").is_none(), "nothing is listed before the connection is opened"); + h.get_by_label("Local").click(); + for _ in 0..100 { + h.step(); + if h.query_by_label("A").is_some() { + break; + } + std::thread::sleep(std::time::Duration::from_millis(20)); + } + assert!(h.query_by_label("A").is_some(), "the layers are listed under the connection"); + let before = h.state().session.map().tiles.len(); + h.get_by_label("A").click(); + h.step(); + h.get_by_label("B").click(); + h.step(); + assert_eq!(h.state().ui.connection_tree.picks.len(), 2); + // The selection is added from a dataset's menu: each layer is its own undo step. + h.get_by_label("B").click_secondary(); + h.step(); + h.step(); + h.get_by_label("Add Selected (2)").click(); + h.step(); + h.step(); + assert_eq!(h.state().session.map().tiles.len(), before + 2); + assert!(h.state().ui.connection_tree.picks.is_empty()); + h.state_mut().run("edit.undo", json!(null)).unwrap(); + // Each layer was its own step: one undo takes one layer. + assert_eq!(h.state().session.map().tiles.len(), before + 1); + h.state_mut().run("edit.undo", json!(null)).unwrap(); + assert_eq!(h.state().session.map().tiles.len(), before); + // A double-click adds the one dataset, leaving nothing selected. + // (egui counts clicks in a row: let the earlier ones age out of its triple-click window first.) + for _ in 0..14 { + h.step(); + } + h.get_by_label("A").click(); + h.step(); + h.get_by_label("A").click(); + h.step(); + h.step(); + assert_eq!(h.state().session.map().tiles.len(), before + 1); + assert!(h.state().ui.connection_tree.picks.is_empty()); + h.state_mut().run("edit.undo", json!(null)).unwrap(); + assert_eq!(h.state().session.map().tiles.len(), before); + h.get_by_label("Local").click_secondary(); + h.step(); + h.get_by_label("Refresh").click(); + h.step(); + h.step(); + assert!(h.query_by_label("A").is_none(), "refreshing forgets the list until it is read again"); +} + /// Save in the web app (issue #197): the browser has no save dialog, so the first Save and every /// Save As ask for the file name, and Save then downloads under it; an opened file keeps its name. #[test] diff --git a/docs/user-guide/data.md b/docs/user-guide/data.md index c7fc11601..30e51d982 100644 --- a/docs/user-guide/data.md +++ b/docs/user-guide/data.md @@ -334,9 +334,16 @@ places data comes from: tables, Zarr variables, STAC assets, sample data), newest first. - **Folders**: the folders connected to the Browser (**Add Folder…**; desktop). - **Connections**: named services, databases, STAC APIs and S3 endpoints, organised by optional - groups. **New Connection ▾** opens a kind-specific form; right-click a row to edit, test, - duplicate, copy its address or remove it. A click opens the existing service, database, - STAC or S3 add dialog, connected or prefilled. + groups. **New Connection ▾** opens a kind-specific form. Click a connection to open it: its + datasets are listed under it, a level at a time as you open its folders, with **Refresh** + (a connection's or a folder's right-click menu; **⟳** refreshes every connection) and an error + with **Retry** (and **Log In…** for databases and keyed S3) when it can't be read. Click a + dataset to select it (click again to clear it); double-click, or right-click › **Add**, adds + one, and right-click › **Add Selected (N)** adds them all (**Clear Selection** empties the + selection). Each dataset is added by one command, so each is one undo step. On a table, a + service layer or a STAC collection, **Add With Options…** opens the database, service or STAC + add dialog on it; on a connection, it opens the kind's add dialog. Right-click a connection to + edit, test, duplicate, copy its address or remove it. - **Library**: well-known public sources: OpenStreetMap tiles, EOX Sentinel-2 cloudless and NASA GIBS (WMTS), USGS imagery, topographic and hydrography maps (ArcGIS), GEBCO bathymetry (WMS), swisstopo's maps and imagery (geo.admin.ch, WMTS), Earth Search and Planetary Computer (STAC), and the Natural Earth sample layers. @@ -389,6 +396,19 @@ databases). For non-database kinds, pass that response text as `answer` with the to parse it without a network request. Database kinds ignore `answer` and test through their driver. An anonymous S3 test needs at least one named bucket. +`connection.browse` lists what a connection offers, a level at a time: the tables of a database +(under their schemas), the layers of a service, the records of a catalogue with the services and +files they link to, the collections of a STAC API and the items and assets of a collection (for a +static STAC catalog, its catalogs, collections and items, each item opened for its assets), or +the buckets, folders and files of an S3 connection. Each child has a `key`, a `name` and what it +is (`what`); a child with a `node` is browsed by asking again with that `node`; one with `add` +names the command and parameters that add it (`service.add`, `layer.addDatabase`, +`stac.addAsset`, `layer.addUrl`). A level lists at most 500 children and says `truncated` when +it left some out. Like `connection.test`, it reads the address itself on the desktop and the +command line; with `probe: true` it returns the address to fetch, and the app passes what it +fetched back as `answer` (and further `documents` when a service asks for them, as `next`). +Browsing never changes the project or the saved connections. + Addresses are saved without passwords or tokens. `connection.login` remembers a database `password`, or S3 `accessKeyId`, `secretAccessKey` and optional `sessionToken`, for this session only, using the same credential stores as the existing add commands. With login parameters, @@ -400,7 +420,7 @@ connection also affects a session entry of the same name created there. Renaming clears the runtime entries under both the old and new names, so the renamed connection cannot inherit an unrelated login at its destination name. Web-service logins are not supported. Plugins may list connections but cannot manage, log in -to or test them. The in-app assistant asks for approval each time for those operations; +to, test or browse them. The in-app assistant asks for approval each time for those operations; external assistants using MCP cannot run them. Database passwords are shared by public DSN for the session, including existing layers and other named connections using that DSN. Editing or removing a named connection does not revoke