From b29700445aacb3c0ac28691b95a54e8ba06b857c Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Thu, 22 Oct 2026 19:00:56 +0200 Subject: [PATCH 1/9] feat(connections): browse and add datasets from the Connections tree (#699) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Expanding a saved connection in Data Sources › Connections lists its datasets a level at a time, with Refresh, an inline error and Retry/Log In…. Click selects, double-click adds, Add Selected adds each as its own undo step, Add With Options… opens the kind's dialog. New engine command connection.browse (probe/answer, like connection.test) for all 13 kinds, so the CLI and the assistant can browse too; user-only for plugins. --- crates/engine/src/cmd/connection_browse.rs | 476 +++++++++++++++++++ crates/engine/src/cmd/connections.rs | 17 +- crates/engine/src/cmd/layer.rs | 19 + crates/engine/src/cmd/plugins.rs | 1 + crates/engine/src/tests.rs | 137 +++++- crates/ui-egui/src/connectiontree.rs | 514 +++++++++++++++++++++ crates/ui-egui/src/dialogs.rs | 63 +-- crates/ui-egui/src/lib.rs | 131 +++--- crates/ui-egui/src/serviceui.rs | 68 +++ crates/ui-egui/src/sourcespane.rs | 138 +++--- crates/ui-egui/tests/smoke.rs | 59 +++ docs/user-guide/data.md | 25 +- 12 files changed, 1473 insertions(+), 175 deletions(-) create mode 100644 crates/engine/src/cmd/connection_browse.rs create mode 100644 crates/ui-egui/src/connectiontree.rs diff --git a/crates/engine/src/cmd/connection_browse.rs b/crates/engine/src/cmd/connection_browse.rs new file mode 100644 index 000000000..538b1012d --- /dev/null +++ b/crates/engine/src/cmd/connection_browse.rs @@ -0,0 +1,476 @@ +//! `connection.browse`: the datasets a saved connection offers, one level at a time, each with +//! the command that adds it. +//! +//! The app reads over the network in the background, so browsing is a probe/answer pair like +//! `connection.test`: `probe` names the address a level reads, the app fetches it and asks again +//! with the `answer`. Without an answer, the command line and the assistant read it themselves. + +use serde_json::{Value, json}; +use std::borrow::Cow; + +use super::super::{b, ps}; +use crate::connections::{Connection, DESKTOP_ONLY}; +use crate::render::rustgis_tiles::{address, s3, stac}; +use crate::{CmdResult, Session}; + +/// The most children of one level that are listed. +const MAX_CHILDREN: usize = 500; +/// The items of a STAC collection that are listed. +const ITEMS: usize = 20; +/// The longest node a caller may give. +const MAX_NODE: usize = 4096; + +/// What a node of a connection stands for. +enum Level { + Database, + Service { url: String, kind: Option }, + Catalogue, + StacRoot { api: String }, + StacCollection { api: String, id: String }, + StacCatalog { url: String }, + S3Root, + S3Prefix { bucket: String, prefix: String }, +} + +/// The types a `service::` node names (`auto` leaves the type to be guessed). +const SERVICE_TYPES: [&str; 7] = ["auto", "wms", "wmts", "wfs", "wcs", "ogcapi", "arcgis"]; + +/// The level a connection's `node` names (its top level when empty). +fn level(c: &Connection, node: &str) -> Result { + let kind = c.kind.as_str(); + let not_a_node = || format!("'{node}' isn't a node of {}", c.name); + if node.is_empty() { + return Ok(match kind { + "postgres" | "mysql" | "sqlserver" => { + if cfg!(target_arch = "wasm32") { + return Err(DESKTOP_ONLY.into()); + } + Level::Database + } + "service" => Level::Service { url: c.address.clone(), kind: None }, + "wms" | "wmts" | "wfs" | "wcs" | "ogcapi" | "arcgis" => Level::Service { url: c.address.clone(), kind: Some(c.kind.clone()) }, + "csw" => Level::Catalogue, + "stac" => Level::StacRoot { api: stac::api(Some(&c.address))? }, + "s3" => Level::S3Root, + _ => return Err(format!("unknown kind '{}'", c.kind)), + }); + } + if let Some(rest) = node.strip_prefix("service:") { + if !matches!(kind, "service" | "wms" | "wmts" | "wfs" | "wcs" | "ogcapi" | "arcgis" | "csw") { + return Err(not_a_node()); + } + let (t, url) = rest.split_once(':').filter(|(t, _)| SERVICE_TYPES.contains(t)).ok_or_else(not_a_node)?; + // Judged as a document of the connection would be: a link may not lead into the local network. + address::check(url, Some(&c.address))?; + return Ok(Level::Service { url: url.to_string(), kind: (t != "auto").then(|| t.to_string()) }); + } + if let Some(id) = node.strip_prefix("collection:") { + if kind != "stac" || id.is_empty() { + return Err(not_a_node()); + } + let api = stac::api(Some(&c.address))?; + if stac::catalog::is_static(&api) { + return Err(not_a_node()); + } + return Ok(Level::StacCollection { api, id: id.to_string() }); + } + if let Some(url) = node.strip_prefix("catalog:") { + if kind != "stac" { + return Err(not_a_node()); + } + address::check(url, Some(&c.address))?; + return Ok(Level::StacCatalog { url: url.to_string() }); + } + if let Some(rest) = node.strip_prefix("s3:") { + let (bucket, prefix) = rest + .split_once('/') + .filter(|(bucket, prefix)| kind == "s3" && s3::valid_bucket(bucket) && prefix.len() <= 1024) + .ok_or_else(not_a_node)?; + return Ok(Level::S3Prefix { bucket: bucket.to_string(), prefix: prefix.to_string() }); + } + Err(not_a_node()) +} + +/// The session's S3 connection for an S3 level, logging in with what needs no typing. +fn s3_login(s: &mut Session, c: &Connection, level: &Level) -> Result, String> { + if !matches!(level, Level::S3Root | Level::S3Prefix { .. }) { + return Ok(None); + } + if s3::connection(&c.name).is_none() && c.options.get("credentials").is_none_or(|v| v != "keys") { + super::login(s, c, &Value::Null)?; + } + let conn = s3::connection(&c.name).ok_or("log in first: connection.login {id, accessKeyId, secretAccessKey}")?; + if let Level::S3Prefix { bucket, .. } = level { + s3::grant(&conn.name, bucket); + } + Ok(Some(conn)) +} + +/// The address a level reads (`None`: nothing is fetched, or the database driver reads it). +fn fetch_url(s: &Session, c: &Connection, level: &Level, conn: Option<&s3::Connection>) -> Result, String> { + Ok(match level { + Level::Database => None, + Level::Service { url, kind } => Some(rustgis_data::ogc::capabilities_url(url, kind.as_deref())), + Level::Catalogue => Some(super::super::service::records_request(s, &json!({"url": c.address}))?), + Level::StacRoot { api } => Some(if stac::catalog::is_static(api) { api.clone() } else { stac::collections_url(api) }), + Level::StacCollection { api, id } => { + Some(stac::search_url(api, &stac::Search { collections: vec![id.clone()], limit: ITEMS, ..stac::Search::default() })?) + } + Level::StacCatalog { url } => Some(url.clone()), + Level::S3Root => match (super::first_bucket(c), conn) { + (Some(_), _) => None, + (None, Some(conn)) if conn.credentials.is_some() => Some(s3::buckets_url(conn)), + _ => return Err("name a bucket under Buckets to browse an anonymous S3 connection".into()), + }, + Level::S3Prefix { bucket, prefix } => Some(s3::list_url(bucket, prefix, None, conn)), + }) +} + +/// What a level reads, fetched here (waiting for it). +#[cfg(not(target_arch = "wasm32"))] +fn download(url: &str) -> Result { + let bytes = crate::render::rustgis_tiles::cache::download_blocking(url)?; + Ok(String::from_utf8(bytes).unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned())) +} + +#[cfg(target_arch = "wasm32")] +fn download(_: &str) -> Result { + Err("in the web app, expand the connection under Data Sources › Connections".into()) +} + +/// The children of a level, and what is known of the rest. +#[derive(Default)] +struct Found { + children: Vec, + truncated: bool, + matched: Option, +} + +impl Found { + /// Keeps the first [`MAX_CHILDREN`] of a list (collected with one more, to know it was cut). + fn new(mut children: Vec) -> Found { + let truncated = children.len() > MAX_CHILDREN; + children.truncate(MAX_CHILDREN); + Found { children, truncated, matched: None } + } +} + +/// A node that fits what [`browse`] accepts. +fn node_ok(node: &str) -> bool { + node.len() <= MAX_NODE && !node.chars().any(char::is_control) +} + +pub(super) fn browse(s: &mut Session, p: &Value) -> CmdResult { + let c = super::found(s, p)?.clone(); + let node = ps(p, "node").unwrap_or(""); + if !node_ok(node) { + return Err("node is a child's node, as connection.browse lists it".into()); + } + let level = level(&c, node)?; + let conn = s3_login(s, &c, &level)?; + let url = fetch_url(s, &c, &level, conn.as_ref())?; + let mut out = json!({"connection": c.id, "node": node}); + if b(p, "probe").unwrap_or(false) { + out["fetch"] = json!(url); + return Ok(out); + } + let found = match &level { + Level::Database => database(&c)?, + Level::Service { url, kind } => service(s, p, &mut out, url, kind.as_deref())?, + Level::S3Root if url.is_none() => { + buckets(c.options.get("buckets").map(String::as_str).unwrap_or("").split(',').map(str::to_string)) + } + _ => { + let at = url.as_deref().unwrap_or(""); + let text: Cow = match ps(p, "answer") { + Some(a) => Cow::Borrowed(a), + None => Cow::Owned(download(at)?), + }; + let bytes = text.as_bytes(); + match &level { + Level::Catalogue => catalogue(s, &c, &text, &mut out)?, + Level::StacRoot { api } if stac::catalog::is_static(api) => catalogs(bytes, api)?, + Level::StacRoot { .. } => collections(bytes)?, + Level::StacCatalog { url } => catalogs(bytes, url)?, + Level::StacCollection { .. } => items(bytes, at)?, + Level::S3Root => buckets(s3::parse_buckets(bytes)?.into_iter()), + Level::S3Prefix { bucket, prefix } => objects(bytes, bucket, prefix)?, + Level::Database | Level::Service { .. } => Found::default(), + } + } + }; + out["children"] = json!(found.children); + if found.truncated { + out["truncated"] = json!(true); + } + if let Some(m) = found.matched { + out["matched"] = json!(m); + } + Ok(out) +} + +/// A live database's geometry tables, under their schemas. +fn database(c: &Connection) -> Result { + #[cfg(target_arch = "wasm32")] + { + let _ = c; + Err(DESKTOP_ONLY.into()) + } + #[cfg(not(target_arch = "wasm32"))] + { + use std::collections::BTreeMap; + + let tables = super::super::layer::database_table_details(&json!({"database": c.address}))?; + let text = |t: &Value, k: &str| t.get(k).and_then(Value::as_str).unwrap_or("").to_string(); + let mut schemas: BTreeMap> = BTreeMap::new(); + let truncated = tables.len() > MAX_CHILDREN; + for t in tables.iter().take(MAX_CHILDREN) { + let (label, name) = (text(t, "label"), text(t, "name")); + let shown = if label.contains(':') { format!("{name}:{}", text(t, "column")) } else { name }; + schemas.entry(text(t, "schema")).or_default().push(json!({ + "key": label, + "name": shown, + "what": "table", + "detail": text(t, "kind"), + "add": {"command": "layer.addDatabase", "params": {"database": c.address, "layer": label}}, + "options": true, + })); + } + let children = schemas + .into_iter() + .map(|(s, tables)| json!({"key": format!("schema:{s}"), "name": s, "what": "schema", "children": tables})) + .collect(); + Ok(Found { children, truncated, matched: None }) + } +} + +/// A service's layers (capabilities read here, or the app's `answer` and `documents`). +fn service(s: &mut Session, p: &Value, out: &mut Value, url: &str, kind: Option<&str>) -> Result { + let mut q = json!({"url": url}); + if let Some(k) = kind { + q["type"] = json!(k); + } + if let Some(a) = ps(p, "answer") { + q["text"] = json!(a); + if let Some(d) = p.get("documents") { + q["documents"] = d.clone(); + } + } + let r = super::super::service::layers(s, &q)?; + out["service"] = json!({"url": url, "type": kind.unwrap_or("auto")}); + if let Some(n) = r.get("notes").filter(|n| n.as_array().is_some_and(|a| !a.is_empty())) { + out["notes"] = n.clone(); + } + if let Some(next) = r.get("next").filter(|n| !n.is_null()) { + out["next"] = next.clone(); + } + // The dialog imports an OGC API's collections together: a features layer isn't added alone there. + let grouped = r.get("kind").and_then(Value::as_str).is_some_and(|k| k.starts_with("OGC API")); + let text = |l: &Value, k: &str| l.get(k).and_then(Value::as_str).unwrap_or("").to_string(); + let children = r + .get("layers") + .and_then(Value::as_array) + .into_iter() + .flatten() + .take(MAX_CHILDREN + 1) + .map(|l| { + let (name, title) = (text(l, "name"), text(l, "title")); + let offer = l.get("offer").cloned().unwrap_or(Value::Null); + let offer_type = offer.get("type").and_then(Value::as_str).unwrap_or(""); + if offer_type == "service" { + let node = format!("service:arcgis:{}", text(&offer, "url")); + let what = if name.ends_with('/') { "folder" } else { "service" }; + return json!({"key": node, "node": node, "name": title.trim_end_matches('/'), "what": what}); + } + let what = match offer_type { + "vectorTiles" => "vectorTiles", + "features" => "features", + "coverage" => "coverage", + "style" => "style", + _ => "map", + }; + let source = text(l, "source_url"); + let mut params = json!({"url": url, "layer": name, "source_url": source}); + if let Some(k) = kind { + params["type"] = json!(k); + } + let mut child = json!({ + "key": format!("{source}#{name}"), + "name": if title.is_empty() { name.as_str() } else { title.as_str() }, + "what": what, + "detail": name, + "layer": l, + "options": true, + }); + if !(grouped && what == "features") { + child["add"] = json!({"command": "service.add", "params": params}); + } + child + }) + .collect(); + Ok(Found::new(children)) +} + +/// A catalogue's records, each with the services and files it links to. +fn catalogue(s: &mut Session, c: &Connection, text: &str, out: &mut Value) -> Result { + let r = super::super::service::catalog_search(s, &json!({"url": c.address, "text": text}))?; + if let Some(n) = r.get("notes") { + out["notes"] = n.clone(); + } + let str_of = |v: &Value, k: &str| v.get(k).and_then(Value::as_str).unwrap_or("").to_string(); + let mut dropped = false; + let children = r + .get("records") + .and_then(Value::as_array) + .into_iter() + .flatten() + .take(MAX_CHILDREN + 1) + .enumerate() + .map(|(i, rec)| { + let id = str_of(rec, "id"); + let title = str_of(rec, "title"); + let links: Vec = rec + .get("links") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|l| { + let (url, kind, name) = (str_of(l, "url"), str_of(l, "kind"), str_of(l, "name")); + match kind.as_str() { + "wms" | "wmts" | "wfs" | "wcs" | "ogcapi" | "arcgis" => { + let node = format!("service:{kind}:{url}"); + if !node_ok(&node) { + dropped = true; + return None; + } + Some(json!({"key": node, "node": node, "name": if name.is_empty() { url.as_str() } else { name.as_str() }, "what": "service", "detail": kind})) + } + "download" => { + let last = url.split(['?', '#']).next().unwrap_or("").trim_end_matches('/').rsplit('/').next().unwrap_or(""); + Some(json!({ + "key": url, + "name": if name.is_empty() { last } else { name.as_str() }, + "what": "download", + "add": {"command": "layer.addUrl", "params": {"url": url, "name": title}}, + })) + } + _ => None, + } + }) + .collect(); + let summary: String = str_of(rec, "abstract").chars().take(200).collect(); + json!({"key": format!("record:{}", if id.is_empty() { i.to_string() } else { id }), "name": title, "what": "record", "detail": summary, "children": links}) + }) + .collect(); + let mut found = Found::new(children); + found.matched = r.get("matched").and_then(Value::as_u64); + found.truncated |= dropped || r.get("next").is_some_and(|n| !n.is_null()); + Ok(found) +} + +/// A STAC API's collections. +fn collections(bytes: &[u8]) -> Result { + let list = stac::parse_collections(bytes)?; + Ok(Found::new( + list.into_iter() + .take(MAX_CHILDREN + 1) + .map(|c| { + let node = format!("collection:{}", c.id); + json!({"key": node, "node": node, "name": if c.title.is_empty() { c.id.as_str() } else { c.title.as_str() }, "what": "collection", "detail": c.id, "options": true}) + }) + .collect(), + )) +} + +/// The catalogs and collections below a static catalog's document fetched from `from`. +fn catalogs(bytes: &[u8], from: &str) -> Result { + let list = stac::catalog::parse_children(bytes, from)?; + let mut dropped = false; + let children = list + .into_iter() + .take(MAX_CHILDREN + 1) + .filter_map(|c| { + let node = format!("catalog:{}", c.url); + let ok = node_ok(&node); + dropped |= !ok; + ok.then(|| json!({"key": node, "node": node, "name": c.title, "what": "catalog"})) + }) + .collect(); + let mut found = Found::new(children); + found.truncated |= dropped; + Ok(found) +} + +/// The items of a collection's search, each with the assets that can be added. +fn items(bytes: &[u8], from: &str) -> Result { + let (list, next) = stac::parse_items(bytes, from)?; + let matched = stac::parse_matched(bytes); + let truncated = next.is_some() || matched.is_some_and(|m| m > list.len() as u64); + let children = list + .iter() + .take(MAX_CHILDREN) + .map(|item| { + let assets: Vec = item + .assets + .iter() + .take(MAX_CHILDREN) + .filter_map(|a| { + let (key, href) = item.asset_href(Some(&a.key)).ok()?; + let mut params = json!({"asset": key, "name": format!("{} {key}", item.id)}); + match &item.url { + Some(u) => params["url"] = json!(u), + None => params["item"] = stac::item_json_with(item, false), + } + Some(json!({ + "key": href, + "name": if a.title.is_empty() { key.as_str() } else { a.title.as_str() }, + "what": "asset", + "detail": a.media_type, + "href": href, + "add": {"command": "stac.addAsset", "params": params}, + })) + }) + .collect(); + json!({"key": format!("item:{}", item.id), "name": item.id, "what": "item", "detail": item.datetime, "children": assets}) + }) + .collect(); + Ok(Found { children, truncated, matched }) +} + +/// Buckets to browse, each a node. +fn buckets(names: impl Iterator) -> Found { + Found::new( + names + .map(|n| n.trim().to_string()) + .filter(|n| s3::valid_bucket(n)) + .take(MAX_CHILDREN + 1) + .map(|n| { + let node = format!("s3:{n}/"); + json!({"key": node, "node": node, "name": n, "what": "bucket"}) + }) + .collect(), + ) +} + +/// One level of a bucket: its folders (each a node) and its files. +fn objects(bytes: &[u8], bucket: &str, prefix: &str) -> Result { + let list = s3::parse_list(bytes)?; + let last = |k: &str| k.trim_end_matches('/').rsplit('/').next().unwrap_or("").to_string(); + let folders = list.prefixes.iter().map(|p| { + let node = format!("s3:{bucket}/{p}"); + json!({"key": node, "node": node, "name": format!("{}/", last(p)), "what": "prefix"}) + }); + let files = list.objects.iter().filter(|o| o.key != prefix && !o.key.ends_with('/')).map(|o| { + let url = format!("s3://{bucket}/{}", o.key); + json!({ + "key": url, + "name": last(&o.key), + "what": "object", + "detail": o.size.to_string(), + "add": {"command": "layer.addUrl", "params": {"url": url}}, + }) + }); + let mut found = Found::new(folders.chain(files).take(MAX_CHILDREN + 1).collect()); + found.truncated |= list.next.is_some(); + Ok(found) +} diff --git a/crates/engine/src/cmd/connections.rs b/crates/engine/src/cmd/connections.rs index 833ffc6d4..f2d7765f7 100644 --- a/crates/engine/src/cmd/connections.rs +++ b/crates/engine/src/cmd/connections.rs @@ -10,6 +10,9 @@ use crate::connections::{Connection, DATABASE_KINDS, DESKTOP_ONLY, KINDS}; use crate::render::rustgis_tiles::{s3, stac}; use crate::{CmdResult, CommandSpec as C, Session}; +#[path = "connection_browse.rs"] +mod browse; + pub fn commands() -> Vec { vec![ C::new("connection.list", "List Connections", "Home › Add › Data Sources", list).view().params( @@ -29,6 +32,9 @@ pub fn commands() -> Vec { C::new("connection.test", "Test Connection", "Home › Add › Data Sources", test).view().params( r#"{"id"?: "a connection's id or name", "kind"?: "postgres"|"mysql"|"sqlserver"|"service"|"wms"|"wmts"|"wfs"|"wcs"|"ogcapi"|"arcgis"|"csw"|"stac"|"s3", "name"?: text, "group"?: text, "address"?: text, "options"?: "an unsaved connection, as connection.new (when no id)", "password"?: text, "accessKeyId"?: text, "secretAccessKey"?: text, "sessionToken"?: "as connection.login (logs in first)", "probe"?: "true: only the address the test reads, {fetch} (null for databases, which their driver tests)", "answer"?: "non-database kinds only: what that address returned, when the app fetched it (no network request then); databases ignore answer and test through their driver"}"#, ), + C::new("connection.browse", "Browse Connection", "Home › Add › Data Sources", browse::browse).view().params( + r#"{"id": "a connection's id or name", "node"?: "a child's node, as a listing gives it (the connection's top level when absent)", "probe"?: "true: only the address this level reads, {fetch} (null: read without a fetch, or by the database driver)", "answer"?: "what that address returned, when the app fetched it", "documents"?: [{"url": "a further document a listing asked for (next)", "text": "its answer"}]} → {connection, node, children: [{key, name, what, detail?, node? (browse it), children? (listed already), add? {command, params}: the command that adds it}], truncated?, matched?, notes?, service? {url, type}, next? (fetch it, then ask again with answer and documents)}"#, + ), ] } @@ -53,16 +59,7 @@ fn shown(c: &Connection) -> Value { } else { false }; - let add_with = match c.kind.as_str() { - "postgres" | "mysql" | "sqlserver" => "info.databaseTables {database: address}, then layer.addDatabase {database: address, layer}", - "service" => "service.layers {url: address}, then service.add {url, layer}", - "wms" | "wmts" | "wfs" | "ogcapi" | "arcgis" => "service.layers {url: address, type: kind}, then service.add {url, type, layer}", - "wcs" => "service.layers {url: address, type: wcs}, then service.coverage {url, layer}", - "csw" => "catalog.search {url: address, query}", - "stac" => "stac.collections {url: address}, then stac.search {url, collections}", - "s3" => "connection.login {id} once, then s3.list {connection: name}, then layer.addUrl {url: s3://…}", - _ => "", - }; + let add_with = "connection.browse {id}, then {id, node} for a child's node; run a dataset's add {command, params}"; let mut out = serde_json::to_value(c).unwrap_or(Value::Null); out["available"] = json!(available); if !available { diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index d8b671a8a..9d7246ab2 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -1280,6 +1280,15 @@ pub(super) fn database_tables(_s: &mut Session, p: &Value) -> CmdResult { } } +/// The geometry tables of a live database server, with the schema, name, column and type of +/// each (`connection.browse` groups them by schema). +#[cfg(not(target_arch = "wasm32"))] +pub(super) fn database_table_details(p: &Value) -> Result, String> { + let db = database_param(p).ok_or("database is required")?; + let live = LiveDb::connect(&db).ok_or("not a database server")?.map_err(|e| format!("{}: {e}", db.short()))?; + Ok(live.tables().map_err(|e| format!("{}: {e}", db.short()))?.iter().map(LiveTable::detail).collect()) +} + /// Adds the tables of a database (a PostgreSQL connection or a DuckDB file) that have a geometry /// column, or the one `layer` names. fn add_database(s: &mut Session, p: &Value) -> CmdResult { @@ -1399,6 +1408,16 @@ impl LiveTable { LiveTable::Sqlserver(t) => t.label(), } } + + /// The table as a browser lists it: its label for `layer.addDatabase`, and what it is made of. + fn detail(&self) -> Value { + let (schema, name, column, kind) = match self { + LiveTable::Postgis(t) => (&t.schema, &t.name, &t.column, &t.kind), + LiveTable::Mysql(t) => (&t.schema, &t.name, &t.column, &t.kind), + LiveTable::Sqlserver(t) => (&t.schema, &t.name, &t.column, &t.kind), + }; + json!({"label": self.label(), "schema": schema, "name": name, "column": column, "kind": kind}) + } } #[cfg(not(target_arch = "wasm32"))] diff --git a/crates/engine/src/cmd/plugins.rs b/crates/engine/src/cmd/plugins.rs index f71652219..bc00ee10b 100644 --- a/crates/engine/src/cmd/plugins.rs +++ b/crates/engine/src/cmd/plugins.rs @@ -276,6 +276,7 @@ const NEVER: &[&str] = &[ "connection.remove", "connection.login", "connection.test", + "connection.browse", "s3.connect", "s3.disconnect", "earthdata.login", diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 11bd4e55a..cce8f26f7 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -1148,6 +1148,7 @@ fn hostile_params_never_panic() { json!({"id": "", "kind": "postgres", "address": "postgres:host=127.0.0.1 port=9 password=x", "options": "x"}), json!({"kind": "wms", "address": "http://127.0.0.1:9/wms", "probe": "x", "answer": 7}), json!({"kind": "stac", "address": "http://127.0.0.1:9/stac", "answer": "\u{0}{"}), + json!({"id": "c1", "node": "s3:/\u{7}", "probe": true, "answer": 7, "documents": "x"}), ]; let ids: Vec<&str> = Session::new(Project::default()).registry.commands.iter().map(|c| c.id).collect(); let streamed = grid_fgb("hostile", 20); @@ -8856,7 +8857,15 @@ fn connections_commands() { assert_eq!(renamed["name"], "wms (2)"); assert_ne!(default["id"], renamed["id"]); assert!(crate::sources::holds_secret(&json!({"id":"x","password":"p"}))); - for id in ["connection.new", "connection.edit", "connection.duplicate", "connection.remove", "connection.login", "connection.test"] { + for id in [ + "connection.new", + "connection.edit", + "connection.duplicate", + "connection.remove", + "connection.login", + "connection.test", + "connection.browse", + ] { assert!(crate::cmd::plugins::user_only(id), "{id}"); } assert!(!crate::cmd::plugins::user_only("connection.list")); @@ -8967,6 +8976,132 @@ fn connection_test_probes_and_answers() { assert!(s.connections.connections.is_empty()); } +/// Browsing a saved connection lists what each kind offers, a level at a time, with the command +/// that adds each dataset; offline, with the answers supplied. A node that isn't the +/// connection's is refused, and nothing browsed is saved or undoable. +#[test] +fn connection_browse_levels_and_adds() { + use crate::render::rustgis_tiles::s3; + let mut s = Session::new(Project::default()); + let new = |s: &mut Session, p: Value| s.run("connection.new", &p).unwrap(); + let wms_caps = r#"Localimage/pngEPSG:3857aAbB"#; + + // WMS: the address read, its layers, and the add that adds one. + new(&mut s, json!({"kind": "wms", "address": "https://a.example/wms", "name": "W"})); + let v = s.run("connection.browse", &json!({"id": "W", "probe": true})).unwrap(); + assert!(v["fetch"].as_str().is_some_and(|f| f.contains("REQUEST=GetCapabilities")), "{v}"); + let v = s.run("connection.browse", &json!({"id": "W", "answer": wms_caps})).unwrap(); + assert_eq!(v["service"], json!({"url": "https://a.example/wms", "type": "wms"})); + let first = v["children"][0].clone(); + assert_eq!(first["what"], "map"); + assert_eq!(first["name"], "A"); + assert_eq!(first["add"]["command"], "service.add"); + assert_eq!(first["add"]["params"]["url"], "https://a.example/wms"); + assert_eq!(first["add"]["params"]["type"], "wms"); + assert_eq!(first["add"]["params"]["layer"], "a"); + assert!(first["add"]["params"].get("source_url").is_some()); + assert_eq!(v["children"].as_array().unwrap().len(), 2); + let before = s.map().tiles.len(); + let mut add = first["add"]["params"].clone(); + add["text"] = json!(wms_caps); + s.run("service.add", &add).unwrap(); + assert_eq!(s.map().tiles.len(), before + 1); + s.run("edit.undo", &Value::Null).unwrap(); + assert_eq!(s.map().tiles.len(), before); + + // ArcGIS: a directory's folder is a node of its own address. + new(&mut s, json!({"kind": "arcgis", "address": "https://x.example.org/arcgis/rest/services", "name": "Arc"})); + let dir = r#"{"currentVersion": 11, "folders": ["Hydro"], "services": [{"name": "Topo", "type": "MapServer"}]}"#; + let v = s.run("connection.browse", &json!({"id": "Arc", "answer": dir})).unwrap(); + let folder = "service:arcgis:https://x.example.org/arcgis/rest/services/Hydro"; + let child = v["children"].as_array().unwrap().iter().find(|c| c["node"] == folder).unwrap_or_else(|| panic!("{v}")); + assert_eq!(child["what"], "folder"); + assert_eq!(child["name"], "Hydro"); + let v = s.run("connection.browse", &json!({"id": "Arc", "node": folder, "probe": true})).unwrap(); + assert!(v["fetch"].as_str().is_some_and(|f| f.starts_with("https://x.example.org/arcgis/rest/services/Hydro")), "{v}"); + + // CSW: a record, the service it links to as a node, and more records than listed. + new(&mut s, json!({"kind": "csw", "address": "https://cat.example.org/csw", "name": "Cat"})); + let records = r#" + Forests + https://geo.example.org/wmshttp://192.168.1.2/wfs + "#; + let v = s.run("connection.browse", &json!({"id": "Cat", "answer": records})).unwrap(); + assert_eq!(v["children"].as_array().unwrap().len(), 1, "{v}"); + assert_eq!(v["children"][0]["what"], "record"); + assert_eq!(v["children"][0]["children"][0]["node"], "service:wms:https://geo.example.org/wms"); + assert_eq!((v["matched"].as_u64(), v["truncated"].as_bool()), (Some(3), Some(true))); + assert!(v.get("next").is_none()); + + // STAC: collections, a collection's search, and an item's asset with the add of it. + new(&mut s, json!({"kind": "stac", "address": "https://s.example/v1", "name": "Stac"})); + let v = s.run("connection.browse", &json!({"id": "Stac", "probe": true})).unwrap(); + assert_eq!(v["fetch"], "https://s.example/v1/collections"); + let v = s.run("connection.browse", &json!({"id": "Stac", "answer": r#"{"collections":[{"id":"c1","title":"C"}]}"#})).unwrap(); + assert_eq!(v["children"][0]["node"], "collection:c1"); + assert_eq!(v["children"][0]["options"], true); + let v = s.run("connection.browse", &json!({"id": "Stac", "node": "collection:c1", "probe": true})).unwrap(); + let fetch = v["fetch"].as_str().unwrap(); + assert!(fetch.contains("/search?") && fetch.contains("collections=c1") && fetch.contains("limit=20"), "{fetch}"); + let self_link = "https://s.example/v1/collections/c1/items/i1"; + let items = json!({"type": "FeatureCollection", "numberMatched": 1, "features": [{ + "type": "Feature", "id": "i1", "collection": "c1", "bbox": [0, 0, 1, 1], "geometry": null, + "properties": {"datetime": "2024-01-01T00:00:00Z"}, + "assets": {"visual": {"href": "https://s.example/a.tif", "type": "image/tiff; application=geotiff"}}, + "links": [{"rel": "self", "href": self_link}]}]}); + let v = s.run("connection.browse", &json!({"id": "Stac", "node": "collection:c1", "answer": items.to_string()})).unwrap(); + assert_eq!(v["children"][0]["what"], "item"); + assert_eq!( + v["children"][0]["children"][0]["add"], + json!({"command": "stac.addAsset", "params": {"url": self_link, "asset": "visual", "name": "i1 visual"}}) + ); + assert!(v.get("truncated").is_none()); + + // S3: buckets from the connection's options need no fetch; a bucket's level is read. + new(&mut s, json!({"kind": "s3", "name": "BrowseB", "options": {"credentials": "none", "buckets": "bkt1"}})); + let v = s.run("connection.browse", &json!({"id": "BrowseB", "probe": true})).unwrap(); + assert!(v["fetch"].is_null(), "{v}"); + let v = s.run("connection.browse", &json!({"id": "BrowseB"})).unwrap(); + assert_eq!(v["children"][0]["node"], "s3:bkt1/"); + let v = s.run("connection.browse", &json!({"id": "BrowseB", "node": "s3:bkt1/", "probe": true})).unwrap(); + assert!(v["fetch"].as_str().is_some_and(|f| f.contains("bkt1")), "{v}"); + let listing = "p/p.tif5q/0"; + let v = s.run("connection.browse", &json!({"id": "BrowseB", "node": "s3:bkt1/", "answer": listing})).unwrap(); + assert_eq!(v["children"][0]["node"], "s3:bkt1/p/"); + assert_eq!(v["children"][0]["name"], "p/"); + assert_eq!(v["children"][1]["add"]["params"]["url"], "s3://bkt1/p.tif"); + assert_eq!(v["children"].as_array().unwrap().len(), 2, "{v}"); + s3::disconnect("BrowseB"); + + // Nodes that aren't the connection's, and ones that are hostile. + let long = "x".repeat(5000); + for (id, node) in [ + ("W", "collection:x"), + ("BrowseB", "s3:BAD_/"), + ("W", long.as_str()), + ("W", "service:ftp:x"), + ("W", "service:wms:http://127.0.0.1:9/wms"), + ("Stac", "catalog:http://127.0.0.1:9/c.json"), + ("Stac", "s3:bkt1/"), + ("W", "nope"), + ("W", "a\u{7}b"), + ] { + let e = s.run("connection.browse", &json!({"id": id, "node": node})).unwrap_err(); + assert!(!e.is_empty()); + } + assert!(s.run("connection.browse", &json!({"id": "missing"})).unwrap_err().contains("no connection")); + #[cfg(not(target_arch = "wasm32"))] + { + new(&mut s, json!({"kind": "postgres", "address": "postgres:host=127.0.0.1 port=9 dbname=x user=u", "name": "Pg"})); + assert!(!s.run("connection.browse", &json!({"id": "Pg"})).unwrap_err().is_empty()); + } + let saved = s.connections.connections.len(); + assert!(saved >= 5); + s.run("connection.browse", &json!({"id": "W", "answer": wms_caps})).unwrap(); + assert_eq!(s.connections.connections.len(), saved); + assert!(!s.can_undo()); +} + fn rustgis_engine_sources_roundtrip(text: &str) -> crate::sources::Sources { crate::sources::Sources::from_json(text) } diff --git a/crates/ui-egui/src/connectiontree.rs b/crates/ui-egui/src/connectiontree.rs new file mode 100644 index 000000000..a1c312f9b --- /dev/null +++ b/crates/ui-egui/src/connectiontree.rs @@ -0,0 +1,514 @@ +//! The datasets a saved connection offers, listed under it in Data Sources › Connections: one +//! level at a time as it is expanded (`connection.browse` names what each level reads, the app +//! fetches it off the frame and asks again with the answer), with the selection that +//! Add Selected adds. Each dataset added is its own undo step. + +use std::collections::HashMap; +use std::sync::Arc; + +use egui::collapsing_header::CollapsingState; +use egui::{Label, RichText, Sense, Ui}; +use rustgis_engine::connections::{Connection, DATABASE_KINDS}; +use rustgis_engine::render::rustgis_tiles::stac; +use rustgis_engine::sources::Source; +use serde_json::{Value, json}; + +use crate::serviceui::{self, ServiceSource}; +use crate::theme::Tokens; +use crate::{RustGisApp, Slot, i18n}; + +type Key = (String, String); + +#[derive(Default)] +pub struct State { + /// What each level of each connection lists: (connection id, node; empty for the top). + entries: HashMap, + /// The definition each connection's entries were made for: an edit makes them stale. + seen: HashMap, + pub picks: Vec, + /// Signed addresses of STAC assets waiting on a token: (token address, asset, name, answer). + tokens: Vec<(String, String, String, Slot, String>>)>, +} + +pub enum Entry { + Loading(Fetch), + Ready(Arc), + Failed(String), +} + +/// A level whose answer is on its way. +pub struct Fetch { + node: String, + /// The first answer, and the further documents a service's discovery asked for. + first: Option, + documents: Vec, + /// The address being fetched. + pending: String, + slot: Slot, String>>, +} + +pub struct Listing { + pub children: Vec, + pub truncated: bool, + pub matched: Option, + pub notes: Vec, + /// The service its layers belong to, to add them from the documents already read. + pub source: Option>, +} + +/// A dataset selected for Add Selected. +pub struct Pick { + pub conn: String, + pub key: String, + pub name: String, + pub child: Value, + pub source: Option>, +} + +fn params(id: &str, node: &str) -> Value { + let mut p = json!({"id": id}); + if !node.is_empty() { + p["node"] = json!(node); + } + p +} + +/// Starts reading a level: what it needs fetched is asked for, and a level read without a fetch +/// (a database's tables, an S3 connection's own buckets) is read now. Errors go into the entry. +fn start(app: &mut RustGisApp, c: &Connection, node: &str) { + let mut probe = params(&c.id, node); + probe["probe"] = json!(true); + let entry = match app.session.run("connection.browse", &probe) { + Err(e) => Entry::Failed(e), + Ok(v) => match v["fetch"].as_str() { + None => match app.session.run("connection.browse", ¶ms(&c.id, node)) { + Ok(v) => Entry::Ready(Arc::new(listing(&v, None, Vec::new()))), + Err(e) => Entry::Failed(e), + }, + Some(url) => Entry::Loading(Fetch { + node: node.to_string(), + first: None, + documents: Vec::new(), + pending: url.to_string(), + slot: crate::fetch_into(url), + }), + }, + }; + app.ui.connection_tree.entries.insert((c.id.clone(), node.to_string()), entry); +} + +fn listing(v: &Value, first: Option, documents: Vec) -> Listing { + let source = v.get("service").filter(|s| s.is_object()).map(|s| { + Arc::new(ServiceSource { + url: s["url"].as_str().unwrap_or("").to_string(), + kind: serviceui::kind_of(s["type"].as_str().unwrap_or("")), + text: first.unwrap_or_default(), + documents, + }) + }); + Listing { + children: v["children"].as_array().cloned().unwrap_or_default(), + truncated: v["truncated"] == true, + matched: v["matched"].as_u64(), + notes: v["notes"].as_array().into_iter().flatten().filter_map(Value::as_str).map(str::to_string).collect(), + source, + } +} + +/// An answer arrived: asks again with it (and what came before). A service's discovery may need +/// further documents, which are fetched in turn. +fn advance(app: &mut RustGisApp, id: &str, mut f: Fetch, bytes: Vec) -> Entry { + let Ok(text) = crate::service_text(bytes) else { + return Entry::Failed(tl!("The answer isn't UTF-8 text").to_string()); + }; + if f.first.is_none() { + f.first = Some(text); + } else { + f.documents.push(json!({"url": f.pending, "text": text})); + } + let mut p = params(id, &f.node); + p["answer"] = json!(f.first); + if !f.documents.is_empty() { + p["documents"] = json!(f.documents); + } + match app.session.run("connection.browse", &p) { + Err(e) => Entry::Failed(e), + Ok(v) => match v["next"].as_str() { + Some(next) => { + f.pending = next.to_string(); + f.slot = crate::fetch_into(next); + Entry::Loading(f) + } + None => Entry::Ready(Arc::new(listing(&v, f.first, f.documents))), + }, + } +} + +pub fn poll(app: &mut RustGisApp, ctx: &egui::Context) { + let tree = &mut app.ui.connection_tree; + let mut landed = Vec::new(); + for (key, entry) in &tree.entries { + if let Entry::Loading(f) = entry + && let Some(r) = f.slot.lock().unwrap_or_else(|e| e.into_inner()).take() + { + landed.push((key.clone(), r)); + } + } + for (key, r) in landed { + let Some(Entry::Loading(f)) = app.ui.connection_tree.entries.remove(&key) else { continue }; + let entry = match r { + Ok(bytes) => advance(app, &key.0, f, bytes), + Err(e) => Entry::Failed(e), + }; + app.ui.connection_tree.entries.insert(key, entry); + } + // STAC assets that needed a token: signed and added when it lands. + for (token_url, href, name, slot) in std::mem::take(&mut app.ui.connection_tree.tokens) { + let Some(r) = slot.lock().unwrap_or_else(|e| e.into_inner()).take() else { + app.ui.connection_tree.tokens.push((token_url, href, name, slot)); + continue; + }; + match r.and_then(|b| stac::parse_sas(&b)) { + Ok(token) => { + stac::remember_token(&token_url, &token); + if let Err(e) = app.run("ui.addUrl", json!({"url": stac::sign(&href, &token), "name": name, "zoom": true})) { + app.session.notify(e, true); + } + app.map.invalidate(); + } + Err(e) => app.session.notify(e, true), + } + } + let tree = &app.ui.connection_tree; + if tree.entries.values().any(|e| matches!(e, Entry::Loading(_))) || !tree.tokens.is_empty() || !app.ui.service_steps.is_empty() { + ctx.request_repaint_after(std::time::Duration::from_millis(100)); + } +} + +/// Adds one dataset a listing offers. Each branch is one command, so each is one undo step. +pub fn add(app: &mut RustGisApp, child: &Value, source: Option<&ServiceSource>) -> Result<(), String> { + let add = child.get("add").filter(|a| a.is_object()).ok_or_else(|| tl!("Nothing to add here").to_string())?; + let command = add["command"].as_str().unwrap_or(""); + let params = add["params"].clone(); + match (source, child.get("layer").filter(|l| l.is_object())) { + (Some(source), Some(layer)) => { + serviceui::add_layer(app, source, layer, "", false)?; + } + _ => match command { + "layer.addUrl" => { + app.run("ui.addUrl", params)?; + } + "stac.addAsset" => add_asset(app, child["href"].as_str().unwrap_or(""), params["name"].as_str().unwrap_or(""))?, + _ => { + app.run(command, params)?; + } + }, + } + app.map.invalidate(); + Ok(()) +} + +/// A STAC asset by its address, signed first when it needs a token (one in hand is used; one to +/// fetch is fetched off the frame and the asset added when it lands). +fn add_asset(app: &mut RustGisApp, href: &str, name: &str) -> Result<(), String> { + if !stac::needs_signing(href) { + app.run("ui.addUrl", json!({"url": href, "name": name, "zoom": true}))?; + return Ok(()); + } + let token_url = stac::sas_token_url(href)?; + if let Some(token) = stac::cached_token(&token_url) { + app.run("ui.addUrl", json!({"url": stac::sign(href, &token), "name": name, "zoom": true}))?; + } else { + let slot = crate::fetch_into(&token_url); + app.ui.connection_tree.tokens.push((token_url, href.to_string(), name.to_string(), slot)); + } + Ok(()) +} + +/// Opens the kind's own add dialog on a dataset: a table in the database dialog, a service layer +/// in the service dialog, a STAC collection in the STAC dialog. +fn add_with_options(app: &mut RustGisApp, c: &Connection, child: &Value, source: Option<&ServiceSource>) -> Result<(), String> { + match child["what"].as_str().unwrap_or("") { + "table" => crate::sourcespane::open( + app, + &Source { kind: "database".into(), name: c.name.clone(), address: c.address.clone(), extra: json!({"layer": child["key"]}) }, + ), + "collection" => { + crate::sourcespane::open( + app, + &Source { kind: "stac".into(), name: c.name.clone(), address: c.address.clone(), extra: json!({"api": "true"}) }, + )?; + app.ui.stac.collection = child["detail"].as_str().unwrap_or("").to_string(); + Ok(()) + } + _ => match (source, child.get("layer").filter(|l| l.is_object())) { + (Some(source), Some(layer)) => { + app.run("ui.addService", json!({"url": source.url, "type": source.kind}))?; + app.ui.service_browse.filter = layer["name"].as_str().unwrap_or("").to_string(); + Ok(()) + } + _ => Err(tl!("Nothing to add here").to_string()), + }, + } +} + +/// What a listed child is, in words. +fn what_label(what: &str) -> &'static str { + match what { + "schema" => tl!("schema"), + "table" => tl!("table"), + "folder" => tl!("folder"), + "service" => tl!("service"), + "map" => tl!("map"), + "vectorTiles" => tl!("vector tiles"), + "features" => tl!("features"), + "coverage" => tl!("coverage"), + "style" => tl!("style"), + "record" => tl!("record"), + "download" => tl!("download"), + "collection" => tl!("collection"), + "catalog" => tl!("catalog"), + "item" => tl!("item"), + "asset" => tl!("asset"), + "bucket" => tl!("bucket"), + "prefix" => tl!("folder"), + "object" => tl!("file"), + _ => "", + } +} + +fn picked(app: &RustGisApp, conn: &str, key: &str) -> bool { + app.ui.connection_tree.picks.iter().any(|p| p.conn == conn && p.key == key) +} + +fn toggle(app: &mut RustGisApp, c: &Connection, child: &Value, source: Option<&Arc>) { + let key = child["key"].as_str().unwrap_or(""); + let picks = &mut app.ui.connection_tree.picks; + if let Some(i) = picks.iter().position(|p| p.conn == c.id && p.key == key) { + picks.remove(i); + } else { + picks.push(Pick { + conn: c.id.clone(), + key: key.to_string(), + name: child["name"].as_str().unwrap_or("").to_string(), + child: child.clone(), + source: source.cloned(), + }); + } +} + +/// Forgets what a connection (or, with `None`, every connection) listed and what was selected +/// from it: the next expansion reads it again. +pub fn refresh(app: &mut RustGisApp, id: Option<&str>) { + let tree = &mut app.ui.connection_tree; + match id { + Some(id) => { + tree.entries.retain(|(c, _), _| c != id); + tree.picks.retain(|p| p.conn != id); + } + None => { + tree.entries.clear(); + tree.picks.clear(); + } + } +} + +/// Add Selected and Clear Selection, shown while datasets are selected. +pub fn picks_bar(app: &mut RustGisApp, ui: &mut Ui) { + // A pick made for a connection that was edited or removed since (even while its row is + // folded, when its list isn't refreshed) would add from the old definition: dropped. + let (connections, tree) = (&app.session.connections, &mut app.ui.connection_tree); + let seen = &tree.seen; + tree.picks.retain(|p| connections.find(&p.conn).is_some_and(|c| seen.get(&p.conn) == Some(c))); + if tree.picks.is_empty() { + return; + } + ui.horizontal(|ui| { + let n = app.ui.connection_tree.picks.len().to_string(); + if ui.button(i18n::fmt("Add Selected ({n})", &[("n", &n)])).clicked() { + for pick in std::mem::take(&mut app.ui.connection_tree.picks) { + // A connection removed since it was selected is skipped. + if app.session.connections.find(&pick.conn).is_none() { + continue; + } + if let Err(e) = add(app, &pick.child, pick.source.as_deref()) { + app.session.notify(e, true); + } + } + } + if ui.button(tl!("Clear Selection")).clicked() { + app.ui.connection_tree.picks.clear(); + } + }); +} + +/// A connection's expanded body: its top level, listed (or its error). +pub fn show(app: &mut RustGisApp, ui: &mut Ui, t: &Tokens, c: &Connection) { + // An edited connection lists what it now offers. + if app.ui.connection_tree.seen.get(&c.id) != Some(c) { + refresh(app, Some(&c.id)); + app.ui.connection_tree.seen.insert(c.id.clone(), c.clone()); + } + level(app, ui, t, c, ""); +} + +/// One level: loading, its error with the ways to go on, or its children. +fn level(app: &mut RustGisApp, ui: &mut Ui, t: &Tokens, c: &Connection, node: &str) { + let key = (c.id.clone(), node.to_string()); + let shown = match app.ui.connection_tree.entries.get(&key) { + None => None, + Some(Entry::Loading(_)) => Some(Err(None)), + Some(Entry::Failed(e)) => Some(Err(Some(e.clone()))), + Some(Entry::Ready(l)) => Some(Ok(l.clone())), + }; + match shown { + None => { + start(app, c, node); + ui.ctx().request_repaint(); + } + Some(Err(None)) => { + ui.horizontal(|ui| { + ui.spinner(); + ui.label(tl!("Loading…")); + }); + } + Some(Err(Some(e))) => { + ui.label(RichText::new(e).small().color(t.red)); + ui.horizontal(|ui| { + if ui.small_button(tl!("Retry")).clicked() { + app.ui.connection_tree.entries.remove(&key); + } + let keys = c.kind == "s3" && c.options.get("credentials").is_some_and(|v| v == "keys"); + if (DATABASE_KINDS.contains(&c.kind.as_str()) || keys) && ui.small_button(tl!("Log In…")).clicked() { + match app.run("ui.connection", json!({"id": c.id})) { + Ok(_) => { + app.dialogs.info = Some(tl!("Type the password or keys to log in for this session, then Save.").to_string()) + } + Err(e) => app.session.notify(e, true), + } + } + }); + } + Some(Ok(l)) => { + children(app, ui, t, c, &l, &l.children); + if l.children.is_empty() { + ui.label(RichText::new(tl!("Nothing here")).small().weak()); + } + if l.truncated { + let n = l.children.len().to_string(); + let text = match l.matched { + Some(m) => i18n::fmt("{n} of {m} are listed.", &[("n", &n), ("m", &m.to_string())]), + None => i18n::fmt("The first {n} are listed.", &[("n", &n)]), + }; + ui.label(RichText::new(text).small().weak()); + } + for note in &l.notes { + ui.label(RichText::new(note).small().weak()); + } + } + } +} + +/// A folder in the tree: a header that opens on a click, and a body drawn when it is open. +/// `node` is set when the body is a level of its own (read when first opened, refreshed from +/// the header's menu). +fn branch( + app: &mut RustGisApp, + ui: &mut Ui, + c: &Connection, + child: &Value, + node: Option<&str>, + body: impl FnOnce(&mut RustGisApp, &mut Ui), +) { + let salt = node.or(child["key"].as_str()).unwrap_or(""); + let id = ui.make_persistent_id(("conntree", &c.id, salt)); + let state = CollapsingState::load_with_default_open(ui.ctx(), id, false); + let open = state.is_open(); + let (mut clicked, mut refreshed) = (false, false); + let mut header = state.show_header(ui, |ui| { + ui.label(RichText::new(what_label(child["what"].as_str().unwrap_or(""))).small().weak()); + let r = ui.add(Label::new(child["name"].as_str().unwrap_or("")).sense(Sense::click())); + clicked = r.clicked(); + if node.is_some() { + r.context_menu(|ui| { + if ui.button(tl!("Refresh")).clicked() { + refreshed = true; + ui.close(); + } + }); + } + }); + if clicked { + header.set_open(!open); + } + header.body(|ui| body(app, ui)); + if refreshed && let Some(node) = node { + app.ui.connection_tree.entries.remove(&(c.id.clone(), node.to_string())); + } +} + +fn children(app: &mut RustGisApp, ui: &mut Ui, t: &Tokens, c: &Connection, listing: &Listing, list: &[Value]) { + for child in list { + if let Some(node) = child["node"].as_str() { + branch(app, ui, c, child, Some(node), |app, ui| level(app, ui, t, c, node)); + continue; + } + if let Some(inline) = child["children"].as_array() { + branch(app, ui, c, child, None, |app, ui| { + children(app, ui, t, c, listing, inline); + if inline.is_empty() { + ui.label(RichText::new(tl!("Nothing here")).small().weak()); + } + }); + continue; + } + dataset(app, ui, c, listing, child); + } +} + +/// A dataset: click selects it, double-click adds it, the menu adds it (with options). +fn dataset(app: &mut RustGisApp, ui: &mut Ui, c: &Connection, listing: &Listing, child: &Value) { + let name = child["name"].as_str().unwrap_or(""); + let is_picked = picked(app, &c.id, child["key"].as_str().unwrap_or("")); + let r = ui + .horizontal(|ui| { + let r = ui.selectable_label(is_picked, name); + ui.label(RichText::new(what_label(child["what"].as_str().unwrap_or(""))).small().weak()); + r + }) + .inner; + let mut hover = tl!("Click to select, double-click to add").to_string(); + if let Some(detail) = child["detail"].as_str().filter(|d| !d.is_empty()) { + hover = format!("{hover}\n{detail}"); + } + let r = r.on_hover_text(hover); + let (mut add_now, mut with_options) = (false, false); + r.context_menu(|ui| { + if ui.button(tl!("Add")).clicked() { + add_now = true; + ui.close(); + } + if child["options"] == true && ui.button(tl!("Add With Options…")).clicked() { + with_options = true; + ui.close(); + } + }); + if r.double_clicked() { + // The first click selected it: a double-click adds it alone. + let key = child["key"].as_str().unwrap_or(""); + app.ui.connection_tree.picks.retain(|p| !(p.conn == c.id && p.key == key)); + add_now = true; + } else if r.clicked() { + toggle(app, c, child, listing.source.as_ref()); + } + let result = if with_options { + add_with_options(app, c, child, listing.source.as_deref()) + } else if add_now { + add(app, child, listing.source.as_deref()) + } else { + Ok(()) + }; + if let Err(e) = result { + app.session.notify(e, true); + } +} diff --git a/crates/ui-egui/src/dialogs.rs b/crates/ui-egui/src/dialogs.rs index c7041460f..06cfdb4ca 100644 --- a/crates/ui-egui/src/dialogs.rs +++ b/crates/ui-egui/src/dialogs.rs @@ -800,60 +800,17 @@ fn add_service(app: &mut RustGisApp, ui: &mut egui::Ui) -> bool { }); } if let (Some(l), Some(conn)) = (add, &app.ui.service) { - let (url, kind, text) = (conn.url.clone(), conn.kind, conn.text.clone().unwrap_or_default()); - let name = l.get("name").and_then(Value::as_str).unwrap_or("").to_string(); - let title = l.get("title").and_then(Value::as_str).unwrap_or(&name).to_string(); - let offer = l.get("offer").cloned().unwrap_or_default(); - // Features are downloaded in the background, as From URL… does; map layers are added now. - let r = if offer.get("type").and_then(Value::as_str) == Some("features") { - let items = offer.get("url").and_then(Value::as_str).unwrap_or("").to_string(); - app.run("ui.addUrl", json!({"url": items, "name": title, "service": true})).map(|_| tl!("Downloading…")) - } else if offer.get("type").and_then(Value::as_str) == Some("coverage") { - // A coverage: its description first, off the frame (poll_describe asks for it). - let describe = offer.get("describe").and_then(Value::as_str).unwrap_or("").to_string(); - let mut p = json!({"url": url, "layer": name, "text": text, "name": title, "documents": conn.documents}); - if kind != "auto" { - p["type"] = json!(kind); - } - let slot = crate::fetch_into(&describe); - if let Some(c) = &mut app.ui.service { - c.describing = Some((p, slot)); - } - Ok(tl!("Describing")) - } else { - let mut p = json!({"url": url, "layer": name, "text": text, "documents": conn.documents, "source_url": l.get("source_url")}); - if kind != "auto" { - p["type"] = json!(kind); - } - let time = app.ui.service_browse.time.trim(); - if l.get("time").is_some() && !time.is_empty() { - p["time"] = json!(time); - } - match l.get("tilesets").and_then(Value::as_str) { - // An OGC API layer: its tileset's advertised template is read first, off the - // frame (poll_tiles_resolve adds the layer). - Some(list) => { - p["name"] = json!(title); - let fallback = offer.get("url").and_then(Value::as_str).unwrap_or("").to_string(); - let vector = offer.get("type").and_then(Value::as_str) == Some("vectorTiles"); - let slot = crate::fetch_into(list); - if let Some(c) = &mut app.ui.service { - c.resolving = Some(crate::TilesResolve { - params: p, - list: list.to_string(), - doc: list.to_string(), - read: 0, - vector, - fallback, - slot, - }); - } - Ok(tl!("Reading the tileset of")) - } - None => app.run("service.add", p).map(|_| tl!("Added")), - } + let src = crate::serviceui::ServiceSource { + url: conn.url.clone(), + kind: conn.kind, + text: conn.text.clone().unwrap_or_default(), + documents: conn.documents.clone(), }; - match r { + let name = l.get("name").and_then(Value::as_str).unwrap_or(""); + let title = l.get("title").and_then(Value::as_str).unwrap_or(name).to_string(); + // Features are downloaded in the background, as From URL… does; map layers are added now. + let time = app.ui.service_browse.time.trim().to_string(); + match crate::serviceui::add_layer(app, &src, &l, &time, true) { Ok(m) => app.dialogs.info = Some(format!("{m} {title}")), Err(e) => app.dialogs.error = Some(e), } diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index e4040665c..100a0abbd 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -17,6 +17,7 @@ mod bookmarkpane; mod catalogsui; mod chrome; mod comets; +mod connectiontree; mod connectionui; #[cfg(not(target_arch = "wasm32"))] mod connectui; @@ -291,11 +292,6 @@ pub struct ServiceConn { pub error: Option, /// A capabilities request that failed, with what the server said (#509). pub failure: Option, - /// An OGC API layer whose advertised tile template is being read before it is added. - pub resolving: Option, - /// A WCS coverage whose description is being read before it is asked for: `service.coverage`'s - /// parameters and the description on its way. - pub describing: Option<(Value, Slot, String>>)>, } /// How Add From Service's list is browsed. @@ -324,6 +320,16 @@ pub struct TilesResolve { pub slot: Slot, String>>, } +/// A service layer waiting on a document before it is added. `dialog` tells whether the add came +/// from the add-service dialog (its result shows there) or from the Connections tree (a notice). +pub enum ServiceStep { + /// An OGC API layer whose advertised tile template is being read. + Tiles { resolve: TilesResolve, dialog: bool }, + /// A WCS coverage whose description is being read before it is asked for: `service.coverage`'s + /// parameters and the description on its way. + Coverage { params: Value, slot: Slot, String>>, dialog: bool }, +} + /// Fetches an address into a slot, off the frame. pub(crate) fn fetch_into(url: &str) -> Slot, String>> { fetch_into_limited(url, rustgis_engine::render::rustgis_tiles::cache::MAX_DOWNLOAD) @@ -365,7 +371,7 @@ pub enum LayersView { } /// Retains the GUI's legacy XML decoding while refusing malformed JSON text. -fn service_text(bytes: Vec) -> Result { +pub(crate) fn service_text(bytes: Vec) -> Result { String::from_utf8(bytes).or_else(|error| { let bytes = error.as_bytes(); let without_bom = bytes.strip_prefix(b"\xef\xbb\xbf").unwrap_or(bytes); @@ -515,6 +521,9 @@ pub struct UiState { /// Add From Service's list: its filter, a time-enabled layer's date, and the directories /// opened on the way to the service shown (Back returns to them). pub service_browse: ServiceBrowse, + /// Service layers added from the dialog or the Connections tree that wait on a document + /// (a tileset list, a coverage's description) before they are added. + pub service_steps: Vec, /// The Web Data panels of single services (NASA Earthdata, EnviroAtlas, FEMA NFHL, The /// National Map, Fields of the World): their documents, searches and choices. pub web_services: webdataui::State, @@ -528,6 +537,8 @@ pub struct UiState { pub s3: s3ui::State, /// New/Edit Connection: the form and the Test under way. pub connection: connectionui::State, + /// The datasets saved connections list under Data Sources › Connections, and the selection. + pub connection_tree: connectiontree::State, /// OpenStreetMap and Boundaries: their choices and the request under way. pub portals: portalsui::State, /// Overture Maps and Source Cooperative: choices, results and the request under way. @@ -649,6 +660,7 @@ impl Default for UiState { link_zoomed: false, service: None, service_browse: ServiceBrowse::default(), + service_steps: Vec::new(), web_services: Default::default(), service_import: None, service_group_name: "My OGC Layer".into(), @@ -656,6 +668,7 @@ impl Default for UiState { stac: Default::default(), s3: Default::default(), connection: Default::default(), + connection_tree: Default::default(), portals: Default::default(), catalogs: Default::default(), opendata: Default::default(), @@ -2113,8 +2126,6 @@ impl RustGisApp { listing: None, error: None, failure: None, - resolving: None, - describing: None, }); } "ui.addData" => { @@ -2993,60 +3004,71 @@ impl RustGisApp { } } - /// Moves an OGC API layer's template reading along: the list, then the tileset when the list - /// doesn't carry the template; the layer is added with the template read, or the usual path. - fn poll_tiles_resolve(&mut self) { - use rustgis_engine::data::ogc::{TilesStep, resolve_tiles}; - let Some(conn) = &mut self.ui.service else { return }; - let Some(res) = &mut conn.resolving else { return }; - let Some(r) = res.slot.lock().unwrap_or_else(|e| e.into_inner()).take() else { return }; - let step = r.ok().and_then(|body| resolve_tiles(&res.list, &res.doc, &body, res.vector)); - let template = match step { - Some(TilesStep::Next(next)) if res.read < 2 => { - res.read += 1; - res.doc = next.clone(); - res.slot = fetch_into(&next); - return; - } - Some(TilesStep::Template(t)) => t, - _ => res.fallback.clone(), - }; - let Some(res) = conn.resolving.take() else { return }; - let mut p = res.params; - p["tiles"] = json!(template); - let title = p["name"].as_str().unwrap_or("").to_string(); - match self.run("service.add", p) { - Ok(_) => self.dialogs.info = Some(format!("{} {title}", tl!("Added"))), - Err(e) => self.dialogs.error = Some(e), + /// Tells how a service layer added outside the add-service dialog (or in it) went: in the + /// dialog's own line, or as a notice for the Connections tree. + fn report_step(&mut self, dialog: bool, r: Result) { + match (dialog, r) { + (true, Ok(m)) => self.dialogs.info = Some(m), + (true, Err(e)) => self.dialogs.error = Some(e), + (false, Ok(m)) => self.session.notify(m, false), + (false, Err(e)) => self.session.notify(e, true), } - self.map.invalidate(); } - /// Asks for a WCS coverage once its description has arrived: its GetCoverage address for the - /// view, downloaded as From URL… does. - fn poll_describe(&mut self) { - let Some(conn) = &mut self.ui.service else { return }; - let Some((_, slot)) = &conn.describing else { return }; - let Some(r) = slot.lock().unwrap_or_else(|e| e.into_inner()).take() else { return }; - let Some((mut p, _)) = conn.describing.take() else { return }; - let title = p["name"].as_str().unwrap_or("").to_string(); - let r = r.and_then(|bytes| { - p["describe"] = json!(String::from_utf8_lossy(&bytes)); - // Not `self.run`: an error belongs in the dialog, not the notices. - let c = self.session.run("service.coverage", &p)?; - let get = c.get("url").and_then(Value::as_str).unwrap_or("").to_string(); - self.run("ui.addUrl", json!({"url": get, "name": title, "coverage": true})) - }); - match r { - Ok(_) => self.dialogs.info = Some(format!("{} {title}", tl!("Downloading…"))), - Err(e) => self.dialogs.error = Some(e), + /// Moves the service layers waiting on a document along: an OGC API layer's tile template + /// (read from its tileset list, then the tileset when the list doesn't carry it; the layer is + /// added with the template read, or the usual path) and a WCS coverage's description (the + /// coverage is then asked for the view and downloaded as From URL… does). + fn poll_service_steps(&mut self) { + use rustgis_engine::data::ogc::{TilesStep, resolve_tiles}; + for step in std::mem::take(&mut self.ui.service_steps) { + match step { + ServiceStep::Tiles { mut resolve, dialog } => { + let Some(r) = resolve.slot.lock().unwrap_or_else(|e| e.into_inner()).take() else { + self.ui.service_steps.push(ServiceStep::Tiles { resolve, dialog }); + continue; + }; + let next = r.ok().and_then(|body| resolve_tiles(&resolve.list, &resolve.doc, &body, resolve.vector)); + let template = match next { + Some(TilesStep::Next(next)) if resolve.read < 2 => { + resolve.read += 1; + resolve.doc = next.clone(); + resolve.slot = fetch_into(&next); + self.ui.service_steps.push(ServiceStep::Tiles { resolve, dialog }); + continue; + } + Some(TilesStep::Template(t)) => t, + _ => resolve.fallback.clone(), + }; + let mut p = resolve.params; + p["tiles"] = json!(template); + let title = p["name"].as_str().unwrap_or("").to_string(); + let r = self.run("service.add", p).map(|_| format!("{} {title}", tl!("Added"))); + self.report_step(dialog, r); + self.map.invalidate(); + } + ServiceStep::Coverage { mut params, slot, dialog } => { + let Some(r) = slot.lock().unwrap_or_else(|e| e.into_inner()).take() else { + self.ui.service_steps.push(ServiceStep::Coverage { params, slot, dialog }); + continue; + }; + let title = params["name"].as_str().unwrap_or("").to_string(); + let r = r.and_then(|bytes| { + params["describe"] = json!(String::from_utf8_lossy(&bytes)); + // Not `self.run`: an error belongs where the step reports, not the notices. + let c = self.session.run("service.coverage", ¶ms)?; + let get = c.get("url").and_then(Value::as_str).unwrap_or("").to_string(); + self.run("ui.addUrl", json!({"url": get, "name": title, "coverage": true})) + }); + self.report_step(dialog, r.map(|_| format!("{} {title}", tl!("Downloading…")))); + } + } } } /// Lists a service's layers once its capabilities have arrived. fn poll_service(&mut self) { - self.poll_tiles_resolve(); - self.poll_describe(); + self.poll_service_steps(); let Some(conn) = &mut self.ui.service else { return }; let Some(r) = conn.slot.lock().unwrap_or_else(|e| e.into_inner()).take() else { return }; let document = conn.pending.take().unwrap_or_else(|| conn.url.clone()); @@ -3170,6 +3192,7 @@ impl RustGisApp { stacui::poll(self, ctx); s3ui::poll(self, ctx); connectionui::poll(self, ctx); + connectiontree::poll(self, ctx); portalsui::poll(self, ctx); catalogsui::poll(self, ctx); opendataui::poll(self, ctx); diff --git a/crates/ui-egui/src/serviceui.rs b/crates/ui-egui/src/serviceui.rs index c000294fd..834c6e5a9 100644 --- a/crates/ui-egui/src/serviceui.rs +++ b/crates/ui-egui/src/serviceui.rs @@ -20,6 +20,74 @@ pub struct Import { bytes: usize, } +/// A service whose layers are listed: its address, type, capabilities document and the further +/// documents its discovery read. `service.add` reads layers from them without fetching again. +pub struct ServiceSource { + pub url: String, + /// `wms`, `wmts`, `wfs`, `wcs`, `ogcapi`, `arcgis`, or `auto`. + pub kind: &'static str, + pub text: String, + pub documents: Vec, +} + +/// A service type as the add dialog and `ui.addService` name it, `auto` for any other text. +pub(crate) fn kind_of(t: &str) -> &'static str { + match t { + "wms" => "wms", + "wmts" => "wmts", + "wfs" => "wfs", + "wcs" => "wcs", + "ogcapi" => "ogcapi", + "arcgis" => "arcgis", + _ => "auto", + } +} + +/// Adds one of a service's layers (as `service.layers` lists it): features are downloaded in the +/// background as From URL… does, map layers are added now, and an OGC API layer or a WCS coverage +/// waits on a document (a step the app moves along). `dialog` tells where the result is reported. +pub(crate) fn add_layer(app: &mut RustGisApp, src: &ServiceSource, l: &Value, time: &str, dialog: bool) -> Result<&'static str, String> { + let name = l.get("name").and_then(Value::as_str).unwrap_or("").to_string(); + let title = l.get("title").and_then(Value::as_str).unwrap_or(&name).to_string(); + let offer = l.get("offer").cloned().unwrap_or_default(); + if offer.get("type").and_then(Value::as_str) == Some("features") { + let items = offer.get("url").and_then(Value::as_str).unwrap_or("").to_string(); + return app.run("ui.addUrl", json!({"url": items, "name": title, "service": true})).map(|_| tl!("Downloading…")); + } + if offer.get("type").and_then(Value::as_str) == Some("coverage") { + // A coverage: its description first, off the frame (a step asks for it). + let describe = offer.get("describe").and_then(Value::as_str).unwrap_or("").to_string(); + let mut p = json!({"url": src.url, "layer": name, "text": src.text, "name": title, "documents": src.documents}); + if src.kind != "auto" { + p["type"] = json!(src.kind); + } + let slot = crate::fetch_into(&describe); + app.ui.service_steps.push(crate::ServiceStep::Coverage { params: p, slot, dialog }); + return Ok(tl!("Describing")); + } + let mut p = json!({"url": src.url, "layer": name, "text": src.text, "documents": src.documents, "source_url": l.get("source_url")}); + if src.kind != "auto" { + p["type"] = json!(src.kind); + } + if l.get("time").is_some() && !time.is_empty() { + p["time"] = json!(time); + } + match l.get("tilesets").and_then(Value::as_str) { + // An OGC API layer: its tileset's advertised template is read first, off the frame (a + // step adds the layer). + Some(list) => { + p["name"] = json!(title); + let fallback = offer.get("url").and_then(Value::as_str).unwrap_or("").to_string(); + let vector = offer.get("type").and_then(Value::as_str) == Some("vectorTiles"); + let slot = crate::fetch_into(list); + let resolve = crate::TilesResolve { params: p, list: list.to_string(), doc: list.to_string(), read: 0, vector, fallback, slot }; + app.ui.service_steps.push(crate::ServiceStep::Tiles { resolve, dialog }); + Ok(tl!("Reading the tileset of")) + } + None => app.run("service.add", p).map(|_| tl!("Added")), + } +} + pub fn start(app: &mut RustGisApp, p: &Value, refresh: bool) -> Result<(), String> { if !p.is_object() { return Err(tl!("Give the service's web address").into()); diff --git a/crates/ui-egui/src/sourcespane.rs b/crates/ui-egui/src/sourcespane.rs index 0d01ae54d..2147fe4db 100644 --- a/crates/ui-egui/src/sourcespane.rs +++ b/crates/ui-egui/src/sourcespane.rs @@ -236,17 +236,23 @@ fn empty(ui: &mut Ui, what: &str) { } fn connections(app: &mut RustGisApp, ui: &mut Ui, t: &Tokens, list: &[Connection]) { - ui.menu_button(tl!("New Connection ▾"), |ui| { - for (kind, label) in crate::connectionui::KINDS { - if cfg!(target_arch = "wasm32") && DATABASE_KINDS.contains(&kind) { - continue; - } - if ui.button(tl!(label)).clicked() { - if let Err(e) = app.run("ui.connection", json!({"kind": kind})) { - app.session.notify(e, true); + crate::connectiontree::picks_bar(app, ui); + ui.horizontal(|ui| { + ui.menu_button(tl!("New Connection ▾"), |ui| { + for (kind, label) in crate::connectionui::KINDS { + if cfg!(target_arch = "wasm32") && DATABASE_KINDS.contains(&kind) { + continue; + } + if ui.button(tl!(label)).clicked() { + if let Err(e) = app.run("ui.connection", json!({"kind": kind})) { + app.session.notify(e, true); + } + ui.close(); } - ui.close(); } + }); + if ui.small_button("⟳").on_hover_text(tl!("Refresh every connection's lists")).clicked() { + crate::connectiontree::refresh(app, None); } }); if list.is_empty() { @@ -269,60 +275,86 @@ fn connections(app: &mut RustGisApp, ui: &mut Ui, t: &Tokens, list: &[Connection } fn connection_row(app: &mut RustGisApp, ui: &mut Ui, t: &Tokens, c: &Connection) { - let database = DATABASE_KINDS.contains(&c.kind.as_str()); - let unavailable = database && cfg!(target_arch = "wasm32"); - ui.horizontal(|ui| { - let icon = if database { - "table" - } else { - match c.kind.as_str() { - "stac" => "search", - "s3" => "package", - _ => "globe_view", - } - }; - let (r, _) = ui.allocate_exact_size(vec2(16.0, 16.0), Sense::hover()); - icons::paint(ui.painter(), r, icon, t.icon, t.accent); - let response = if unavailable { - ui.link(RichText::new(&c.name).weak()).on_hover_text(tl!("Database connections are used by the desktop app")) - } else { - ui.link(&c.name).on_hover_text(if c.kind == "s3" && c.address.is_empty() { tl!("Amazon S3") } else { &c.address }) - }; - if response.clicked() - && !unavailable - && let Err(e) = crate::connectionui::open_connection(app, c) - { - app.session.notify(e, true); + let unavailable = DATABASE_KINDS.contains(&c.kind.as_str()) && cfg!(target_arch = "wasm32"); + if unavailable { + ui.horizontal(|ui| { + connection_header(app, ui, t, c, true); + }); + return; + } + // A click on the name opens the row: the datasets the connection offers are listed under it. + let state = + egui::collapsing_header::CollapsingState::load_with_default_open(ui.ctx(), ui.make_persistent_id(("connection", &c.id)), false); + let open = state.is_open(); + let mut clicked = false; + let mut header = state.show_header(ui, |ui| clicked = connection_header(app, ui, t, c, false)); + if clicked { + header.set_open(!open); + } + header.body(|ui| crate::connectiontree::show(app, ui, t, c)); +} + +/// A connection's icon, name, kind and menu. True when the name was clicked. +fn connection_header(app: &mut RustGisApp, ui: &mut Ui, t: &Tokens, c: &Connection, unavailable: bool) -> bool { + let icon = if DATABASE_KINDS.contains(&c.kind.as_str()) { + "table" + } else { + match c.kind.as_str() { + "stac" => "search", + "s3" => "package", + _ => "globe_view", } - ui.label(RichText::new(tl!(crate::connectionui::label(&c.kind))).small().weak()); - response.context_menu(|ui| { - for (label, command, params) in [ - (tl!("Edit…"), "ui.connection", json!({"id":c.id})), - (tl!("Test"), "ui.connection", json!({"id":c.id,"test":true})), - (tl!("Duplicate"), "connection.duplicate", json!({"id":c.id})), - ] { - if params.get("test") == Some(&Value::Bool(true)) && unavailable { - continue; - } - if ui.button(label).clicked() { - if let Err(e) = app.run(command, params) { - app.session.notify(e, true); - } - ui.close(); + }; + let (r, _) = ui.allocate_exact_size(vec2(16.0, 16.0), Sense::hover()); + icons::paint(ui.painter(), r, icon, t.icon, t.accent); + let response = if unavailable { + ui.link(RichText::new(&c.name).weak()).on_hover_text(tl!("Database connections are used by the desktop app")) + } else { + ui.link(&c.name).on_hover_text(if c.kind == "s3" && c.address.is_empty() { tl!("Amazon S3") } else { &c.address }) + }; + ui.label(RichText::new(tl!(crate::connectionui::label(&c.kind))).small().weak()); + response.context_menu(|ui| { + let mut items = vec![ + (tl!("Edit…"), "ui.connection", json!({"id":c.id})), + (tl!("Test"), "ui.connection", json!({"id":c.id,"test":true})), + (tl!("Duplicate"), "connection.duplicate", json!({"id":c.id})), + ]; + if unavailable { + items.remove(1); + } + for (label, command, params) in items { + if ui.button(label).clicked() { + if let Err(e) = app.run(command, params) { + app.session.notify(e, true); } + ui.close(); } - if !c.address.is_empty() && ui.button(tl!("Copy Address")).clicked() { - ui.ctx().copy_text(c.address.clone()); + } + if !unavailable { + if ui.button(tl!("Refresh")).clicked() { + crate::connectiontree::refresh(app, Some(&c.id)); ui.close(); } - if ui.button(tl!("Remove")).clicked() { - if let Err(e) = app.run("connection.remove", json!({"id":c.id})) { + // The kind's own add dialog, as a click on the connection opened it before. + if ui.button(tl!("Add With Options…")).clicked() { + if let Err(e) = crate::connectionui::open_connection(app, c) { app.session.notify(e, true); } ui.close(); } - }); + } + if !c.address.is_empty() && ui.button(tl!("Copy Address")).clicked() { + ui.ctx().copy_text(c.address.clone()); + ui.close(); + } + if ui.button(tl!("Remove")).clicked() { + if let Err(e) = app.run("connection.remove", json!({"id":c.id})) { + app.session.notify(e, true); + } + ui.close(); + } }); + !unavailable && response.clicked() } pub fn show(app: &mut RustGisApp, ui: &mut Ui, t: &Tokens) { diff --git a/crates/ui-egui/tests/smoke.rs b/crates/ui-egui/tests/smoke.rs index a2c79dcc6..a1c6f741d 100644 --- a/crates/ui-egui/tests/smoke.rs +++ b/crates/ui-egui/tests/smoke.rs @@ -1692,6 +1692,65 @@ fn connections_test_feedback_tracks_edited_inputs() { assert!(h.state().session.connections.connections.is_empty()); } +/// Expanding a connection lists its datasets, a click selects and Add Selected adds each as its +/// own undo step, and Refresh forgets the list. +#[test] +fn connections_tree_browses_and_adds() { + use std::io::{Read, Write}; + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + std::thread::spawn(move || { + let body = format!( + r#"Localimage/pngEPSG:3857aAbB"# + ); + for stream in listener.incoming().take(1) { + let Ok(mut s) = stream else { continue }; + let mut buf = [0u8; 2048]; + let _ = s.read(&mut buf); + let _ = write!(s, "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()); + let _ = s.write_all(body.as_bytes()); + } + }); + let mut h = harness(); + h.state_mut().run("connection.new", json!({"kind":"wms","address":format!("http://{addr}/wms"),"name":"Local"})).unwrap(); + h.state_mut().run("ui.pane.sources", json!({})).unwrap(); + h.step(); + assert!(h.query_by_label("A").is_none(), "nothing is listed before the connection is opened"); + h.get_by_label("Local").click(); + for _ in 0..100 { + h.step(); + if h.query_by_label("A").is_some() { + break; + } + std::thread::sleep(std::time::Duration::from_millis(20)); + } + assert!(h.query_by_label("A").is_some(), "the layers are listed under the connection"); + let before = h.state().session.map().tiles.len(); + // The selection bar appears above the list once something is picked, shifting it by a row: + // a second frame lets the harness see the new layout. + h.get_by_label("A").click(); + h.step(); + h.step(); + h.get_by_label("B").click(); + h.step(); + h.step(); + assert_eq!(h.state().ui.connection_tree.picks.len(), 2); + h.get_by_label("Add Selected (2)").click(); + h.step(); + h.step(); + assert_eq!(h.state().session.map().tiles.len(), before + 2); + assert!(h.state().ui.connection_tree.picks.is_empty()); + h.state_mut().run("edit.undo", json!(null)).unwrap(); + h.state_mut().run("edit.undo", json!(null)).unwrap(); + assert_eq!(h.state().session.map().tiles.len(), before); + h.get_by_label("Local").click_secondary(); + h.step(); + h.get_by_label("Refresh").click(); + h.step(); + h.step(); + assert!(h.query_by_label("A").is_none(), "refreshing forgets the list until it is read again"); +} + /// Save in the web app (issue #197): the browser has no save dialog, so the first Save and every /// Save As ask for the file name, and Save then downloads under it; an opened file keeps its name. #[test] diff --git a/docs/user-guide/data.md b/docs/user-guide/data.md index c7fc11601..8d09e9c53 100644 --- a/docs/user-guide/data.md +++ b/docs/user-guide/data.md @@ -334,9 +334,14 @@ places data comes from: tables, Zarr variables, STAC assets, sample data), newest first. - **Folders**: the folders connected to the Browser (**Add Folder…**; desktop). - **Connections**: named services, databases, STAC APIs and S3 endpoints, organised by optional - groups. **New Connection ▾** opens a kind-specific form; right-click a row to edit, test, - duplicate, copy its address or remove it. A click opens the existing service, database, - STAC or S3 add dialog, connected or prefilled. + groups. **New Connection ▾** opens a kind-specific form. Click a connection to open it: its + datasets are listed under it, a level at a time as you open its folders, with **Refresh** + (a connection's or a folder's right-click menu; **⟳** refreshes every connection) and an error + with **Retry** (and **Log In…** for databases and keyed S3) when it can't be read. Click a + dataset to select it and **Add Selected (N)** adds them all; double-click, or right-click › + **Add**, adds one. Each added layer is its own undo step. **Add With Options…** opens the + existing service, database, STAC or S3 add dialog, connected or prefilled. Right-click a row + to edit, test, duplicate, copy its address or remove it. - **Library**: well-known public sources: OpenStreetMap tiles, EOX Sentinel-2 cloudless and NASA GIBS (WMTS), USGS imagery, topographic and hydrography maps (ArcGIS), GEBCO bathymetry (WMS), swisstopo's maps and imagery (geo.admin.ch, WMTS), Earth Search and Planetary Computer (STAC), and the Natural Earth sample layers. @@ -389,6 +394,18 @@ databases). For non-database kinds, pass that response text as `answer` with the to parse it without a network request. Database kinds ignore `answer` and test through their driver. An anonymous S3 test needs at least one named bucket. +`connection.browse` lists what a connection offers, a level at a time: the tables of a database +(under their schemas), the layers of a service, the records of a catalogue with the services and +files they link to, the collections of a STAC API and the items and assets of a collection, or +the buckets, folders and files of an S3 connection. Each child has a `key`, a `name` and what it +is (`what`); a child with a `node` is browsed by asking again with that `node`; one with `add` +names the command and parameters that add it (`service.add`, `layer.addDatabase`, +`stac.addAsset`, `layer.addUrl`). A level lists at most 500 children and says `truncated` when +it left some out. Like `connection.test`, it reads the address itself on the desktop and the +command line; with `probe: true` it returns the address to fetch, and the app passes what it +fetched back as `answer` (and further `documents` when a service asks for them, as `next`). +Browsing never changes the project or the saved connections. + Addresses are saved without passwords or tokens. `connection.login` remembers a database `password`, or S3 `accessKeyId`, `secretAccessKey` and optional `sessionToken`, for this session only, using the same credential stores as the existing add commands. With login parameters, @@ -400,7 +417,7 @@ connection also affects a session entry of the same name created there. Renaming clears the runtime entries under both the old and new names, so the renamed connection cannot inherit an unrelated login at its destination name. Web-service logins are not supported. Plugins may list connections but cannot manage, log in -to or test them. The in-app assistant asks for approval each time for those operations; +to, test or browse them. The in-app assistant asks for approval each time for those operations; external assistants using MCP cannot run them. Database passwords are shared by public DSN for the session, including existing layers and other named connections using that DSN. Editing or removing a named connection does not revoke From ba63e66f80fb14390384441bcd70d0665a406ff2 Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Thu, 22 Oct 2026 19:36:38 +0200 Subject: [PATCH 2/9] feat(connections): add the selection from a dataset's menu instead of a bar The Add Selected / Clear Selection bar shifted the list when the first dataset was picked, which also moved the row between the two clicks of a double-click. They are now entries in a dataset's right-click menu; a double-click adds that dataset. --- crates/ui-egui/src/connectiontree.rs | 59 ++++++++++++++++------------ crates/ui-egui/src/sourcespane.rs | 2 +- crates/ui-egui/tests/smoke.rs | 32 ++++++++++++--- docs/user-guide/data.md | 5 ++- 4 files changed, 64 insertions(+), 34 deletions(-) diff --git a/crates/ui-egui/src/connectiontree.rs b/crates/ui-egui/src/connectiontree.rs index a1c312f9b..4669b9b54 100644 --- a/crates/ui-egui/src/connectiontree.rs +++ b/crates/ui-egui/src/connectiontree.rs @@ -1,7 +1,7 @@ //! The datasets a saved connection offers, listed under it in Data Sources › Connections: one //! level at a time as it is expanded (`connection.browse` names what each level reads, the app //! fetches it off the frame and asks again with the answer), with the selection that -//! Add Selected adds. Each dataset added is its own undo step. +//! Add Selected (in a dataset's menu) adds. Each dataset added is its own undo step. use std::collections::HashMap; use std::sync::Arc; @@ -313,33 +313,22 @@ pub fn refresh(app: &mut RustGisApp, id: Option<&str>) { } } -/// Add Selected and Clear Selection, shown while datasets are selected. -pub fn picks_bar(app: &mut RustGisApp, ui: &mut Ui) { - // A pick made for a connection that was edited or removed since (even while its row is - // folded, when its list isn't refreshed) would add from the old definition: dropped. +/// Drops the picks of a connection that was edited or removed since (even while its row is +/// folded, when its list isn't refreshed): they would add from the old definition. +pub fn prune(app: &mut RustGisApp) { let (connections, tree) = (&app.session.connections, &mut app.ui.connection_tree); let seen = &tree.seen; tree.picks.retain(|p| connections.find(&p.conn).is_some_and(|c| seen.get(&p.conn) == Some(c))); - if tree.picks.is_empty() { - return; - } - ui.horizontal(|ui| { - let n = app.ui.connection_tree.picks.len().to_string(); - if ui.button(i18n::fmt("Add Selected ({n})", &[("n", &n)])).clicked() { - for pick in std::mem::take(&mut app.ui.connection_tree.picks) { - // A connection removed since it was selected is skipped. - if app.session.connections.find(&pick.conn).is_none() { - continue; - } - if let Err(e) = add(app, &pick.child, pick.source.as_deref()) { - app.session.notify(e, true); - } - } - } - if ui.button(tl!("Clear Selection")).clicked() { - app.ui.connection_tree.picks.clear(); +} + +/// Adds every selected dataset, each as its own undo step, and clears the selection. +fn add_picks(app: &mut RustGisApp) { + prune(app); + for pick in std::mem::take(&mut app.ui.connection_tree.picks) { + if let Err(e) = add(app, &pick.child, pick.source.as_deref()) { + app.session.notify(e, true); } - }); + } } /// A connection's expanded body: its top level, listed (or its error). @@ -477,12 +466,13 @@ fn dataset(app: &mut RustGisApp, ui: &mut Ui, c: &Connection, listing: &Listing, r }) .inner; - let mut hover = tl!("Click to select, double-click to add").to_string(); + let mut hover = tl!("Click to select, double-click to add, right-click for more").to_string(); if let Some(detail) = child["detail"].as_str().filter(|d| !d.is_empty()) { hover = format!("{hover}\n{detail}"); } let r = r.on_hover_text(hover); - let (mut add_now, mut with_options) = (false, false); + let (mut add_now, mut with_options, mut add_selected, mut clear) = (false, false, false, false); + let selected = app.ui.connection_tree.picks.len(); r.context_menu(|ui| { if ui.button(tl!("Add")).clicked() { add_now = true; @@ -492,6 +482,17 @@ fn dataset(app: &mut RustGisApp, ui: &mut Ui, c: &Connection, listing: &Listing, with_options = true; ui.close(); } + if selected > 0 { + ui.separator(); + if ui.button(i18n::fmt("Add Selected ({n})", &[("n", &selected.to_string())])).clicked() { + add_selected = true; + ui.close(); + } + if ui.button(tl!("Clear Selection")).clicked() { + clear = true; + ui.close(); + } + } }); if r.double_clicked() { // The first click selected it: a double-click adds it alone. @@ -501,6 +502,12 @@ fn dataset(app: &mut RustGisApp, ui: &mut Ui, c: &Connection, listing: &Listing, } else if r.clicked() { toggle(app, c, child, listing.source.as_ref()); } + if clear { + app.ui.connection_tree.picks.clear(); + } + if add_selected { + add_picks(app); + } let result = if with_options { add_with_options(app, c, child, listing.source.as_deref()) } else if add_now { diff --git a/crates/ui-egui/src/sourcespane.rs b/crates/ui-egui/src/sourcespane.rs index 2147fe4db..7ac0d7ae4 100644 --- a/crates/ui-egui/src/sourcespane.rs +++ b/crates/ui-egui/src/sourcespane.rs @@ -236,7 +236,7 @@ fn empty(ui: &mut Ui, what: &str) { } fn connections(app: &mut RustGisApp, ui: &mut Ui, t: &Tokens, list: &[Connection]) { - crate::connectiontree::picks_bar(app, ui); + crate::connectiontree::prune(app); ui.horizontal(|ui| { ui.menu_button(tl!("New Connection ▾"), |ui| { for (kind, label) in crate::connectionui::KINDS { diff --git a/crates/ui-egui/tests/smoke.rs b/crates/ui-egui/tests/smoke.rs index a1c6f741d..5ce3e6af0 100644 --- a/crates/ui-egui/tests/smoke.rs +++ b/crates/ui-egui/tests/smoke.rs @@ -1711,7 +1711,15 @@ fn connections_tree_browses_and_adds() { let _ = s.write_all(body.as_bytes()); } }); - let mut h = harness(); + // Frames 0.05 s apart, so two clicks fall within egui's double-click delay. + let mut h = egui_kittest::Harness::builder().with_size(egui::vec2(1400.0, 860.0)).with_max_steps(64).with_step_dt(0.05).build_ui_state( + |ui, app: &mut RustGisApp| { + let ctx = ui.ctx().clone(); + app.logic(&ctx); + app.ui(ui); + }, + RustGisApp::with_sample(), + ); h.state_mut().run("connection.new", json!({"kind":"wms","address":format!("http://{addr}/wms"),"name":"Local"})).unwrap(); h.state_mut().run("ui.pane.sources", json!({})).unwrap(); h.step(); @@ -1726,15 +1734,15 @@ fn connections_tree_browses_and_adds() { } assert!(h.query_by_label("A").is_some(), "the layers are listed under the connection"); let before = h.state().session.map().tiles.len(); - // The selection bar appears above the list once something is picked, shifting it by a row: - // a second frame lets the harness see the new layout. h.get_by_label("A").click(); h.step(); - h.step(); h.get_by_label("B").click(); h.step(); - h.step(); assert_eq!(h.state().ui.connection_tree.picks.len(), 2); + // The selection is added from a dataset's menu: each layer is its own undo step. + h.get_by_label("B").click_secondary(); + h.step(); + h.step(); h.get_by_label("Add Selected (2)").click(); h.step(); h.step(); @@ -1743,6 +1751,20 @@ fn connections_tree_browses_and_adds() { h.state_mut().run("edit.undo", json!(null)).unwrap(); h.state_mut().run("edit.undo", json!(null)).unwrap(); assert_eq!(h.state().session.map().tiles.len(), before); + // A double-click adds the one dataset, leaving nothing selected. + // (egui counts clicks in a row: let the earlier ones age out of its triple-click window first.) + for _ in 0..14 { + h.step(); + } + h.get_by_label("A").click(); + h.step(); + h.get_by_label("A").click(); + h.step(); + h.step(); + assert_eq!(h.state().session.map().tiles.len(), before + 1); + assert!(h.state().ui.connection_tree.picks.is_empty()); + h.state_mut().run("edit.undo", json!(null)).unwrap(); + assert_eq!(h.state().session.map().tiles.len(), before); h.get_by_label("Local").click_secondary(); h.step(); h.get_by_label("Refresh").click(); diff --git a/docs/user-guide/data.md b/docs/user-guide/data.md index 8d09e9c53..ae1dd1c9c 100644 --- a/docs/user-guide/data.md +++ b/docs/user-guide/data.md @@ -338,8 +338,9 @@ places data comes from: datasets are listed under it, a level at a time as you open its folders, with **Refresh** (a connection's or a folder's right-click menu; **⟳** refreshes every connection) and an error with **Retry** (and **Log In…** for databases and keyed S3) when it can't be read. Click a - dataset to select it and **Add Selected (N)** adds them all; double-click, or right-click › - **Add**, adds one. Each added layer is its own undo step. **Add With Options…** opens the + dataset to select it (click again to clear it); double-click, or right-click › **Add**, adds + one, and right-click › **Add Selected (N)** adds them all (**Clear Selection** empties the + selection). Each added layer is its own undo step. **Add With Options…** opens the existing service, database, STAC or S3 add dialog, connected or prefilled. Right-click a row to edit, test, duplicate, copy its address or remove it. - **Library**: well-known public sources: OpenStreetMap tiles, EOX Sentinel-2 cloudless and From f782b0ae7f15c1106da3f33582601b0496d2c9f9 Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Thu, 22 Oct 2026 19:54:26 +0200 Subject: [PATCH 3/9] =?UTF-8?q?docs:=20say=20where=20Add=20With=20Options?= =?UTF-8?q?=E2=80=A6=20is=20offered=20in=20the=20Connections=20tree?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/user-guide/data.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/user-guide/data.md b/docs/user-guide/data.md index ae1dd1c9c..ee99e9085 100644 --- a/docs/user-guide/data.md +++ b/docs/user-guide/data.md @@ -340,9 +340,10 @@ places data comes from: with **Retry** (and **Log In…** for databases and keyed S3) when it can't be read. Click a dataset to select it (click again to clear it); double-click, or right-click › **Add**, adds one, and right-click › **Add Selected (N)** adds them all (**Clear Selection** empties the - selection). Each added layer is its own undo step. **Add With Options…** opens the - existing service, database, STAC or S3 add dialog, connected or prefilled. Right-click a row - to edit, test, duplicate, copy its address or remove it. + selection). Each dataset is added by one command, so each is one undo step. On a table, a + service layer or a STAC collection, **Add With Options…** opens the database, service or STAC + add dialog on it; on a connection, it opens the kind's add dialog. Right-click a connection to + edit, test, duplicate, copy its address or remove it. - **Library**: well-known public sources: OpenStreetMap tiles, EOX Sentinel-2 cloudless and NASA GIBS (WMTS), USGS imagery, topographic and hydrography maps (ArcGIS), GEBCO bathymetry (WMS), swisstopo's maps and imagery (geo.admin.ch, WMTS), Earth Search and Planetary Computer (STAC), and the Natural Earth sample layers. From a5d0afc461abeff28960742609532c296f1ae350 Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Thu, 22 Oct 2026 20:17:36 +0200 Subject: [PATCH 4/9] fix(connections): address review of the Connections tree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - a dataset without an add (grouped OGC API features) can't be selected or double-clicked, and has no Add entry: it keeps Add With Options… - a CSW download link too long to follow is left out (and the list says it is truncated); the name and title kept with it are cut at 200 characters - connection.browse refuses an answer larger than a service document may be (64 MB) - the engine test runs the add a browsed WCS coverage lists --- crates/engine/src/cmd/connection_browse.rs | 15 +++++++++++++-- crates/engine/src/tests.rs | 12 ++++++++++++ crates/ui-egui/src/connectiontree.rs | 12 ++++++++---- 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/crates/engine/src/cmd/connection_browse.rs b/crates/engine/src/cmd/connection_browse.rs index 538b1012d..41ffa2793 100644 --- a/crates/engine/src/cmd/connection_browse.rs +++ b/crates/engine/src/cmd/connection_browse.rs @@ -166,6 +166,10 @@ pub(super) fn browse(s: &mut Session, p: &Value) -> CmdResult { if !node_ok(node) { return Err("node is a child's node, as connection.browse lists it".into()); } + // The app's answers come from a fetch the cache already limits; a caller's may be any size. + if ps(p, "answer").is_some_and(|a| a.len() > super::super::service::MAX_GROUP_BYTES) { + return Err("answer is too large to read".into()); + } let level = level(&c, node)?; let conn = s3_login(s, &c, &level)?; let url = fetch_url(s, &c, &level, conn.as_ref())?; @@ -346,12 +350,19 @@ fn catalogue(s: &mut Session, c: &Connection, text: &str, out: &mut Value) -> Re Some(json!({"key": node, "node": node, "name": if name.is_empty() { url.as_str() } else { name.as_str() }, "what": "service", "detail": kind})) } "download" => { + // Like a service's node, a link that is too long to follow is left out. + if !node_ok(&url) { + dropped = true; + return None; + } let last = url.split(['?', '#']).next().unwrap_or("").trim_end_matches('/').rsplit('/').next().unwrap_or(""); + let shown: String = if name.is_empty() { last } else { name.as_str() }.chars().take(200).collect(); + let named: String = title.chars().take(200).collect(); Some(json!({ "key": url, - "name": if name.is_empty() { last } else { name.as_str() }, + "name": shown, "what": "download", - "add": {"command": "layer.addUrl", "params": {"url": url, "name": title}}, + "add": {"command": "layer.addUrl", "params": {"url": url, "name": named}}, })) } _ => None, diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index cce8f26f7..b8f3450b2 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -6801,6 +6801,18 @@ fn wcs_coverages_and_catalogue_records() { ); let full = s.run("service.coverage", &json!({"url": url, "layer": "soc", "text": caps, "describe": describe, "bbox": "full"})).unwrap(); assert_eq!(full["bbox"], json!([5.0, 49.2, 5.8, 50.0])); + // The same coverage browsed from a saved connection: the add it lists runs as it is. + s.run("connection.new", &json!({"kind": "wcs", "address": url, "name": "Soils"})).unwrap(); + let browsed = s.run("connection.browse", &json!({"id": "Soils", "answer": caps})).unwrap(); + let child = &browsed["children"][0]; + assert_eq!((child["what"].as_str(), child["add"]["command"].as_str()), (Some("coverage"), Some("service.add")), "{browsed}"); + let mut listed = child["add"]["params"].clone(); + listed["text"] = json!(caps); + listed["describe"] = json!(describe); + listed["bbox"] = json!("full"); + let layers = s.map().layers.len(); + s.run("service.add", &listed).unwrap(); + assert_eq!(s.map().layers.len(), layers + 1); let (err, _) = serve_file(b"no".to_vec(), false); let p = json!({"url": format!("{err}/wcs"), "layer": "soc", "text": caps, "describe": describe, "bbox": "full"}); assert!(s.run("service.add", &p).unwrap_err().contains("error")); diff --git a/crates/ui-egui/src/connectiontree.rs b/crates/ui-egui/src/connectiontree.rs index 4669b9b54..f952a2885 100644 --- a/crates/ui-egui/src/connectiontree.rs +++ b/crates/ui-egui/src/connectiontree.rs @@ -466,7 +466,11 @@ fn dataset(app: &mut RustGisApp, ui: &mut Ui, c: &Connection, listing: &Listing, r }) .inner; - let mut hover = tl!("Click to select, double-click to add, right-click for more").to_string(); + // A dataset without an add (an OGC API feature collection, imported together) is added + // with options only. + let addable = child.get("add").is_some_and(Value::is_object); + let hint = if addable { tl!("Click to select, double-click to add, right-click for more") } else { tl!("Right-click for more") }; + let mut hover = hint.to_string(); if let Some(detail) = child["detail"].as_str().filter(|d| !d.is_empty()) { hover = format!("{hover}\n{detail}"); } @@ -474,7 +478,7 @@ fn dataset(app: &mut RustGisApp, ui: &mut Ui, c: &Connection, listing: &Listing, let (mut add_now, mut with_options, mut add_selected, mut clear) = (false, false, false, false); let selected = app.ui.connection_tree.picks.len(); r.context_menu(|ui| { - if ui.button(tl!("Add")).clicked() { + if addable && ui.button(tl!("Add")).clicked() { add_now = true; ui.close(); } @@ -494,12 +498,12 @@ fn dataset(app: &mut RustGisApp, ui: &mut Ui, c: &Connection, listing: &Listing, } } }); - if r.double_clicked() { + if addable && r.double_clicked() { // The first click selected it: a double-click adds it alone. let key = child["key"].as_str().unwrap_or(""); app.ui.connection_tree.picks.retain(|p| !(p.conn == c.id && p.key == key)); add_now = true; - } else if r.clicked() { + } else if addable && r.clicked() { toggle(app, c, child, listing.source.as_ref()); } if clear { From 2fe961b3b15d514d3879c881c91d6e3bedf9364f Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Thu, 22 Oct 2026 20:28:53 +0200 Subject: [PATCH 5/9] test(connections): cover the answer size limit, long catalogue links and grouped OGC API features --- crates/engine/src/cmd/connection_browse.rs | 5 ----- crates/engine/src/tests.rs | 20 ++++++++++++++++++++ 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/crates/engine/src/cmd/connection_browse.rs b/crates/engine/src/cmd/connection_browse.rs index 41ffa2793..bd23f7970 100644 --- a/crates/engine/src/cmd/connection_browse.rs +++ b/crates/engine/src/cmd/connection_browse.rs @@ -350,11 +350,6 @@ fn catalogue(s: &mut Session, c: &Connection, text: &str, out: &mut Value) -> Re Some(json!({"key": node, "node": node, "name": if name.is_empty() { url.as_str() } else { name.as_str() }, "what": "service", "detail": kind})) } "download" => { - // Like a service's node, a link that is too long to follow is left out. - if !node_ok(&url) { - dropped = true; - return None; - } let last = url.split(['?', '#']).next().unwrap_or("").trim_end_matches('/').rsplit('/').next().unwrap_or(""); let shown: String = if name.is_empty() { last } else { name.as_str() }.chars().take(200).collect(); let named: String = title.chars().take(200).collect(); diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index b8f3450b2..9e645cb92 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -9107,6 +9107,26 @@ fn connection_browse_levels_and_adds() { new(&mut s, json!({"kind": "postgres", "address": "postgres:host=127.0.0.1 port=9 dbname=x user=u", "name": "Pg"})); assert!(!s.run("connection.browse", &json!({"id": "Pg"})).unwrap_err().is_empty()); } + // A catalogue's download link too long to be one (the parser keeps addresses to 2000 bytes) is not listed. + let long_link = format!("https://files.example.org/{}.tif", "a".repeat(5000)); + let records = format!( + r#"Files + https://files.example.org/a.tif{long_link}"# + ); + let v = s.run("connection.browse", &json!({"id": "Cat", "answer": records})).unwrap(); + let links = v["children"][0]["children"].as_array().unwrap(); + assert_eq!(links.len(), 1, "{v}"); + assert_eq!(links[0]["add"]["params"]["url"], "https://files.example.org/a.tif"); + assert!(v.get("truncated").is_none(), "{v}"); + // An answer larger than a service document may be is refused before it is parsed. + let huge = "x".repeat(65 << 20); + assert!(s.run("connection.browse", &json!({"id": "W", "answer": huge})).unwrap_err().contains("too large")); + // An OGC API's feature collections are imported together: listed, with options, not added alone. + new(&mut s, json!({"kind": "ogcapi", "address": "https://o.example/collections", "name": "Ogc"})); + let collections = r#"{"collections":[{"id":"lakes","title":"Lakes","links":[{"rel":"items","type":"application/geo+json","href":"https://o.example/collections/lakes/items"}]}]}"#; + let v = s.run("connection.browse", &json!({"id": "Ogc", "answer": collections})).unwrap(); + assert_eq!((v["children"][0]["what"].as_str(), v["children"][0]["options"].as_bool()), (Some("features"), Some(true)), "{v}"); + assert!(v["children"][0].get("add").is_none(), "{v}"); let saved = s.connections.connections.len(); assert!(saved >= 5); s.run("connection.browse", &json!({"id": "W", "answer": wms_caps})).unwrap(); From 4c3570641d4abe1c0ec54ff1982da3522561d8be Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Thu, 22 Oct 2026 20:41:10 +0200 Subject: [PATCH 6/9] feat(connections): list a static STAC catalog's items, each opened for its assets A static catalog or collection now lists its item links as nodes next to its child catalogs; opening an item reads its own file and lists the assets that can be added. --- crates/engine/src/cmd/connection_browse.rs | 95 +++++++++++++--------- crates/engine/src/tests.rs | 31 +++++++ crates/tiles/src/stac/catalog.rs | 32 ++++++++ docs/user-guide/data.md | 3 +- 4 files changed, 123 insertions(+), 38 deletions(-) diff --git a/crates/engine/src/cmd/connection_browse.rs b/crates/engine/src/cmd/connection_browse.rs index bd23f7970..722e6f026 100644 --- a/crates/engine/src/cmd/connection_browse.rs +++ b/crates/engine/src/cmd/connection_browse.rs @@ -28,6 +28,7 @@ enum Level { StacRoot { api: String }, StacCollection { api: String, id: String }, StacCatalog { url: String }, + StacItem { url: String }, S3Root, S3Prefix { bucket: String, prefix: String }, } @@ -81,6 +82,14 @@ fn level(c: &Connection, node: &str) -> Result { address::check(url, Some(&c.address))?; return Ok(Level::StacCatalog { url: url.to_string() }); } + if let Some(url) = node.strip_prefix("item:") { + // A static catalog's item: its own file, read when opened. + if kind != "stac" { + return Err(not_a_node()); + } + address::check(url, Some(&c.address))?; + return Ok(Level::StacItem { url: url.to_string() }); + } if let Some(rest) = node.strip_prefix("s3:") { let (bucket, prefix) = rest .split_once('/') @@ -116,7 +125,7 @@ fn fetch_url(s: &Session, c: &Connection, level: &Level, conn: Option<&s3::Conne Level::StacCollection { api, id } => { Some(stac::search_url(api, &stac::Search { collections: vec![id.clone()], limit: ITEMS, ..stac::Search::default() })?) } - Level::StacCatalog { url } => Some(url.clone()), + Level::StacCatalog { url } | Level::StacItem { url } => Some(url.clone()), Level::S3Root => match (super::first_bucket(c), conn) { (Some(_), _) => None, (None, Some(conn)) if conn.credentials.is_some() => Some(s3::buckets_url(conn)), @@ -196,6 +205,7 @@ pub(super) fn browse(s: &mut Session, p: &Value) -> CmdResult { Level::StacRoot { api } if stac::catalog::is_static(api) => catalogs(bytes, api)?, Level::StacRoot { .. } => collections(bytes)?, Level::StacCatalog { url } => catalogs(bytes, url)?, + Level::StacItem { url } => item_file(bytes, url)?, Level::StacCollection { .. } => items(bytes, at)?, Level::S3Root => buckets(s3::parse_buckets(bytes)?.into_iter()), Level::S3Prefix { bucket, prefix } => objects(bytes, bucket, prefix)?, @@ -388,25 +398,59 @@ fn collections(bytes: &[u8]) -> Result { )) } -/// The catalogs and collections below a static catalog's document fetched from `from`. +/// What a static catalog's document lists below it: its catalogs and collections (`child` links) +/// and its items (`item` links), each a node read when it is opened (an item's file holds its +/// assets). fn catalogs(bytes: &[u8], from: &str) -> Result { - let list = stac::catalog::parse_children(bytes, from)?; let mut dropped = false; - let children = list - .into_iter() - .take(MAX_CHILDREN + 1) - .filter_map(|c| { - let node = format!("catalog:{}", c.url); - let ok = node_ok(&node); - dropped |= !ok; - ok.then(|| json!({"key": node, "node": node, "name": c.title, "what": "catalog"})) - }) - .collect(); + let mut children = Vec::new(); + for (prefix, what, list) in [ + ("catalog", "catalog", stac::catalog::parse_children(bytes, from)?), + ("item", "item", stac::catalog::parse_item_links(bytes, from)?), + ] { + for c in list { + let node = format!("{prefix}:{}", c.url); + if node_ok(&node) { + children.push(json!({"key": node, "node": node, "name": c.title, "what": what})); + } else { + dropped = true; + } + } + } let mut found = Found::new(children); found.truncated |= dropped; Ok(found) } +/// The assets of an item that can be added, each with the `stac.addAsset` that adds it. +fn assets_of(item: &stac::Item) -> Vec { + item.assets + .iter() + .take(MAX_CHILDREN) + .filter_map(|a| { + let (key, href) = item.asset_href(Some(&a.key)).ok()?; + let mut params = json!({"asset": key, "name": format!("{} {key}", item.id)}); + match &item.url { + Some(u) => params["url"] = json!(u), + None => params["item"] = stac::item_json_with(item, false), + } + Some(json!({ + "key": href, + "name": if a.title.is_empty() { key.as_str() } else { a.title.as_str() }, + "what": "asset", + "detail": a.media_type, + "href": href, + "add": {"command": "stac.addAsset", "params": params}, + })) + }) + .collect() +} + +/// A static catalog's item (its own file): its assets. +fn item_file(bytes: &[u8], from: &str) -> Result { + Ok(Found::new(assets_of(&stac::parse_item(bytes, from)?))) +} + /// The items of a collection's search, each with the assets that can be added. fn items(bytes: &[u8], from: &str) -> Result { let (list, next) = stac::parse_items(bytes, from)?; @@ -415,30 +459,7 @@ fn items(bytes: &[u8], from: &str) -> Result { let children = list .iter() .take(MAX_CHILDREN) - .map(|item| { - let assets: Vec = item - .assets - .iter() - .take(MAX_CHILDREN) - .filter_map(|a| { - let (key, href) = item.asset_href(Some(&a.key)).ok()?; - let mut params = json!({"asset": key, "name": format!("{} {key}", item.id)}); - match &item.url { - Some(u) => params["url"] = json!(u), - None => params["item"] = stac::item_json_with(item, false), - } - Some(json!({ - "key": href, - "name": if a.title.is_empty() { key.as_str() } else { a.title.as_str() }, - "what": "asset", - "detail": a.media_type, - "href": href, - "add": {"command": "stac.addAsset", "params": params}, - })) - }) - .collect(); - json!({"key": format!("item:{}", item.id), "name": item.id, "what": "item", "detail": item.datetime, "children": assets}) - }) + .map(|item| json!({"key": format!("item:{}", item.id), "name": item.id, "what": "item", "detail": item.datetime, "children": assets_of(item)})) .collect(); Ok(Found { children, truncated, matched }) } diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 9e645cb92..042181d53 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -9069,6 +9069,37 @@ fn connection_browse_levels_and_adds() { ); assert!(v.get("truncated").is_none()); + // A static STAC catalog: catalogs and collections, then a collection's items, each opened + // for its assets. + new(&mut s, json!({"kind": "stac", "address": "https://s.example/static/catalog.json", "name": "Static"})); + let root = json!({"type": "Catalog", "id": "r", "links": [{"rel": "child", "href": "./c1/collection.json", "title": "C1"}]}); + let v = s.run("connection.browse", &json!({"id": "Static", "answer": root.to_string()})).unwrap(); + assert_eq!(v["children"][0]["node"], "catalog:https://s.example/static/c1/collection.json"); + let collection = json!({"type": "Collection", "id": "c1", "links": [ + {"rel": "item", "href": "./i1/i1.json", "title": "Scene 1"}, {"rel": "child", "href": "./sub/catalog.json", "title": "Sub"}]}); + let node = "catalog:https://s.example/static/c1/collection.json"; + let v = s.run("connection.browse", &json!({"id": "Static", "node": node, "answer": collection.to_string()})).unwrap(); + let kinds: Vec<_> = + v["children"].as_array().unwrap().iter().map(|c| (c["what"].as_str().unwrap(), c["node"].as_str().unwrap())).collect(); + assert_eq!( + kinds, + [("catalog", "catalog:https://s.example/static/c1/sub/catalog.json"), ("item", "item:https://s.example/static/c1/i1/i1.json")] + ); + let item_node = "item:https://s.example/static/c1/i1/i1.json"; + let v = s.run("connection.browse", &json!({"id": "Static", "node": item_node, "probe": true})).unwrap(); + assert_eq!(v["fetch"], "https://s.example/static/c1/i1/i1.json"); + let item = json!({"type": "Feature", "id": "i1", "collection": "c1", "bbox": [0, 0, 1, 1], "geometry": null, + "properties": {"datetime": "2024-01-01T00:00:00Z"}, + "assets": {"visual": {"href": "https://s.example/static/a.tif", "type": "image/tiff; application=geotiff"}}, + "links": [{"rel": "self", "href": "https://s.example/static/c1/i1/i1.json"}]}); + let v = s.run("connection.browse", &json!({"id": "Static", "node": item_node, "answer": item.to_string()})).unwrap(); + assert_eq!( + v["children"][0]["add"], + json!({"command": "stac.addAsset", "params": {"url": "https://s.example/static/c1/i1/i1.json", "asset": "visual", "name": "i1 visual"}}) + ); + // An item node belongs to a STAC connection, and is judged as its documents are. + assert!(s.run("connection.browse", &json!({"id": "W", "node": item_node})).is_err()); + assert!(s.run("connection.browse", &json!({"id": "Static", "node": "item:http://127.0.0.1:9/i.json"})).is_err()); // S3: buckets from the connection's options need no fetch; a bucket's level is read. new(&mut s, json!({"kind": "s3", "name": "BrowseB", "options": {"credentials": "none", "buckets": "bkt1"}})); let v = s.run("connection.browse", &json!({"id": "BrowseB", "probe": true})).unwrap(); diff --git a/crates/tiles/src/stac/catalog.rs b/crates/tiles/src/stac/catalog.rs index 1bd505334..c7a11a441 100644 --- a/crates/tiles/src/stac/catalog.rs +++ b/crates/tiles/src/stac/catalog.rs @@ -69,6 +69,22 @@ pub fn parse_children(bytes: &[u8], from: &str) -> Result, String> { Ok(out) } +/// The items (`item` links) of a catalog's or collection's document fetched from `from`: web +/// addresses only, in the order the document gives them. +pub fn parse_item_links(bytes: &[u8], from: &str) -> Result, String> { + let v = super::json(bytes)?; + if !v.is_object() || !matches!(v.get("type").and_then(Value::as_str), Some("Catalog" | "Collection")) { + return Err("the answer isn't a STAC catalog or collection".into()); + } + let mut seen = HashSet::new(); + Ok(links(&v, "item", from) + .into_iter() + .filter(|(url, _)| is_web(url) && crate::address::check(url, Some(from)).is_ok() && seen.insert(url.clone())) + .take(MAX_CHILDREN) + .map(|(url, title)| Child { title: if title.is_empty() { label_of(&url) } else { title }, url }) + .collect()) +} + /// The links of a relation, resolved against `from`: (address, title), the first /// [`MAX_LINKS`] of them (counted among that relation's alone). fn links(v: &Value, rel: &str, from: &str) -> Vec<(String, String)> { @@ -499,6 +515,22 @@ mod tests { assert_eq!(parse_children(br#"{"type": "Catalog", "links": 7}"#, "https://x.org/c.json"), Ok(vec![])); assert_eq!(label_of("https://x.org/events/catalog.json"), "events"); assert_eq!(label_of("https://x.org/events/a.json"), "a"); + // A collection lists its items by `item` links, in the order it gives them. + let coll = json!({"type": "Collection", "id": "c", "links": [ + {"rel": "item", "href": "./i2/i2.json", "title": "Second"}, + {"rel": "item", "href": "./i1/i1.json"}, + {"rel": "item", "href": "./i2/i2.json"}, + {"rel": "item", "href": "file:///etc/passwd"}, + {"rel": "child", "href": "./sub/catalog.json"}]}); + let i = parse_item_links(coll.to_string().as_bytes(), "https://x.org/c/collection.json").unwrap(); + assert_eq!( + i, + vec![ + Child { title: "Second".into(), url: "https://x.org/c/i2/i2.json".into() }, + Child { title: "i1".into(), url: "https://x.org/c/i1/i1.json".into() }, + ] + ); + assert!(parse_item_links(br#"{"type": "Feature"}"#, "https://x.org/c.json").is_err()); } #[test] diff --git a/docs/user-guide/data.md b/docs/user-guide/data.md index ee99e9085..30e51d982 100644 --- a/docs/user-guide/data.md +++ b/docs/user-guide/data.md @@ -398,7 +398,8 @@ driver. An anonymous S3 test needs at least one named bucket. `connection.browse` lists what a connection offers, a level at a time: the tables of a database (under their schemas), the layers of a service, the records of a catalogue with the services and -files they link to, the collections of a STAC API and the items and assets of a collection, or +files they link to, the collections of a STAC API and the items and assets of a collection (for a +static STAC catalog, its catalogs, collections and items, each item opened for its assets), or the buckets, folders and files of an S3 connection. Each child has a `key`, a `name` and what it is (`what`); a child with a `node` is browsed by asking again with that `node`; one with `add` names the command and parameters that add it (`service.add`, `layer.addDatabase`, From a6e39382f1a2cf2e200c6e1020682c5e358de098 Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Thu, 22 Oct 2026 20:45:24 +0200 Subject: [PATCH 7/9] test(connections): a static item's relative asset address resolves against its file --- crates/engine/src/tests.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 042181d53..01749fd22 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -9097,6 +9097,11 @@ fn connection_browse_levels_and_adds() { v["children"][0]["add"], json!({"command": "stac.addAsset", "params": {"url": "https://s.example/static/c1/i1/i1.json", "asset": "visual", "name": "i1 visual"}}) ); + // A relative asset address is resolved against the item's own file. + let relative = item.to_string().replace(r#""assets":{"#, r#""assets":{"data":{"href":"./data/b.tif","type":"image/tiff"},"#); + let v = s.run("connection.browse", &json!({"id": "Static", "node": item_node, "answer": relative})).unwrap(); + let hrefs: Vec<&str> = v["children"].as_array().unwrap().iter().filter_map(|c| c["href"].as_str()).collect(); + assert!(hrefs.contains(&"https://s.example/static/c1/i1/data/b.tif"), "{hrefs:?}"); // An item node belongs to a STAC connection, and is judged as its documents are. assert!(s.run("connection.browse", &json!({"id": "W", "node": item_node})).is_err()); assert!(s.run("connection.browse", &json!({"id": "Static", "node": "item:http://127.0.0.1:9/i.json"})).is_err()); From 1227ed5eb4d9e7171d85ce5f137423fd0c82fe50 Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Thu, 22 Oct 2026 20:59:08 +0200 Subject: [PATCH 8/9] fix(connections): address second review round - an asset's key names its item, so the same file under two items is two rows (selection and fold state no longer collide) - the documents of a browse count against the same size limit as its answer - the probe's S3 login is documented in the command's parameters --- crates/engine/src/cmd/connection_browse.rs | 9 ++++++--- crates/engine/src/cmd/connections.rs | 2 +- crates/engine/src/tests.rs | 7 +++++++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/crates/engine/src/cmd/connection_browse.rs b/crates/engine/src/cmd/connection_browse.rs index 722e6f026..880ea5720 100644 --- a/crates/engine/src/cmd/connection_browse.rs +++ b/crates/engine/src/cmd/connection_browse.rs @@ -101,6 +101,8 @@ fn level(c: &Connection, node: &str) -> Result { } /// The session's S3 connection for an S3 level, logging in with what needs no typing. +/// (A probe logs in too, as `connection.test` does: the address it names, and the signature the +/// fetch needs, depend on the connection.) fn s3_login(s: &mut Session, c: &Connection, level: &Level) -> Result, String> { if !matches!(level, Level::S3Root | Level::S3Prefix { .. }) { return Ok(None); @@ -176,8 +178,9 @@ pub(super) fn browse(s: &mut Session, p: &Value) -> CmdResult { return Err("node is a child's node, as connection.browse lists it".into()); } // The app's answers come from a fetch the cache already limits; a caller's may be any size. - if ps(p, "answer").is_some_and(|a| a.len() > super::super::service::MAX_GROUP_BYTES) { - return Err("answer is too large to read".into()); + let documents = p.get("documents").and_then(Value::as_array).into_iter().flatten().filter_map(|d| ps(d, "text")).map(str::len); + if ps(p, "answer").map(str::len).into_iter().chain(documents).sum::() > super::super::service::MAX_GROUP_BYTES { + return Err("answer and documents are too large to read".into()); } let level = level(&c, node)?; let conn = s3_login(s, &c, &level)?; @@ -435,7 +438,7 @@ fn assets_of(item: &stac::Item) -> Vec { None => params["item"] = stac::item_json_with(item, false), } Some(json!({ - "key": href, + "key": format!("{}#{key}", item.url.as_deref().unwrap_or(&item.id)), "name": if a.title.is_empty() { key.as_str() } else { a.title.as_str() }, "what": "asset", "detail": a.media_type, diff --git a/crates/engine/src/cmd/connections.rs b/crates/engine/src/cmd/connections.rs index f2d7765f7..b4f2e3c5a 100644 --- a/crates/engine/src/cmd/connections.rs +++ b/crates/engine/src/cmd/connections.rs @@ -33,7 +33,7 @@ pub fn commands() -> Vec { r#"{"id"?: "a connection's id or name", "kind"?: "postgres"|"mysql"|"sqlserver"|"service"|"wms"|"wmts"|"wfs"|"wcs"|"ogcapi"|"arcgis"|"csw"|"stac"|"s3", "name"?: text, "group"?: text, "address"?: text, "options"?: "an unsaved connection, as connection.new (when no id)", "password"?: text, "accessKeyId"?: text, "secretAccessKey"?: text, "sessionToken"?: "as connection.login (logs in first)", "probe"?: "true: only the address the test reads, {fetch} (null for databases, which their driver tests)", "answer"?: "non-database kinds only: what that address returned, when the app fetched it (no network request then); databases ignore answer and test through their driver"}"#, ), C::new("connection.browse", "Browse Connection", "Home › Add › Data Sources", browse::browse).view().params( - r#"{"id": "a connection's id or name", "node"?: "a child's node, as a listing gives it (the connection's top level when absent)", "probe"?: "true: only the address this level reads, {fetch} (null: read without a fetch, or by the database driver)", "answer"?: "what that address returned, when the app fetched it", "documents"?: [{"url": "a further document a listing asked for (next)", "text": "its answer"}]} → {connection, node, children: [{key, name, what, detail?, node? (browse it), children? (listed already), add? {command, params}: the command that adds it}], truncated?, matched?, notes?, service? {url, type}, next? (fetch it, then ask again with answer and documents)}"#, + r#"{"id": "a connection's id or name", "node"?: "a child's node, as a listing gives it (the connection's top level when absent)", "probe"?: "true: only the address this level reads, {fetch} (null: read without a fetch, or by the database driver); an S3 connection logs in first, as connection.test does", "answer"?: "what that address returned, when the app fetched it", "documents"?: [{"url": "a further document a listing asked for (next)", "text": "its answer"}]} → {connection, node, children: [{key, name, what, detail?, node? (browse it), children? (listed already), add? {command, params}: the command that adds it}], truncated?, matched?, notes?, service? {url, type}, next? (fetch it, then ask again with answer and documents)}"#, ), ] } diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 01749fd22..9bfe9a1e1 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -9097,6 +9097,8 @@ fn connection_browse_levels_and_adds() { v["children"][0]["add"], json!({"command": "stac.addAsset", "params": {"url": "https://s.example/static/c1/i1/i1.json", "asset": "visual", "name": "i1 visual"}}) ); + // An asset's key names its item, so the same file under two items is two rows. + assert_eq!(v["children"][0]["key"], "https://s.example/static/c1/i1/i1.json#visual"); // A relative asset address is resolved against the item's own file. let relative = item.to_string().replace(r#""assets":{"#, r#""assets":{"data":{"href":"./data/b.tif","type":"image/tiff"},"#); let v = s.run("connection.browse", &json!({"id": "Static", "node": item_node, "answer": relative})).unwrap(); @@ -9157,6 +9159,11 @@ fn connection_browse_levels_and_adds() { // An answer larger than a service document may be is refused before it is parsed. let huge = "x".repeat(65 << 20); assert!(s.run("connection.browse", &json!({"id": "W", "answer": huge})).unwrap_err().contains("too large")); + assert!( + s.run("connection.browse", &json!({"id": "W", "answer": "", "documents": [{"url": "https://a.example/x", "text": huge}]})) + .unwrap_err() + .contains("too large") + ); // An OGC API's feature collections are imported together: listed, with options, not added alone. new(&mut s, json!({"kind": "ogcapi", "address": "https://o.example/collections", "name": "Ogc"})); let collections = r#"{"collections":[{"id":"lakes","title":"Lakes","links":[{"rel":"items","type":"application/geo+json","href":"https://o.example/collections/lakes/items"}]}]}"#; From a4f0e7dc9e94bec7b8a79a2e50f91957b8e47351 Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Thu, 22 Oct 2026 22:28:07 +0200 Subject: [PATCH 9/9] fix(connections): address third review round - an ArcGIS folder or service whose address is too long to follow is left out (and the list is marked truncated), as for catalogues - an API item's key names its address, so items with one id in two collections stay apart - the smoke test checks that one undo takes one layer of Add Selected --- crates/engine/src/cmd/connection_browse.rs | 15 ++++++++++++--- crates/ui-egui/tests/smoke.rs | 2 ++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/crates/engine/src/cmd/connection_browse.rs b/crates/engine/src/cmd/connection_browse.rs index 880ea5720..9d49dc822 100644 --- a/crates/engine/src/cmd/connection_browse.rs +++ b/crates/engine/src/cmd/connection_browse.rs @@ -296,6 +296,9 @@ fn service(s: &mut Session, p: &Value, out: &mut Value, url: &str, kind: Option< let offer_type = offer.get("type").and_then(Value::as_str).unwrap_or(""); if offer_type == "service" { let node = format!("service:arcgis:{}", text(&offer, "url")); + if !node_ok(&node) { + return Value::Null; + } let what = if name.ends_with('/') { "folder" } else { "service" }; return json!({"key": node, "node": node, "name": title.trim_end_matches('/'), "what": what}); } @@ -324,8 +327,14 @@ fn service(s: &mut Session, p: &Value, out: &mut Value, url: &str, kind: Option< } child }) - .collect(); - Ok(Found::new(children)) + .collect::>(); + // A folder or service whose address is too long to follow as a node is left out, and said so. + let listed = children.len(); + let children: Vec = children.into_iter().filter(|c| !c.is_null()).collect(); + let left_out = children.len() < listed; + let mut found = Found::new(children); + found.truncated |= left_out; + Ok(found) } /// A catalogue's records, each with the services and files it links to. @@ -462,7 +471,7 @@ fn items(bytes: &[u8], from: &str) -> Result { let children = list .iter() .take(MAX_CHILDREN) - .map(|item| json!({"key": format!("item:{}", item.id), "name": item.id, "what": "item", "detail": item.datetime, "children": assets_of(item)})) + .map(|item| json!({"key": format!("item:{}", item.url.as_deref().unwrap_or(&item.id)), "name": item.id, "what": "item", "detail": item.datetime, "children": assets_of(item)})) .collect(); Ok(Found { children, truncated, matched }) } diff --git a/crates/ui-egui/tests/smoke.rs b/crates/ui-egui/tests/smoke.rs index e723edf16..e488d14d4 100644 --- a/crates/ui-egui/tests/smoke.rs +++ b/crates/ui-egui/tests/smoke.rs @@ -1752,6 +1752,8 @@ fn connections_tree_browses_and_adds() { assert_eq!(h.state().session.map().tiles.len(), before + 2); assert!(h.state().ui.connection_tree.picks.is_empty()); h.state_mut().run("edit.undo", json!(null)).unwrap(); + // Each layer was its own step: one undo takes one layer. + assert_eq!(h.state().session.map().tiles.len(), before + 1); h.state_mut().run("edit.undo", json!(null)).unwrap(); assert_eq!(h.state().session.map().tiles.len(), before); // A double-click adds the one dataset, leaving nothing selected.