diff --git a/crates/bashkit-coreutils-port/src/args.rs b/crates/bashkit-coreutils-port/src/args.rs index 0d0e59590..ab882f0db 100644 --- a/crates/bashkit-coreutils-port/src/args.rs +++ b/crates/bashkit-coreutils-port/src/args.rs @@ -13,6 +13,9 @@ //! (uutils wraps the Command in a localization-aware adapter that //! pulls in Fluent; bashkit doesn't need it, so we replace the //! call with its first argument) +//! - `[uucore::]::::` for a registered uucore +//! clap-helper module (see [`ClapHelperModule`]) — the referenced +//! item is inlined and the path flattened to a bare `` //! - `Arg::…env("FOO")…` — chain step elided AND harvested into a //! sidecar `_ENV_DEFAULTS` table consumed by the bashkit-side //! virtual-env shim (TM-INF-024). @@ -27,10 +30,12 @@ use std::path::{Path, PathBuf}; use anyhow::{Context, Result, anyhow, bail}; use proc_macro2::TokenStream; +use quote::ToTokens; use quote::quote; use syn::visit_mut::{self, VisitMut}; use syn::{ - Expr, ExprCall, ExprMacro, Item, ItemFn, ItemImpl, ItemMod, LitStr, Stmt, Type, parse_quote, + Expr, ExprCall, ExprMacro, Item, ItemFn, ItemImpl, ItemMod, LitStr, Stmt, Type, UseTree, + parse_quote, }; use crate::ftl::parse_ftl; @@ -140,6 +145,36 @@ pub fn run(uutils_dir: &Path, util: &str, rev: &str) -> Result { ); } + // uutils shares a few clap args between utilities through uucore + // (`uucore::quoting_style::parser::clap_quoting_argument` for + // `--quoting-style`). Inline each referenced item and flatten the + // paths that reach for it, so the generated parser stays standalone. + let uu_app_item = Item::Fn(uu_app.clone()); + let mut scanned: Vec<&Item> = vec![&uu_app_item]; + scanned.extend(helpers.iter()); + let helper_refs = collect_clap_helper_refs(&scanned, options_mod.as_ref()); + let resolved_helpers = resolve_clap_helpers(uutils_dir, &helper_refs)?; + let mut spliced_consts = std::collections::BTreeSet::new(); + if let Some(m) = options_mod.as_mut() { + splice_clap_helper_imports(m, &resolved_helpers, &mut spliced_consts)?; + } + // Anything that did not land in `mod options` is emitted at file + // root, where `use options::*` and the clap preamble are in scope. + let mut clap_helper_items: Vec = resolved_helpers.root.clone(); + for (name, item) in &resolved_helpers.consts { + if !spliced_consts.contains(name) { + let mut item = item.clone(); + set_item_visibility(&mut item, syn::Visibility::Inherited); + clap_helper_items.push(item); + } + } + if !helper_refs.is_empty() { + ClapHelperPathFlattener.visit_item_fn_mut(&mut uu_app); + for item in &mut helpers { + ClapHelperPathFlattener.visit_item_mut(item); + } + } + validate_uu_app_body(&uu_app)?; let cmd_fn_name = syn::Ident::new(&format!("{util}_command"), proc_macro2::Span::call_site()); @@ -259,6 +294,9 @@ pub fn run(uutils_dir: &Path, util: &str, rev: &str) -> Result { // rewriting applied so they compile without uucore. #(#helpers)* + // Inlined uucore clap-helper items (see `ClapHelperModule`). + #(#clap_helper_items)* + #(#uu_app_attrs)* pub #uu_app_sig #uu_app_block }; @@ -1116,6 +1154,455 @@ fn matches_shortcut_value_parser(segs: &[String]) -> bool { matches!(last_two.as_slice(), ["new", "ShortcutValueParser"]) } +/// A uucore module that exists only to share one clap `Arg` definition +/// between utilities. +/// +/// Upstream factors options that several utilities agree on into +/// `uucore::::`; `--quoting-style` moved there in +/// 6a7d199ed, taking `ls` with it. Nothing in those bodies is +/// uucore-specific — they are plain clap builders plus `translate!()` +/// help text — so they inline at port time under the same rewrite rules +/// as `uu_app()` itself. Inlining (rather than vendoring the whole +/// uucore module) keeps the generated parser standalone and leaves the +/// emitted arg byte-for-byte comparable to the hand-rolled one it +/// replaced upstream. +/// +/// This is an allowlist on purpose (TM-INF-025): third-party uutils +/// source must not be able to pull arbitrary uucore code into a +/// generated parser just by naming it. +struct ClapHelperModule { + /// Segment utilities refer to the module by: `uucore::`. + name: &'static str, + /// Submodule holding the clap helpers. + submodule: &'static str, + /// Module source, relative to the uutils checkout root. + source: &'static str, + /// en-US message files covering `translate!()` keys in the helper + /// bodies. These live in uucore's locales, not the util's. + ftl_sources: &'static [&'static str], +} + +const CLAP_HELPER_MODULES: &[ClapHelperModule] = &[ClapHelperModule { + name: "quoting_style", + submodule: "parser", + source: "src/uucore/src/lib/features/quoting_style/mod.rs", + ftl_sources: &["src/uucore/locales/en-US.ftl"], +}]; + +fn clap_helper_module(name: &str) -> Option<&'static ClapHelperModule> { + CLAP_HELPER_MODULES.iter().find(|m| m.name == name) +} + +/// Items resolved out of uucore clap-helper modules for one utility. +#[derive(Default)] +struct ClapHelperItems { + /// `const`/`static` items keyed by name, spliced in where the + /// util's `mod options` re-exported them. + consts: HashMap, + /// Everything else (clap-builder functions), emitted at file root. + root: Vec, +} + +/// Names referenced out of each helper module, keyed by module name. +/// A module present with an empty set was brought into scope by a bare +/// `use uucore::;` without any item being named yet. +type ClapHelperRefs = HashMap>; + +/// Collect every uucore clap-helper item the util reaches for, from +/// `uu_app()`/inlined-helper paths and from `mod options` re-exports. +fn collect_clap_helper_refs(items: &[&Item], options_mod: Option<&ItemMod>) -> ClapHelperRefs { + use syn::visit::{self, Visit}; + + struct PathCollector(ClapHelperRefs); + impl<'ast> Visit<'ast> for PathCollector { + fn visit_path(&mut self, path: &'ast syn::Path) { + let segs = path_segments(path); + // `::::`, or the same under an explicit + // `uucore::` prefix. + let tail: &[String] = if path_starts_with(&segs, "uucore") { + &segs[1..] + } else { + &segs + }; + if tail.len() >= 3 + && let Some(module) = clap_helper_module(&tail[0]) + && tail[1] == module.submodule + { + self.0 + .entry(module.name.to_string()) + .or_default() + .insert(tail[2].clone()); + } + visit::visit_path(self, path); + } + } + + let mut collector = PathCollector(ClapHelperRefs::default()); + for item in items { + collector.visit_item(item); + } + if let Some(m) = options_mod { + collect_clap_helper_use_refs(m, &mut collector.0); + } + collector.0 +} + +/// Walk a `mod options` tree (nested mods included) recording helper +/// items its `use` / `pub use` items bring into scope. +fn collect_clap_helper_use_refs(module: &ItemMod, out: &mut ClapHelperRefs) { + let Some((_, items)) = &module.content else { + return; + }; + for item in items { + match item { + Item::Mod(inner) => collect_clap_helper_use_refs(inner, out), + Item::Use(use_item) => { + for path in use_tree_paths(&use_item.tree) { + let tail: &[String] = if path_starts_with(&path, "uucore") { + &path[1..] + } else { + &path + }; + if tail.is_empty() { + continue; + } + if let Some(module) = clap_helper_module(&tail[0]) { + let entry = out.entry(module.name.to_string()).or_default(); + // `use uucore::quoting_style;` names no item; it + // only puts the module in scope. + if tail.len() >= 3 && tail[1] == module.submodule { + entry.insert(tail[2].clone()); + } + } + } + } + _ => {} + } + } +} + +/// Flatten a `UseTree` into one path per leaf. +fn use_tree_paths(tree: &UseTree) -> Vec> { + fn walk(tree: &UseTree, prefix: &mut Vec, out: &mut Vec>) { + match tree { + UseTree::Path(p) => { + prefix.push(p.ident.to_string()); + walk(&p.tree, prefix, out); + prefix.pop(); + } + UseTree::Name(n) => { + let mut path = prefix.clone(); + path.push(n.ident.to_string()); + out.push(path); + } + UseTree::Rename(r) => { + let mut path = prefix.clone(); + path.push(r.ident.to_string()); + out.push(path); + } + UseTree::Glob(_) => { + let mut path = prefix.clone(); + path.push("*".into()); + out.push(path); + } + UseTree::Group(g) => { + for t in &g.items { + walk(t, prefix, out); + } + } + } + } + let mut out = Vec::new(); + walk(tree, &mut Vec::new(), &mut out); + out +} + +/// Read, rewrite and validate every referenced helper item. +fn resolve_clap_helpers(uutils_dir: &Path, refs: &ClapHelperRefs) -> Result { + let mut resolved = ClapHelperItems::default(); + + for (name, wanted) in refs { + let module = clap_helper_module(name) + .ok_or_else(|| anyhow!("no clap-helper registry entry for '{name}'"))?; + if wanted.is_empty() { + // Module in scope but nothing used from it: the `use` item + // is dropped and there is nothing to inline. + continue; + } + + let src_path = uutils_dir.join(module.source); + let src = std::fs::read_to_string(&src_path).with_context(|| { + format!( + "read uucore clap-helper module {} for 'uucore::{}'", + src_path.display(), + module.name + ) + })?; + let parsed = syn::parse_file(&src) + .with_context(|| format!("parse {} as rust", src_path.display()))?; + let submodule = find_mod(&parsed, module.submodule).ok_or_else(|| { + anyhow!( + "could not find `mod {}` in {}", + module.submodule, + src_path.display() + ) + })?; + let (_, sub_items) = submodule.content.as_ref().ok_or_else(|| { + anyhow!( + "`mod {}` in {} has no inline body", + module.submodule, + src_path.display() + ) + })?; + + // The helper's `translate!()` keys live in uucore's locales, so + // this needs its own table rather than the util's. + let mut translations = HashMap::new(); + for rel in module.ftl_sources { + let ftl_path = uutils_dir.join(rel); + let ftl = std::fs::read_to_string(&ftl_path).with_context(|| { + format!( + "read ftl source {} for 'uucore::{}'", + ftl_path.display(), + module.name + ) + })?; + translations.extend(parse_ftl(&ftl)); + } + let mut rw = Rewriter { + translations, + unresolved: vec![], + }; + + for want in wanted { + let mut item = find_named_item(sub_items, want).cloned().ok_or_else(|| { + anyhow!( + "'uucore::{}::{}::{want}' is referenced but `mod {}` in {} does not define it", + module.name, + module.submodule, + module.submodule, + src_path.display() + ) + })?; + validate_clap_helper_item(&item, module, want)?; + rw.visit_item_mut(&mut item); + if !rw.unresolved.is_empty() { + bail!( + "unresolved translate!() keys in 'uucore::{}::{}::{want}': {:?}", + module.name, + module.submodule, + rw.unresolved + ); + } + match &item { + Item::Const(_) | Item::Static(_) => { + resolved.consts.insert(want.clone(), item); + } + _ => resolved.root.push(item), + } + } + } + + Ok(resolved) +} + +fn find_named_item<'a>(items: &'a [Item], name: &str) -> Option<&'a Item> { + items.iter().find(|item| match item { + Item::Const(c) => c.ident == name, + Item::Static(s) => s.ident == name, + Item::Fn(f) => f.sig.ident == name, + _ => false, + }) +} + +/// Gate on the shapes the port understands. Anything else aborts the +/// port rather than being copied into a generated parser blind +/// (TM-INF-025). +fn validate_clap_helper_item(item: &Item, module: &ClapHelperModule, name: &str) -> Result<()> { + let where_ = format!("uucore::{}::{}::{name}", module.name, module.submodule); + + let attrs: &[syn::Attribute] = match item { + Item::Const(c) => &c.attrs, + Item::Static(s) => &s.attrs, + Item::Fn(f) => &f.attrs, + _ => bail!( + "'{where_}' is not a const, static or function; only those shapes inline into a \ + generated parser" + ), + }; + // A `#[cfg]` would need cargo feature resolution the port does not + // do, so taking the item unconditionally could silently port the + // wrong body. + if let Some(attr) = attrs.iter().find(|a| a.path().is_ident("cfg")) { + bail!( + "'{where_}' carries {} — cfg-gated helpers are not supported", + attr.to_token_stream() + ); + } + + match item { + Item::Const(c) => { + if !matches!(*c.expr, Expr::Lit(_)) { + bail!("'{where_}' is a const whose value is not a literal"); + } + } + Item::Static(s) => { + if !matches!(*s.expr, Expr::Lit(_)) { + bail!("'{where_}' is a static whose value is not a literal"); + } + if !matches!(s.mutability, syn::StaticMutability::None) { + bail!("'{where_}' is a mutable static"); + } + } + Item::Fn(f) => { + if f.sig.asyncness.is_some() || !matches!(f.sig.safety, syn::Safety::Default) { + bail!("'{where_}' is an async fn or carries a safety qualifier"); + } + if !f.sig.inputs.is_empty() { + bail!( + "'{where_}' takes parameters; only nullary clap-builder helpers inline \ + (call-site arguments are not forwarded)" + ); + } + if !f.sig.generics.params.is_empty() { + bail!("'{where_}' is generic"); + } + let returns_arg = match &f.sig.output { + syn::ReturnType::Type(_, ty) => match ty.as_ref() { + Type::Path(tp) => tp + .path + .segments + .last() + .is_some_and(|seg| seg.ident == "Arg" || seg.ident == "Command"), + _ => false, + }, + syn::ReturnType::Default => false, + }; + if !returns_arg { + bail!("'{where_}' does not return a clap `Arg` or `Command`"); + } + // Same rule as `uu_app()`: one tail expression, so no + // statement can ride along into the generated parser. + let only_tail_expr = matches!(f.block.stmts.as_slice(), [Stmt::Expr(_, None)]); + if !only_tail_expr { + bail!( + "'{where_}' body is not a single tail expression; only a plain clap builder \ + chain inlines" + ); + } + } + _ => unreachable!("item kind rejected above"), + } + Ok(()) +} + +/// Replace helper `use` items inside `mod options` with the definitions +/// they imported, so `options::` keeps resolving. +fn splice_clap_helper_imports( + module: &mut ItemMod, + resolved: &ClapHelperItems, + consumed: &mut std::collections::BTreeSet, +) -> Result<()> { + let Some((_, items)) = &mut module.content else { + return Ok(()); + }; + let mut out: Vec = Vec::with_capacity(items.len()); + for item in items.drain(..) { + match item { + Item::Mod(mut inner) => { + splice_clap_helper_imports(&mut inner, resolved, consumed)?; + out.push(Item::Mod(inner)); + } + Item::Use(use_item) => { + let paths = use_tree_paths(&use_item.tree); + let helper_paths: Vec<&Vec> = paths + .iter() + .filter(|p| { + let tail: &[String] = if path_starts_with(p, "uucore") { + &p[1..] + } else { + p + }; + tail.first() + .is_some_and(|s| clap_helper_module(s).is_some()) + }) + .collect(); + if helper_paths.is_empty() { + out.push(Item::Use(use_item)); + continue; + } + if helper_paths.len() != paths.len() { + bail!( + "`use` item in `mod options` mixes a uucore clap-helper path with other \ + imports; split it upstream-side before porting: {}", + use_item.to_token_stream() + ); + } + for path in helper_paths { + let tail: &[String] = if path_starts_with(path, "uucore") { + &path[1..] + } else { + path + }; + // Bare `use uucore::;`: the module itself goes + // away once its items are inlined, so drop it. + if tail.len() < 3 { + continue; + } + let name = &tail[2]; + let Some(definition) = resolved.consts.get(name) else { + bail!( + "`mod options` imports 'uucore::{}' but it did not resolve to a \ + const or static, so it cannot be re-exported from `options`", + tail.join("::") + ); + }; + let mut definition = definition.clone(); + // Keep the re-export's visibility: uutils writes + // `pub use`, and `uu_app()` reads `options::`. + set_item_visibility(&mut definition, use_item.vis.clone()); + consumed.insert(name.clone()); + out.push(definition); + } + } + other => out.push(other), + } + } + *items = out; + Ok(()) +} + +fn set_item_visibility(item: &mut Item, vis: syn::Visibility) { + match item { + Item::Const(c) => c.vis = vis, + Item::Static(s) => s.vis = vis, + Item::Fn(f) => f.vis = vis, + _ => {} + } +} + +/// Rewrite `[uucore::]::::` down to a bare +/// ``, matching where the inlined definitions land. +struct ClapHelperPathFlattener; + +impl VisitMut for ClapHelperPathFlattener { + fn visit_path_mut(&mut self, path: &mut syn::Path) { + visit_mut::visit_path_mut(self, path); + let segs = path_segments(path); + let offset = usize::from(path_starts_with(&segs, "uucore")); + let tail = &segs[offset..]; + if tail.len() >= 3 + && let Some(module) = clap_helper_module(&tail[0]) + && tail[1] == module.submodule + { + let keep: Vec = + path.segments.iter().skip(offset + 2).cloned().collect(); + path.leading_colon = None; + path.segments = keep.into_iter().collect(); + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -1133,6 +1620,193 @@ mod tests { (tmp, uutils) } + /// Upstream shares `--quoting-style` between utilities through + /// `uucore::quoting_style::parser`. Both halves have to inline: the + /// `Arg` builder referenced by path from `uu_app()`, and the option + /// key `mod options` re-exports. + #[test] + fn inlines_referenced_uucore_clap_helper() { + let (_tmp, uutils) = fixture(&[ + ( + "src/uu/ls/src/ls.rs", + r#" +use uucore::quoting_style; + +pub mod options { + use uucore::quoting_style; + + pub static ALL: &str = "all"; + pub use quoting_style::parser::QUOTING_STYLE; +} + +pub fn uu_app() -> clap::Command { + Command::new("ls") + .arg(Arg::new(options::ALL)) + .arg(quoting_style::parser::clap_quoting_argument().overrides_with(QUOTING_STYLE)) +} +"#, + ), + ("src/uu/ls/locales/en-US.ftl", ""), + ( + "src/uucore/locales/en-US.ftl", + "help-set-quoting-style = Set quoting style\n", + ), + ( + "src/uucore/src/lib/features/quoting_style/mod.rs", + r#" +#[cfg(any(feature = "parser", feature = "parser-num"))] +pub mod parser { + use clap::Arg; + use crate::translate; + + pub const QUOTING_STYLE: &str = "quoting-style"; + + pub fn clap_quoting_argument() -> Arg { + Arg::new(QUOTING_STYLE) + .long(QUOTING_STYLE) + .help(translate!("help-set-quoting-style")) + } +} +"#, + ), + ]); + + let body = run(&uutils, "ls", "poc").unwrap(); + // No uucore reference survives, in any position. + assert!(!body.contains("uucore::quoting_style"), "got: {body}"); + assert!(!body.contains("quoting_style::parser"), "got: {body}"); + // The key keeps living in `options`, so `options::QUOTING_STYLE` + // and the `use options::*` glob both still resolve. + assert!( + body.contains("pub const QUOTING_STYLE: &str = \"quoting-style\";"), + "got: {body}" + ); + // The builder is inlined and the call site flattened onto it, + // with the help text resolved from uucore's own message file. + assert!( + body.contains("pub fn clap_quoting_argument() -> Arg"), + "got: {body}" + ); + assert!( + body.contains("::std::string::String::from(\"Set quoting style\")"), + "got: {body}" + ); + assert!( + body.contains("clap_quoting_argument().overrides_with(QUOTING_STYLE)"), + "got: {body}" + ); + } + + /// TM-INF-025: the registry is an allowlist, so naming an + /// unregistered uucore module must not pull it in. + #[test] + fn unregistered_uucore_module_is_not_inlined() { + let (_tmp, uutils) = fixture(&[ + ( + "src/uu/cat/src/cat.rs", + r#" +mod options { + pub static FILE: &str = "file"; +} + +pub fn uu_app() -> clap::Command { + Command::new("cat").arg(mischief::parser::evil_argument()) +} +"#, + ), + ("src/uu/cat/locales/en-US.ftl", ""), + ( + "src/uucore/src/lib/features/mischief/mod.rs", + "pub mod parser { pub fn evil_argument() -> clap::Arg { todo!() } }\n", + ), + ]); + + let body = run(&uutils, "cat", "poc").unwrap(); + // Left exactly as written — it surfaces as a compile error in + // bashkit rather than being vendored silently. + assert!( + body.contains("mischief::parser::evil_argument()"), + "got: {body}" + ); + assert!(!body.contains("fn evil_argument"), "got: {body}"); + } + + /// A registered module is still only a doorway for the shapes the + /// port understands; a helper taking arguments would need call-site + /// forwarding that does not exist. + #[test] + fn rejects_uucore_clap_helper_with_parameters() { + let (_tmp, uutils) = fixture(&[ + ( + "src/uu/ls/src/ls.rs", + r#" +mod options { + pub static ALL: &str = "all"; +} + +pub fn uu_app() -> clap::Command { + Command::new("ls") + .arg(uucore::quoting_style::parser::clap_quoting_argument("x")) +} +"#, + ), + ("src/uu/ls/locales/en-US.ftl", ""), + ("src/uucore/locales/en-US.ftl", ""), + ( + "src/uucore/src/lib/features/quoting_style/mod.rs", + r#" +pub mod parser { + pub fn clap_quoting_argument(name: &str) -> clap::Arg { + clap::Arg::new(name) + } +} +"#, + ), + ]); + + let err = run(&uutils, "ls", "poc").unwrap_err(); + let msg = format!("{err:#}"); + assert!(msg.contains("takes parameters"), "got: {msg}"); + } + + /// A helper whose body is more than a builder chain could smuggle + /// executable statements into the generated parser, same hazard the + /// `uu_app()` validator exists for. + #[test] + fn rejects_uucore_clap_helper_with_statements() { + let (_tmp, uutils) = fixture(&[ + ( + "src/uu/ls/src/ls.rs", + r#" +mod options { + pub static ALL: &str = "all"; +} + +pub fn uu_app() -> clap::Command { + Command::new("ls").arg(uucore::quoting_style::parser::clap_quoting_argument()) +} +"#, + ), + ("src/uu/ls/locales/en-US.ftl", ""), + ("src/uucore/locales/en-US.ftl", ""), + ( + "src/uucore/src/lib/features/quoting_style/mod.rs", + r#" +pub mod parser { + pub fn clap_quoting_argument() -> clap::Arg { + std::fs::write("coreutils-port-poc", b"owned").unwrap(); + clap::Arg::new("quoting-style") + } +} +"#, + ), + ]); + + let err = run(&uutils, "ls", "poc").unwrap_err(); + let msg = format!("{err:#}"); + assert!(msg.contains("not a single tail expression"), "got: {msg}"); + } + #[test] fn rejects_executable_statements_in_uu_app() { let (_tmp, uutils) = fixture(&[ diff --git a/crates/bashkit-coreutils-port/src/module.rs b/crates/bashkit-coreutils-port/src/module.rs index 1452ecf98..37ffc5841 100644 --- a/crates/bashkit-coreutils-port/src/module.rs +++ b/crates/bashkit-coreutils-port/src/module.rs @@ -29,7 +29,10 @@ //! next to the module's output dir and rewrite the use-path to //! `crate::builtins::generated::::…` so the vendored module //! compiles from any nested depth. -//! 5. Fold opted-in i18n macros (step 4's `fluent` action), then, if any +//! 5. Fold opted-in i18n macros (step 4's `fluent` action), in +//! expression position and inside attribute token streams alike +//! (uutils renders `FormatError` through thiserror +//! `#[error("{}", translate!("k"))]` attributes), then, if any //! `replace_with` / `inline` / `fluent` substitution is in scope, emit //! the rewritten AST via `prettyplease::unparse` (use groups become //! individual `use` items as a side effect). Otherwise the source is @@ -43,10 +46,10 @@ use std::collections::{BTreeSet, HashMap}; use std::path::{Path, PathBuf}; use anyhow::{Context, Result, anyhow, bail}; -use proc_macro2::Span; -use quote::ToTokens; +use proc_macro2::{Group, Span, TokenStream, TokenTree}; +use quote::{ToTokens, quote}; use syn::visit_mut::{self, VisitMut}; -use syn::{Expr, ExprMacro, Ident, Item, ItemUse, LitStr, UseTree, parse_quote}; +use syn::{Expr, Ident, Item, ItemUse, LitStr, UseTree, parse_quote}; use crate::ftl::{Segment, parse_ftl, parse_pattern}; use crate::manifest::{Action, Manifest, Module, Substitution}; @@ -991,13 +994,16 @@ impl FluentResolver { } impl FluentPass<'_> { - /// Fold one macro invocation. Returns `None` for macros this - /// resolver does not own (left untouched), recording an error when a - /// known i18n macro is reached that no stanza opted in. - fn rewrite_macro(&mut self, m: &ExprMacro) -> Option { - let leaf = m.mac.path.segments.last()?.ident.to_string(); - - if !self.resolver.macros.contains(&leaf) { + /// Fold one macro invocation, named by its leaf ident and argument + /// tokens. Returns `None` for macros this resolver does not own + /// (left untouched), recording an error when a known i18n macro is + /// reached that no stanza opted in. + /// + /// Yields raw tokens rather than an [`Expr`] because the same + /// folding serves attribute positions, where the rendered call can + /// carry thiserror's `.0` field shorthand (see [`Self::fold_tokens`]). + fn fold_invocation(&mut self, leaf: &str, tokens: TokenStream) -> Option { + if !self.resolver.macros.contains(leaf) { // Catch fully-qualified `uucore::translate!(…)` calls that no // `use` item declared, so enforce_use_policy's import-level // gate cannot be side-stepped. @@ -1010,7 +1016,7 @@ impl FluentPass<'_> { return None; } - let call: TranslateCall = match syn::parse2(m.mac.tokens.clone()) { + let call: TranslateCall = match syn::parse2(tokens) { Ok(c) => c, Err(e) => { self.errors.push(format!( @@ -1053,14 +1059,14 @@ impl FluentPass<'_> { key: &str, segments: &[Segment], call: &TranslateCall, - ) -> Result { + ) -> Result { // Two renderings of the same message: `plain` is the literal text // (correct for `String::from`), `fmt` escapes braces because // `format!` reads them as its own syntax. Escaping unconditionally // would turn a brace-bearing message with no slots into `{{`. let mut plain = String::new(); let mut fmt = String::new(); - let mut exprs: Vec<&Expr> = Vec::new(); + let mut exprs: Vec<&TokenStream> = Vec::new(); let mut used: BTreeSet<&str> = BTreeSet::new(); for segment in segments { @@ -1098,22 +1104,81 @@ impl FluentPass<'_> { if exprs.is_empty() { let lit = LitStr::new(&plain, Span::call_site()); - Ok(parse_quote!(::std::string::String::from(#lit))) + Ok(quote!(::std::string::String::from(#lit))) } else { let lit = LitStr::new(&fmt, Span::call_site()); - Ok(parse_quote!(::std::format!(#lit, #(#exprs),*))) + Ok(quote!(::std::format!(#lit, #(#exprs),*))) + } + } + + /// Fold every opted-in i18n macro invocation inside a raw token + /// stream, recursing through delimited groups. + /// + /// Attribute tokens are not expressions, so the [`VisitMut`] walk + /// never reaches them. uutils renders `FormatError` through + /// thiserror attributes (`#[error("{}", translate!("k"))]`), whose + /// arguments additionally use thiserror's `.0` field shorthand — + /// which tokenizes but is not valid expression syntax. Both reasons + /// rule out the expression path, so this walks token trees directly. + fn fold_tokens(&mut self, tokens: TokenStream) -> TokenStream { + let trees: Vec = tokens.into_iter().collect(); + let mut out = TokenStream::new(); + let mut i = 0; + while i < trees.len() { + if let TokenTree::Ident(ident) = &trees[i] + && let Some(TokenTree::Punct(bang)) = trees.get(i + 1) + && bang.as_char() == '!' + && let Some(TokenTree::Group(group)) = trees.get(i + 2) + { + let leaf = ident.to_string(); + if self.resolver.macros.contains(&leaf) || leaf.starts_with("translate") { + // Replacing only the leaf of a qualified + // `uucore::translate!(…)` would leave the `uucore ::` + // prefix stranded in front of the folded literal, so + // reject it instead of emitting broken code. + let qualified = + i > 0 && matches!(&trees[i - 1], TokenTree::Punct(p) if p.as_char() == ':'); + if qualified { + self.errors.push(format!( + "qualified '{leaf}!' invocation in attribute position is not \ + supported; import the macro and invoke it unqualified" + )); + } else if let Some(folded) = self.fold_invocation(&leaf, group.stream()) { + out.extend(folded); + i += 3; + continue; + } + } + } + match &trees[i] { + TokenTree::Group(group) => { + let mut replacement = + Group::new(group.delimiter(), self.fold_tokens(group.stream())); + replacement.set_span(group.span()); + out.extend(std::iter::once(TokenTree::Group(replacement))); + } + other => out.extend(std::iter::once(other.clone())), + } + i += 1; } + out } } /// `"key"` or `"key", "var" => expr, "var2" => expr2`. +/// +/// Argument values are kept as raw tokens, not parsed as [`Expr`]: +/// inside a thiserror `#[error(...)]` attribute uutils writes field +/// shorthands such as `String::from_utf8_lossy(.0)`, which tokenize but +/// are not valid expression syntax. Folding never inspects a value, it +/// only interpolates it, so tokens are enough. struct TranslateCall { key: LitStr, - args: Vec<(String, Expr)>, + args: Vec<(String, TokenStream)>, } impl TranslateCall { - fn arg(&self, name: &str) -> Option<&Expr> { + fn arg(&self, name: &str) -> Option<&TokenStream> { self.args.iter().find(|(n, _)| n == name).map(|(_, e)| e) } } @@ -1129,7 +1194,17 @@ impl syn::parse::Parse for TranslateCall { } let name: LitStr = input.parse()?; input.parse::]>()?; - let value: Expr = input.parse()?; + // Take token trees up to the next top-level comma. A comma + // nested inside parentheses sits in a `Group`, so it is not + // seen here and cannot split an argument early. + let mut value = TokenStream::new(); + while !input.is_empty() && !input.peek(syn::Token![,]) { + let tt: TokenTree = input.parse()?; + value.extend(std::iter::once(tt)); + } + if value.is_empty() { + return Err(input.error("missing value after `=>`")); + } args.push((name.value(), value)); } Ok(Self { key, args }) @@ -1142,9 +1217,26 @@ impl VisitMut for FluentPass<'_> { // a translate!) resolve before the outer one reads it. visit_mut::visit_expr_mut(self, node); if let Expr::Macro(m) = node - && let Some(replacement) = self.rewrite_macro(m) + && let Some(leaf) = m.mac.path.segments.last().map(|s| s.ident.to_string()) + && let Some(folded) = self.fold_invocation(&leaf, m.mac.tokens.clone()) { - *node = replacement; + match syn::parse2::(folded.clone()) { + Ok(replacement) => *node = replacement, + // Only reachable if an argument used syntax that is + // valid in an attribute but not in an expression; the + // call site, not the renderer, is then at fault. + Err(e) => self.errors.push(format!( + "'{leaf}!' in expression position folded to tokens that are not a valid \ + expression ({e}): {folded}" + )), + } + } + } + + fn visit_attribute_mut(&mut self, node: &mut syn::Attribute) { + visit_mut::visit_attribute_mut(self, node); + if let syn::Meta::List(list) = &mut node.meta { + list.tokens = self.fold_tokens(list.tokens.clone()); } } } @@ -1329,6 +1421,149 @@ host_env = ["POSIXLY_CORRECT"] ); } + /// Upstream renders `FormatError` through thiserror attributes, so + /// the i18n macros sit in attribute token streams rather than + /// expressions — including thiserror's `.0` field shorthand, which + /// tokenizes but is not valid expression syntax. + #[test] + fn fluent_folds_inside_thiserror_error_attributes() { + let (_tmp, uutils, manifest, out) = fixture( + r#" +[[modules]] +name = "demo" +source = "lib/demo.rs" +out = "demo.rs" + +[[modules.substitutions]] +prefix = "crate::translate" +action = "fluent" +ftl_sources = ["locales/en-US.ftl"] + +[[modules.substitutions]] +prefix = "crate::translate_text" +action = "fluent" +ftl_sources = ["locales/en-US.ftl"] +"#, + &[ + ( + "locales/en-US.ftl", + "demo-plain = nothing left to read\ndemo-slot = bad spec { $spec } here\n", + ), + ( + "lib/demo.rs", + r#" +use crate::{translate, translate_text}; + +#[derive(Debug, thiserror::Error)] +pub enum DemoError { + #[error("{}", translate!("demo-plain"))] + Plain, + #[error("{}", translate_text!("demo-slot", "spec" => String::from_utf8_lossy(.0)))] + Slot(Vec), +} +"#, + ), + ], + ); + + let written = run(&uutils, "demo", "x", &manifest, &out).unwrap(); + let body = fs::read_to_string(&written[0]).unwrap(); + assert!(!body.contains("translate!"), "unfolded macro: {body}"); + assert!(!body.contains("translate_text!"), "unfolded macro: {body}"); + assert!( + body.contains(r#"::std::string::String::from("nothing left to read")"#), + "{body}" + ); + // The `.0` shorthand has to survive verbatim for thiserror to + // bind it to the field. + assert!( + body.contains(r#"::std::format!("bad spec {} here", String::from_utf8_lossy(.0))"#), + "{body}" + ); + // The import the stanza owns is still dropped. + assert!(!body.contains("use crate::translate"), "{body}"); + } + + /// Only the macro's leaf ident is replaced, so a qualified call in + /// attribute position would strand its `uucore ::` prefix in front + /// of the folded literal. Reject rather than emit broken code. + #[test] + fn qualified_fluent_macro_in_attribute_hard_errors() { + let (_tmp, uutils, manifest, out) = fixture( + r#" +[[modules]] +name = "demo" +source = "lib/demo.rs" +out = "demo.rs" + +[[modules.substitutions]] +prefix = "crate::translate" +action = "fluent" +ftl_sources = ["locales/en-US.ftl"] +"#, + &[ + ("locales/en-US.ftl", "demo-plain = nope\n"), + ( + "lib/demo.rs", + r#" +#[derive(Debug, thiserror::Error)] +pub enum DemoError { + #[error("{}", uucore::translate!("demo-plain"))] + Plain, +} +"#, + ), + ], + ); + + let err = run(&uutils, "demo", "x", &manifest, &out).unwrap_err(); + let msg = format!("{err:#}"); + assert!( + msg.contains("qualified 'translate!' invocation in attribute position"), + "got: {msg}" + ); + } + + /// An i18n macro in attribute position that no stanza opted in has + /// to fail the port, exactly as in expression position — otherwise + /// the attribute path would be a hole in the Fluent boundary. + #[test] + fn unopted_fluent_macro_in_attribute_hard_errors() { + let (_tmp, uutils, manifest, out) = fixture( + r#" +[[modules]] +name = "demo" +source = "lib/demo.rs" +out = "demo.rs" + +[[modules.substitutions]] +prefix = "crate::translate" +action = "fluent" +ftl_sources = ["locales/en-US.ftl"] +"#, + &[ + ("locales/en-US.ftl", "demo-plain = nope\n"), + ( + "lib/demo.rs", + r#" +#[derive(Debug, thiserror::Error)] +pub enum DemoError { + #[error("{}", translate_text!("demo-plain", "x" => .0))] + Plain(String), +} +"#, + ), + ], + ); + + let err = run(&uutils, "demo", "x", &manifest, &out).unwrap_err(); + let msg = format!("{err:#}"); + assert!( + msg.contains("translate_text!") && msg.contains("action 'fluent'"), + "got: {msg}" + ); + } + #[test] fn fluent_import_hard_errors() { let (_tmp, uutils, manifest, out) = fixture( diff --git a/crates/bashkit/src/builtins/generated/cat_args.rs b/crates/bashkit/src/builtins/generated/cat_args.rs index b384fef0e..40fa9e85c 100644 --- a/crates/bashkit/src/builtins/generated/cat_args.rs +++ b/crates/bashkit/src/builtins/generated/cat_args.rs @@ -1,6 +1,6 @@ // GENERATED by bashkit-coreutils-port. DO NOT EDIT. // -// Source: uutils/coreutils@9cc868bca src/uu/cat/ +// Source: uutils/coreutils@f56bd018d src/uu/cat/ // Regenerate: cargo run -p bashkit-coreutils-port -- cat // // Original uutils licensed MIT; see THIRD_PARTY_LICENSES. diff --git a/crates/bashkit/src/builtins/generated/extendedbigdecimal.rs b/crates/bashkit/src/builtins/generated/extendedbigdecimal.rs index 63b275feb..d83625278 100644 --- a/crates/bashkit/src/builtins/generated/extendedbigdecimal.rs +++ b/crates/bashkit/src/builtins/generated/extendedbigdecimal.rs @@ -1,6 +1,6 @@ // GENERATED by bashkit-coreutils-port. DO NOT EDIT. // -// Source: uutils/coreutils@9cc868bca src/uucore/src/lib/features/extendedbigdecimal.rs +// Source: uutils/coreutils@f56bd018d src/uucore/src/lib/features/extendedbigdecimal.rs // Regenerate: cargo run -p bashkit-coreutils-port -- port-module format // // Original uutils licensed MIT; see THIRD_PARTY_LICENSES. diff --git a/crates/bashkit/src/builtins/generated/format/argument.rs b/crates/bashkit/src/builtins/generated/format/argument.rs index b432e68bf..440295789 100644 --- a/crates/bashkit/src/builtins/generated/format/argument.rs +++ b/crates/bashkit/src/builtins/generated/format/argument.rs @@ -1,6 +1,6 @@ // GENERATED by bashkit-coreutils-port. DO NOT EDIT. // -// Source: uutils/coreutils@9cc868bca src/uucore/src/lib/features/format/argument.rs +// Source: uutils/coreutils@f56bd018d src/uucore/src/lib/features/format/argument.rs // Regenerate: cargo run -p bashkit-coreutils-port -- port-module format // // Original uutils licensed MIT; see THIRD_PARTY_LICENSES. diff --git a/crates/bashkit/src/builtins/generated/format/escape.rs b/crates/bashkit/src/builtins/generated/format/escape.rs index 1303111bf..95ccd059a 100644 --- a/crates/bashkit/src/builtins/generated/format/escape.rs +++ b/crates/bashkit/src/builtins/generated/format/escape.rs @@ -1,6 +1,6 @@ // GENERATED by bashkit-coreutils-port. DO NOT EDIT. // -// Source: uutils/coreutils@9cc868bca src/uucore/src/lib/features/format/escape.rs +// Source: uutils/coreutils@f56bd018d src/uucore/src/lib/features/format/escape.rs // Regenerate: cargo run -p bashkit-coreutils-port -- port-module format // // Original uutils licensed MIT; see THIRD_PARTY_LICENSES. diff --git a/crates/bashkit/src/builtins/generated/format/human.rs b/crates/bashkit/src/builtins/generated/format/human.rs index 181b94e54..9b0ea3573 100644 --- a/crates/bashkit/src/builtins/generated/format/human.rs +++ b/crates/bashkit/src/builtins/generated/format/human.rs @@ -1,6 +1,6 @@ // GENERATED by bashkit-coreutils-port. DO NOT EDIT. // -// Source: uutils/coreutils@9cc868bca src/uucore/src/lib/features/format/human.rs +// Source: uutils/coreutils@f56bd018d src/uucore/src/lib/features/format/human.rs // Regenerate: cargo run -p bashkit-coreutils-port -- port-module format // // Original uutils licensed MIT; see THIRD_PARTY_LICENSES. diff --git a/crates/bashkit/src/builtins/generated/format/mod.rs b/crates/bashkit/src/builtins/generated/format/mod.rs index 0ce6c552d..867783117 100644 --- a/crates/bashkit/src/builtins/generated/format/mod.rs +++ b/crates/bashkit/src/builtins/generated/format/mod.rs @@ -1,6 +1,6 @@ // GENERATED by bashkit-coreutils-port. DO NOT EDIT. // -// Source: uutils/coreutils@9cc868bca src/uucore/src/lib/features/format/mod.rs +// Source: uutils/coreutils@f56bd018d src/uucore/src/lib/features/format/mod.rs // Regenerate: cargo run -p bashkit-coreutils-port -- port-module format // // Original uutils licensed MIT; see THIRD_PARTY_LICENSES. @@ -49,33 +49,71 @@ pub use argument::FormatArgument; pub use argument::FormatArguments; use os_display::Quotable; pub use spec::Spec; -use std::error::Error; -use std::fmt::Display; use std::io::Write; use std::io::stdout; use std::marker::PhantomData; use std::ops::ControlFlow; use std::ops::Range; -#[derive(Debug)] +#[derive(Debug, thiserror::Error)] pub enum FormatError { /// The spec that failed to parse and its byte range in the format string. + #[error( + "{}", + ::std::format!( + "%{}: invalid conversion specification", + String::from_utf8_lossy(.0) + ) + )] SpecError(Vec, Range), + #[error("{}", ::std::format!("write error: {}", strip_errno(.0)))] IoError(std::io::Error), + #[error("{}", ::std::string::String::from("no more arguments"))] NoMoreArguments, + #[error("{}", ::std::string::String::from("invalid argument"))] InvalidArgument(FormatArgument), + #[error( + "{}", + ::std::format!( + "format '{}' has too many % directives", + String::from_utf8_lossy(.0) + ) + )] TooManySpecs(Vec), + #[error( + "{}", + ::std::format!("format '{}' has no % directive", String::from_utf8_lossy(.0)) + )] NeedAtLeastOneSpec(Vec), + #[error("{}", ::std::string::String::from("wrong % directive type was given"))] WrongSpecType, + #[error("{}", ::std::format!("invalid precision: '{}'", .0))] InvalidPrecision(String), /// The format specifier ends with a %, as in `%f%`. + #[error( + "{}", + ::std::format!("format {} ends in %", String::from_utf8_lossy(.0).quote()) + )] EndsWithPercent(Vec), /// The escape sequence `\x` appears without a literal hexadecimal value. /// Carries its byte range in the format string, when it came from one. + #[error( + "{}", + ::std::string::String::from("missing hexadecimal number in escape") + )] MissingHex(Option>), /// The hexadecimal characters represent a code point that cannot represent a /// Unicode character (e.g., a surrogate code point) /// Carries its byte range in the format string, when it came from one. + #[error( + "{}", + ::std::format!( + "invalid universal character name \\{}{}", + .0, + String::from_utf8_lossy(.1) + ) + )] InvalidCharacter(char, Vec, Option>), + #[error("{0}")] InvalidEncoding(NonUtf8OsStrError), } impl FormatError { @@ -99,7 +137,6 @@ fn spec_error(fmt: &[u8], percent: usize, slice: &[u8]) -> FormatError { debug_assert!(percent < start); FormatError::SpecError(slice.to_vec(), percent..start + slice.len()) } -impl Error for FormatError {} impl UError for FormatError {} impl From for FormatError { fn from(value: std::io::Error) -> Self { @@ -111,49 +148,6 @@ impl From for FormatError { Self::InvalidEncoding(value) } } -impl Display for FormatError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let message = match self { - Self::SpecError(s, _) => { - ::std::format!( - "%{}: invalid conversion specification", - String::from_utf8_lossy(s) - ) - } - Self::TooManySpecs(s) => { - ::std::format!( - "format '{}' has too many % directives", - String::from_utf8_lossy(s) - ) - } - Self::NeedAtLeastOneSpec(s) => { - ::std::format!("format '{}' has no % directive", String::from_utf8_lossy(s)) - } - Self::EndsWithPercent(s) => { - ::std::format!("format {} ends in %", String::from_utf8_lossy(s).quote()) - } - Self::InvalidPrecision(precision) => { - ::std::format!("invalid precision: '{}'", precision) - } - Self::WrongSpecType => ::std::string::String::from("wrong % directive type was given"), - Self::IoError(e) => ::std::format!("write error: {}", strip_errno(e)), - Self::NoMoreArguments => ::std::string::String::from("no more arguments"), - Self::InvalidArgument(_) => ::std::string::String::from("invalid argument"), - Self::MissingHex(_) => { - ::std::string::String::from("missing hexadecimal number in escape") - } - Self::InvalidCharacter(escape_char, digits, _) => { - ::std::format!( - "invalid universal character name \\{}{}", - escape_char, - String::from_utf8_lossy(digits) - ) - } - Self::InvalidEncoding(no) => return no.fmt(f), - }; - f.write_str(&message) - } -} const MAX_FORMAT_WIDTH: usize = 1_000_000; fn check_width(width: usize) -> std::io::Result<()> { if width > MAX_FORMAT_WIDTH { diff --git a/crates/bashkit/src/builtins/generated/format/num_format.rs b/crates/bashkit/src/builtins/generated/format/num_format.rs index 4960cfe54..2ef17576b 100644 --- a/crates/bashkit/src/builtins/generated/format/num_format.rs +++ b/crates/bashkit/src/builtins/generated/format/num_format.rs @@ -1,6 +1,6 @@ // GENERATED by bashkit-coreutils-port. DO NOT EDIT. // -// Source: uutils/coreutils@9cc868bca src/uucore/src/lib/features/format/num_format.rs +// Source: uutils/coreutils@f56bd018d src/uucore/src/lib/features/format/num_format.rs // Regenerate: cargo run -p bashkit-coreutils-port -- port-module format // // Original uutils licensed MIT; see THIRD_PARTY_LICENSES. diff --git a/crates/bashkit/src/builtins/generated/format/spec.rs b/crates/bashkit/src/builtins/generated/format/spec.rs index 441737d5f..c35b88769 100644 --- a/crates/bashkit/src/builtins/generated/format/spec.rs +++ b/crates/bashkit/src/builtins/generated/format/spec.rs @@ -1,6 +1,6 @@ // GENERATED by bashkit-coreutils-port. DO NOT EDIT. // -// Source: uutils/coreutils@9cc868bca src/uucore/src/lib/features/format/spec.rs +// Source: uutils/coreutils@f56bd018d src/uucore/src/lib/features/format/spec.rs // Regenerate: cargo run -p bashkit-coreutils-port -- port-module format // // Original uutils licensed MIT; see THIRD_PARTY_LICENSES. diff --git a/crates/bashkit/src/builtins/generated/ls_args.rs b/crates/bashkit/src/builtins/generated/ls_args.rs index 5072cf453..596896f52 100644 --- a/crates/bashkit/src/builtins/generated/ls_args.rs +++ b/crates/bashkit/src/builtins/generated/ls_args.rs @@ -1,6 +1,6 @@ // GENERATED by bashkit-coreutils-port. DO NOT EDIT. // -// Source: uutils/coreutils@9cc868bca src/uu/ls/ +// Source: uutils/coreutils@f56bd018d src/uu/ls/ // Regenerate: cargo run -p bashkit-coreutils-port -- ls // // Original uutils licensed MIT; see THIRD_PARTY_LICENSES. @@ -66,7 +66,7 @@ pub mod options { pub static DIR_ARGS: &str = "dereference-command-line-symlink-to-dir"; } pub static HELP: &str = "help"; - pub static QUOTING_STYLE: &str = "quoting-style"; + pub const QUOTING_STYLE: &str = "quoting-style"; pub static HIDE_CONTROL_CHARS: &str = "hide-control-chars"; pub static SHOW_CONTROL_CHARS: &str = "show-control-chars"; pub static WIDTH: &str = "width"; @@ -122,6 +122,23 @@ pub static LS_ENV_DEFAULTS: &[crate::builtins::clap_env::EnvDefault] = &[ kind: crate::builtins::clap_env::EnvKind::Single, }, ]; +pub fn clap_quoting_argument() -> Arg { + Arg::new(QUOTING_STYLE) + .long(QUOTING_STYLE) + .help(::std::string::String::from("Set quoting style")) + .value_parser(::clap::builder::PossibleValuesParser::new([ + PossibleValue::new("literal"), + PossibleValue::new("locale"), + PossibleValue::new("shell"), + PossibleValue::new("shell-escape"), + PossibleValue::new("shell-always"), + PossibleValue::new("shell-escape-always"), + PossibleValue::new("clocale"), + PossibleValue::new("c"), + PossibleValue::new("c-maybe"), + PossibleValue::new("escape"), + ])) +} pub fn ls_command() -> Command { Command::new("ls") .version(env!("CARGO_PKG_VERSION")) @@ -306,23 +323,7 @@ pub fn ls_command() -> Command { .action(ArgAction::SetTrue), ) .arg( - Arg::new(QUOTING_STYLE) - .long(QUOTING_STYLE) - .help(::std::string::String::from("Set quoting style.")) - .value_parser( - ::clap::builder::PossibleValuesParser::new([ - PossibleValue::new("literal"), - PossibleValue::new("locale"), - PossibleValue::new("shell"), - PossibleValue::new("shell-escape"), - PossibleValue::new("shell-always"), - PossibleValue::new("shell-escape-always"), - PossibleValue::new("clocale"), - PossibleValue::new("c"), - PossibleValue::new("c-maybe"), - PossibleValue::new("escape"), - ]), - ) + clap_quoting_argument() .overrides_with_all([ QUOTING_STYLE, options::quoting::LITERAL, diff --git a/crates/bashkit/src/builtins/generated/mktemp_args.rs b/crates/bashkit/src/builtins/generated/mktemp_args.rs index 7cdd63188..b748c4378 100644 --- a/crates/bashkit/src/builtins/generated/mktemp_args.rs +++ b/crates/bashkit/src/builtins/generated/mktemp_args.rs @@ -1,6 +1,6 @@ // GENERATED by bashkit-coreutils-port. DO NOT EDIT. // -// Source: uutils/coreutils@9cc868bca src/uu/mktemp/ +// Source: uutils/coreutils@f56bd018d src/uu/mktemp/ // Regenerate: cargo run -p bashkit-coreutils-port -- mktemp // // Original uutils licensed MIT; see THIRD_PARTY_LICENSES. diff --git a/crates/bashkit/src/builtins/generated/mod.rs b/crates/bashkit/src/builtins/generated/mod.rs index 96f39cfa1..a25e28eb5 100644 --- a/crates/bashkit/src/builtins/generated/mod.rs +++ b/crates/bashkit/src/builtins/generated/mod.rs @@ -26,7 +26,7 @@ // and at test time (the invariant assertion). Nothing in the lib's // runtime path needs it, hence `allow(dead_code)`. #[allow(dead_code)] -pub const UUTILS_REVISION: &str = "9cc868bca"; +pub const UUTILS_REVISION: &str = "f56bd018d"; pub mod cat_args; #[allow(clippy::collapsible_if, clippy::unwrap_used)] diff --git a/crates/bashkit/src/builtins/generated/num_parser.rs b/crates/bashkit/src/builtins/generated/num_parser.rs index f4d0e25d6..cd55be831 100644 --- a/crates/bashkit/src/builtins/generated/num_parser.rs +++ b/crates/bashkit/src/builtins/generated/num_parser.rs @@ -1,6 +1,6 @@ // GENERATED by bashkit-coreutils-port. DO NOT EDIT. // -// Source: uutils/coreutils@9cc868bca src/uucore/src/lib/features/parser/num_parser.rs +// Source: uutils/coreutils@f56bd018d src/uucore/src/lib/features/parser/num_parser.rs // Regenerate: cargo run -p bashkit-coreutils-port -- port-module format // // Original uutils licensed MIT; see THIRD_PARTY_LICENSES. diff --git a/crates/bashkit/src/builtins/generated/od_args.rs b/crates/bashkit/src/builtins/generated/od_args.rs index b728ecdcd..548bcf908 100644 --- a/crates/bashkit/src/builtins/generated/od_args.rs +++ b/crates/bashkit/src/builtins/generated/od_args.rs @@ -1,6 +1,6 @@ // GENERATED by bashkit-coreutils-port. DO NOT EDIT. // -// Source: uutils/coreutils@9cc868bca src/uu/od/ +// Source: uutils/coreutils@f56bd018d src/uu/od/ // Regenerate: cargo run -p bashkit-coreutils-port -- od // // Original uutils licensed MIT; see THIRD_PARTY_LICENSES. diff --git a/crates/bashkit/src/builtins/generated/readlink_args.rs b/crates/bashkit/src/builtins/generated/readlink_args.rs index 880c42302..d35096578 100644 --- a/crates/bashkit/src/builtins/generated/readlink_args.rs +++ b/crates/bashkit/src/builtins/generated/readlink_args.rs @@ -1,6 +1,6 @@ // GENERATED by bashkit-coreutils-port. DO NOT EDIT. // -// Source: uutils/coreutils@9cc868bca src/uu/readlink/ +// Source: uutils/coreutils@f56bd018d src/uu/readlink/ // Regenerate: cargo run -p bashkit-coreutils-port -- readlink // // Original uutils licensed MIT; see THIRD_PARTY_LICENSES. diff --git a/crates/bashkit/src/builtins/generated/realpath_args.rs b/crates/bashkit/src/builtins/generated/realpath_args.rs index 7f1cd1b8c..991897523 100644 --- a/crates/bashkit/src/builtins/generated/realpath_args.rs +++ b/crates/bashkit/src/builtins/generated/realpath_args.rs @@ -1,6 +1,6 @@ // GENERATED by bashkit-coreutils-port. DO NOT EDIT. // -// Source: uutils/coreutils@9cc868bca src/uu/realpath/ +// Source: uutils/coreutils@f56bd018d src/uu/realpath/ // Regenerate: cargo run -p bashkit-coreutils-port -- realpath // // Original uutils licensed MIT; see THIRD_PARTY_LICENSES. diff --git a/crates/bashkit/src/builtins/generated/shuf_args.rs b/crates/bashkit/src/builtins/generated/shuf_args.rs index 9aefdac7e..5c5105f21 100644 --- a/crates/bashkit/src/builtins/generated/shuf_args.rs +++ b/crates/bashkit/src/builtins/generated/shuf_args.rs @@ -1,6 +1,6 @@ // GENERATED by bashkit-coreutils-port. DO NOT EDIT. // -// Source: uutils/coreutils@9cc868bca src/uu/shuf/ +// Source: uutils/coreutils@f56bd018d src/uu/shuf/ // Regenerate: cargo run -p bashkit-coreutils-port -- shuf // // Original uutils licensed MIT; see THIRD_PARTY_LICENSES. diff --git a/crates/bashkit/src/builtins/generated/stat_args.rs b/crates/bashkit/src/builtins/generated/stat_args.rs index ee904c5ae..2f1368747 100644 --- a/crates/bashkit/src/builtins/generated/stat_args.rs +++ b/crates/bashkit/src/builtins/generated/stat_args.rs @@ -1,6 +1,6 @@ // GENERATED by bashkit-coreutils-port. DO NOT EDIT. // -// Source: uutils/coreutils@9cc868bca src/uu/stat/ +// Source: uutils/coreutils@f56bd018d src/uu/stat/ // Regenerate: cargo run -p bashkit-coreutils-port -- stat // // Original uutils licensed MIT; see THIRD_PARTY_LICENSES. @@ -87,13 +87,15 @@ pub fn stat_command() -> Command { ), ) .value_name("FORMAT") - .allow_hyphen_values(true), + .allow_hyphen_values(true) + .overrides_with(options::PRINTF), ) .arg( Arg::new(options::PRINTF) .long(options::PRINTF) .value_name("FORMAT") .allow_hyphen_values(true) + .overrides_with(options::FORMAT) .help( ::std::string::String::from( "like --format, but interpret backslash escapes,\nand do not output a mandatory trailing newline;\nif you want a newline, include \\n in FORMAT", diff --git a/crates/bashkit/src/builtins/generated/tac_args.rs b/crates/bashkit/src/builtins/generated/tac_args.rs index f19e5c578..85a941bd9 100644 --- a/crates/bashkit/src/builtins/generated/tac_args.rs +++ b/crates/bashkit/src/builtins/generated/tac_args.rs @@ -1,6 +1,6 @@ // GENERATED by bashkit-coreutils-port. DO NOT EDIT. // -// Source: uutils/coreutils@9cc868bca src/uu/tac/ +// Source: uutils/coreutils@f56bd018d src/uu/tac/ // Regenerate: cargo run -p bashkit-coreutils-port -- tac // // Original uutils licensed MIT; see THIRD_PARTY_LICENSES. diff --git a/crates/bashkit/src/builtins/generated/tee_args.rs b/crates/bashkit/src/builtins/generated/tee_args.rs index 268534666..ebbf23bb8 100644 --- a/crates/bashkit/src/builtins/generated/tee_args.rs +++ b/crates/bashkit/src/builtins/generated/tee_args.rs @@ -1,6 +1,6 @@ // GENERATED by bashkit-coreutils-port. DO NOT EDIT. // -// Source: uutils/coreutils@9cc868bca src/uu/tee/ +// Source: uutils/coreutils@f56bd018d src/uu/tee/ // Regenerate: cargo run -p bashkit-coreutils-port -- tee // // Original uutils licensed MIT; see THIRD_PARTY_LICENSES. diff --git a/crates/bashkit/src/builtins/generated/truncate_args.rs b/crates/bashkit/src/builtins/generated/truncate_args.rs index 17301e7c6..0b2ccb747 100644 --- a/crates/bashkit/src/builtins/generated/truncate_args.rs +++ b/crates/bashkit/src/builtins/generated/truncate_args.rs @@ -1,6 +1,6 @@ // GENERATED by bashkit-coreutils-port. DO NOT EDIT. // -// Source: uutils/coreutils@9cc868bca src/uu/truncate/ +// Source: uutils/coreutils@f56bd018d src/uu/truncate/ // Regenerate: cargo run -p bashkit-coreutils-port -- truncate // // Original uutils licensed MIT; see THIRD_PARTY_LICENSES. diff --git a/knowledge/log.md b/knowledge/log.md index b58c7c1cd..3ae70514b 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -1,5 +1,9 @@ # Bashkit Knowledge Update Log +## 2026-11-09 + +* **Coreutils drift port keeps up with two upstream refactors**: the weekly `Coreutils Args Drift` run went red on 2026-11-09 (13 compile errors against uutils 6a7d199ed), from two unrelated upstream changes that the port tool had no rewrite for. uucore's `FormatError` moved from a hand-written `impl Display` to `#[derive(thiserror::Error)]`, putting every `translate!`/`translate_text!` call inside an `#[error("{}", ...)]` attribute, where the expression-only Fluent pass never reached it; folding now walks attribute token streams too, and macro arguments are carried as raw `TokenStream` rather than `Expr` because thiserror's `.0` field shorthand tokenizes but is not valid expression syntax. Separately, `--quoting-style` moved into the shared `uucore::quoting_style::parser`, so `ls` began re-exporting `QUOTING_STYLE` and calling `clap_quoting_argument()`, both of which made generated `ls_args.rs` reference `uucore`; args mode now inlines registered uucore clap helpers (const/static spliced into `mod options`, builder fns emitted at root, paths flattened, `translate!` keys resolved against uucore's own locales). The helper registry is an allowlist with per-shape validation, same TM-INF-025 reasoning as the `uu_app()` validator: an unregistered path stays untouched and surfaces as a compile error instead of being vendored blind. Only user-visible change is `ls --quoting-style` help text losing its trailing period, which is upstream's wording. See [Coreutils Argument Port](runtimes/coreutils-args-port.md). + ## 2026-11-08 * **Monty forks re-synced to 1.5.0**: the first re-sync since the forks were created, and what the daily upstream watch went red for. Upstream moved ~9k lines and touched all 14 fork-edited files, so the `bashkit fork:` edits were re-applied by 3-way merge (upstream 1.4.0 / fork / upstream 1.5.0) rather than by hand: 9 conflicts, all in the repr helpers and the `fancy-regex` plumbing. Net drift went *down*. Upstream reached fancy-regex 0.20, so that marked version divergence and the marked `Captures<'_, str>` adaptations in `re_match.rs`/`re_pattern.rs` are gone (1.5.0 replaced `Captures` with span buffers); keeping upstream's new `repr_should_truncate` name let every repr call site return to upstream verbatim, leaving the fork's semantics (a time hit raises, a memory hit truncates) in one helper body. The per-VM memory probe and checkpoint hook merged cleanly: upstream's new `ExecutionClock` work touches different parts of `resource.rs`. Behavior changes taken: `'ab' * (2**62)` now raises `OverflowError` up front instead of reaching the memory meter, and the host import callback receives an `ImportArgs` (module plus search path) where it took a bare name -- bashkit answers `NotFound` on the name, so upstream's new file imports stay out. `cpu-clock`/`nix` came along because features and added dependencies are not markable. See [Dependencies](operations/dependencies.md#monty-forks), [Python Builtin](runtimes/python-builtin.md). diff --git a/knowledge/runtimes/coreutils-args-port.md b/knowledge/runtimes/coreutils-args-port.md index af60a38af..e0ff63d74 100644 --- a/knowledge/runtimes/coreutils-args-port.md +++ b/knowledge/runtimes/coreutils-args-port.md @@ -26,6 +26,9 @@ codegen**, not by depending on `uu_*` crates at runtime. 2. Reads `/src/uu//locales/en-US.ftl` for help/about strings. 3. Rewrites the `uu_app()` AST in place: - `translate!("k")` → `String::from("")` + - `[uucore::]::::` for a registered uucore clap-helper + module → the item is inlined and the path flattened to bare + `` (see "Shared uucore clap helpers" below) - `uucore::crate_version!()` → `env!("CARGO_PKG_VERSION")` - `uucore::format_usage(x)` → local `format_usage` shim - `.help_template(uucore::localized_help_template(...))` and @@ -68,6 +71,55 @@ codegen**, not by depending on `uu_*` crates at runtime. 5. Emits `crates/bashkit/src/builtins/generated/_args.rs` with a clean `pub fn _command() -> clap::Command`. +## Shared uucore clap helpers + +Upstream factors options that several utilities agree on into +`uucore::::`. `--quoting-style` moved into +`uucore::quoting_style::parser` in uutils 6a7d199ed, which took `ls` +with it: `mod options` stopped defining `QUOTING_STYLE` and started +re-exporting it (`pub use quoting_style::parser::QUOTING_STYLE;`), and +the arg chain began calling +`quoting_style::parser::clap_quoting_argument()`. Generated `ls_args.rs` +then referenced `uucore`, which bashkit does not link, and the +2026-11-09 drift run went red. + +Nothing in those helper bodies is uucore-specific — plain clap builders +plus `translate!()` help text — so they are **inlined at port time** +under the same rewrite rules as `uu_app()` itself: + +- `const`/`static` items are spliced in where `mod options` re-exported + them, keeping the re-export's visibility, so `options::QUOTING_STYLE` + and the `use options::*` glob both keep resolving. +- Clap-builder functions are emitted at file root and call sites are + flattened from `[uucore::]::::` to bare ``. +- The bare `use uucore::;` that brought the module into scope is + dropped. +- `translate!()` keys in helper bodies resolve against **uucore's** + locales (`src/uucore/locales/en-US.ftl`), not the utility's. + +Inlining rather than vendoring the whole uucore module keeps the +generated parser standalone and leaves the emitted arg equivalent to the +hand-rolled one it replaced upstream. The only user-visible change from +the `ls` move was the help text losing its trailing period, which is +upstream's own wording. + +`CLAP_HELPER_MODULES` in `args.rs` is an **allowlist**, for the same +reason `uu_app()` has a shape validator (TM-INF-025): third-party uutils +source must not be able to pull arbitrary uucore code into a generated +parser just by naming it. An unregistered path is left untouched and +surfaces as a bashkit compile error. Within a registered module, only +these shapes inline, and anything else aborts the port: + +- a `const`/`static` whose value is a literal, and not `static mut`; +- a non-generic, nullary, non-`async` function with no safety qualifier, + returning clap `Arg` or `Command`, whose body is a single tail + expression. + +A `#[cfg]` on a candidate item also aborts: resolving it would need +cargo feature evaluation the port does not do, so taking the body +unconditionally could silently port the wrong one. Adding a module is +one `CLAP_HELPER_MODULES` entry naming its source file and ftl sources. + ## Per-invocation cost: cache a *built* `Command` `_command()` is codegen output, so calling it per invocation rebuilt the @@ -268,6 +320,25 @@ runtime locale data, meaning selectors, plurals, and term references. The rewriter also rejects a `translate*!` invocation that no stanza opted in, so a fully-qualified call cannot side-step the import-level gate. +Folding reaches **attribute token streams**, not only expressions. +Upstream renders `uucore::format`'s `FormatError` through +`#[derive(thiserror::Error)]` with the message in +`#[error("{}", translate!("k"))]`, so an expression-only walk left every +arm unfolded and the vendored module stopped compiling (the 2026-11-09 +drift run). Two consequences shape the implementation: + +- Attribute tokens are opaque to `syn`'s visitor, so the pass walks token + trees directly, recursing through delimited groups. +- A macro argument there may use thiserror's `.0` field shorthand, which + tokenizes but is **not** valid expression syntax. Call arguments are + therefore carried as raw `TokenStream`, not `Expr`, and interpolated + verbatim so thiserror still binds them to the field. In expression + position the rendered tokens are parsed back to an `Expr`. + +Because only the macro's leaf ident is replaced, a *qualified* +`uucore::translate!(…)` in attribute position is a hard error rather than +an emitted `uucore :: "literal"`. + `.ftl` parsing (flat `key = value` + indented continuations) and pattern splitting live in `crates/bashkit-coreutils-port/src/ftl.rs`, shared by both modes. It is deliberately not a Fluent implementation.