From d45223c10e2304914335ecc0c9b5e415e6366324 Mon Sep 17 00:00:00 2001 From: Raul Estrada Date: Sun, 25 Oct 2026 19:16:42 -0600 Subject: [PATCH 1/4] feat: add staged libdevice math generation Signed-off-by: Raul Estrada --- .../crates/cuda-intrinsics-gen/README.md | 58 +- .../cuda-intrinsics-gen/src/generate.rs | 22 +- .../src/libdevice_math/evidence.rs | 159 ++++ .../src/libdevice_math/mod.rs | 408 ++++++++++ .../src/libdevice_math/render.rs | 47 ++ .../src/libdevice_math/tests.rs | 400 ++++++++++ .../crates/cuda-intrinsics-gen/src/main.rs | 18 + .../evidence/libdevice_math/README.md | 33 + .../evidence/libdevice_math/cuda-13.0.json | 71 ++ .../intrinsics/generated/libdevice_math.rs | 742 ++++++++++++++++++ cuda-oxide/intrinsics/libdevice-math.md | 87 ++ .../intrinsics/overlay/libdevice_math.toml | 194 +++++ 12 files changed, 2237 insertions(+), 2 deletions(-) create mode 100644 cuda-oxide/crates/cuda-intrinsics-gen/src/libdevice_math/evidence.rs create mode 100644 cuda-oxide/crates/cuda-intrinsics-gen/src/libdevice_math/mod.rs create mode 100644 cuda-oxide/crates/cuda-intrinsics-gen/src/libdevice_math/render.rs create mode 100644 cuda-oxide/crates/cuda-intrinsics-gen/src/libdevice_math/tests.rs create mode 100644 cuda-oxide/intrinsics/evidence/libdevice_math/README.md create mode 100644 cuda-oxide/intrinsics/evidence/libdevice_math/cuda-13.0.json create mode 100644 cuda-oxide/intrinsics/generated/libdevice_math.rs create mode 100644 cuda-oxide/intrinsics/libdevice-math.md create mode 100644 cuda-oxide/intrinsics/overlay/libdevice_math.toml diff --git a/cuda-oxide/crates/cuda-intrinsics-gen/README.md b/cuda-oxide/crates/cuda-intrinsics-gen/README.md index dc2901a853..02524a79df 100644 --- a/cuda-oxide/crates/cuda-intrinsics-gen/README.md +++ b/cuda-oxide/crates/cuda-intrinsics-gen/README.md @@ -4,7 +4,7 @@ Extractor and deterministic source generator for cuda-oxide's intrinsics. It reads `intrinsics/catalog.json` and writes every generated file in the intrinsic pipeline. `MANAGED_GENERATED_ROOTS` in `src/generate.rs` lists the -nine directories it owns, where `generate` writes outputs and removes stale +ten directories it owns, where `generate` writes outputs and removes stale marker-bearing files: - `cuda-oxide/crates/cuda-intrinsics/src/generated/`: the raw intrinsic declarations; @@ -17,6 +17,7 @@ marker-bearing files: requirements), `cuda-oxide/crates/mir-lower/src/convert` (conversion interfaces), and `cuda-oxide/crates/rustc-codegen-cuda/src` (collector predicates); - `intrinsics/probes/`: the per-intrinsic PTX probe evidence. +- `intrinsics/generated/`: staged libm-to-libdevice mapping metadata. Generated files carry a `// @generated by cuda-intrinsics-gen ... DO NOT EDIT.` banner; edit the @@ -27,6 +28,8 @@ cuda-intrinsics-gen extract --intrinsics-json FILE --nvptx-json FILE cuda-intrinsics-gen extract --llvm-src DIR --llvm-tblgen FILE cuda-intrinsics-gen generate cuda-intrinsics-gen check +cuda-intrinsics-gen extract-libdevice --libdevice FILE --llvm-dis FILE +cuda-intrinsics-gen verify-libdevice --libdevice FILE --llvm-dis FILE cuda-intrinsics-gen coverage [--family NAME] cuda-intrinsics-gen check-abi-history --base-ref REF cuda-intrinsics-gen probe [--all | --intrinsic ID] [--llc FILE] [--skip-terminal] [--per-target] @@ -40,3 +43,56 @@ by each route, and `check-abi-history` keeps the intrinsic ABI ledger append-only. Generation shells out to `rustfmt`, which the pinned toolchain provides. + +## Staged libdevice math family + +`intrinsics/overlay/libdevice_math.toml` records the 72 existing f32/f64 libm +exports, including aliases and handwritten `fmin`/`fmax` and `sincos` routes. +The renderer emits `intrinsics/generated/libdevice_math.rs` and +`intrinsics/libdevice-math.md`. The Rust metadata is compiled by generator tests; +compiler dispatch and its handwritten tables are migrated in a subsequent PR. + +This input is read explicitly by the libdevice family, separately from the +hardware shard index in `intrinsics/overlay.toml`. Its evidence comes from +`libdevice.10.bc`, so it does not receive hardware ABI IDs or change the hardware +catalog hash. The overlay reconciliation guard recognizes this separate reader; +unlisted hardware shards remain errors. Normal `generate` and `check` process +both domains, and stale metadata under `intrinsics/generated/` is reconciled. + +Rows name a Rust signature class, optional libm aliases and cmath spelling, +placeholder suffixes, libdevice symbol, optional native LLVM alternative, +link classification, and optional `handwritten:NAME` handler. `{width}` expands +to `f32`/`f64`; `{suffix}` expands to `f`/the empty string. Unknown fields, +duplicate names, invalid templates, ambiguous ABI signatures, and incompatible +link classifications fail before outputs are written. Tuple signatures require +an explicit handwritten handler. + +Generation checks every referenced C symbol and signature against the pinned +capture in `intrinsics/evidence/libdevice_math/cuda-13.0.json`. Capture provenance +and reproduction instructions are recorded beside it. The cache lets ordinary +`generate` and `check` run without a CUDA installation. + +To verify the same ABI against a locally installed toolkit, use the pinned +toolchain's `llvm-tools` component: + +```bash +libdevice_llvm_dis="$(rustc --print sysroot)/lib/rustlib/$(rustc -vV | sed -n 's/^host: //p')/bin/llvm-dis" +cargo run -p cuda-intrinsics-gen -- verify-libdevice \ + --libdevice "${CUDA_PATH:-/usr/local/cuda}/nvvm/libdevice/libdevice.10.bc" \ + --llvm-dis "$libdevice_llvm_dis" +``` + +Verification accepts compatible newer toolkit bitcode and reports its hash. +To refresh evidence, first review and update the bitcode hash pin in the shard, +then run `extract-libdevice` with the same arguments, followed by `generate` and +`check`. Extraction hashes and disassembles the same bytes, requires exported +definitions with exact return/argument types, and writes the capture only after +all referenced signatures pass. + +Validation from `cuda-oxide/`: + +```bash +cargo test -p cuda-intrinsics-gen --locked --all-targets +cargo clippy -p cuda-intrinsics-gen --locked --all-targets -- -D warnings +cargo run -p cuda-intrinsics-gen --locked -- check +``` diff --git a/cuda-oxide/crates/cuda-intrinsics-gen/src/generate.rs b/cuda-oxide/crates/cuda-intrinsics-gen/src/generate.rs index b9476f2bae..4ecfab910d 100644 --- a/cuda-oxide/crates/cuda-intrinsics-gen/src/generate.rs +++ b/cuda-oxide/crates/cuda-intrinsics-gen/src/generate.rs @@ -22,6 +22,7 @@ const MANAGED_GENERATED_ROOTS: &[&str] = &[ "crates/mir-lower/src/convert", "crates/rustc-codegen-cuda/src", "intrinsics/probes", + "intrinsics/generated", ]; const GENERATED_MARKER: &[u8] = b"@generated by cuda-intrinsics-gen"; @@ -72,7 +73,14 @@ fn unindexed_overlay_shards(repo_root: &Path) -> Result> { "{} lists no shards; refusing to report every overlay file as unindexed", manifest_path.display() ); - let listed: BTreeSet<&str> = index.shards.iter().map(String::as_str).collect(); + // Libdevice is a separate evidence domain, read unconditionally by + // libdevice_math::outputs rather than folded into the hardware ABI catalog. + let listed: BTreeSet<&str> = index + .shards + .iter() + .map(String::as_str) + .chain(std::iter::once(crate::libdevice_math::SHARD)) + .collect(); let intrinsics_root = repo_root.join("intrinsics"); let overlay_dir = intrinsics_root.join("overlay"); @@ -142,6 +150,7 @@ pub fn run(repo_root: &Path, check: bool) -> Result<()> { let catalog_json = pretty_json(&catalog)?; let catalog_hash = sha256_bytes(catalog_json.as_bytes()); let mut outputs = all_outputs(&catalog, catalog_json, &catalog_hash)?; + outputs.extend(crate::libdevice_math::outputs(repo_root)?); let expected: BTreeSet = outputs.keys().cloned().collect(); for (path, contents) in &mut outputs { if path.extension().and_then(|extension| extension.to_str()) == Some("rs") { @@ -300,6 +309,17 @@ mod tests { fs::remove_dir_all(root).unwrap(); } + #[test] + fn libdevice_shard_has_an_explicit_separate_reader() { + let root = overlay_fixture( + "libdevice-indexed", + &["listed.toml"], + &["listed.toml", "libdevice_math.toml"], + ); + assert!(unindexed_overlay_shards(&root).unwrap().is_empty()); + fs::remove_dir_all(root).unwrap(); + } + /// The manifest grammar accepts nested shard paths, so the walk must /// descend: a forgotten file is no less forgotten for sitting in a /// subdirectory. diff --git a/cuda-oxide/crates/cuda-intrinsics-gen/src/libdevice_math/evidence.rs b/cuda-oxide/crates/cuda-intrinsics-gen/src/libdevice_math/evidence.rs new file mode 100644 index 0000000000..afae09de91 --- /dev/null +++ b/cuda-oxide/crates/cuda-intrinsics-gen/src/libdevice_math/evidence.rs @@ -0,0 +1,159 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use super::Evidence; +use crate::util::sha256_bytes; +use anyhow::{Context, Result, bail, ensure}; +use std::collections::BTreeMap; +use std::fs; +use std::io::Write; +use std::path::Path; +use std::process::{Command, Stdio}; + +pub(super) fn capture( + bitcode: &Path, + llvm_dis: &Path, + expected: &BTreeMap, +) -> Result { + let bytes = fs::read(bitcode).with_context(|| format!("read {}", bitcode.display()))?; + ensure!( + bytes.starts_with(b"BC\xc0\xde"), + "input is not LLVM bitcode" + ); + let version = Command::new(llvm_dis) + .arg("--version") + .output() + .with_context(|| format!("run {} --version", llvm_dis.display()))?; + ensure!(version.status.success(), "llvm-dis --version failed"); + let version = String::from_utf8(version.stdout).context("llvm-dis version is not UTF-8")?; + let mut child = Command::new(llvm_dis) + .args(["-o", "-", "-"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .with_context(|| format!("start {}", llvm_dis.display()))?; + // Hash and disassemble the same bytes, rather than reopening a mutable path. + child + .stdin + .take() + .context("llvm-dis stdin unavailable")? + .write_all(&bytes)?; + let output = child.wait_with_output().context("wait for llvm-dis")?; + ensure!( + output.status.success(), + "llvm-dis failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + let ir = String::from_utf8(output.stdout).context("llvm-dis output is not UTF-8")?; + Ok(Evidence { + schema: 1, + bitcode_sha256: sha256_bytes(&bytes), + llvm_dis_version: version.trim().to_string(), + symbols: signatures(&ir, expected)?, + }) +} + +fn llvm_type(text: &str) -> Result { + let mut words = text.split_whitespace(); + let first = words.next().context("missing LLVM parameter type")?; + if first == "ptr" || first.ends_with('*') { + if let Some(space) = words.next().filter(|word| word.starts_with("addrspace(")) { + return Ok(format!("ptr {}", space.trim_end_matches('*'))); + } + return Ok("ptr".into()); + } + ensure!( + matches!(first, "float" | "double" | "i32" | "i64" | "void"), + "unsupported libdevice ABI type {first}" + ); + Ok(first.into()) +} + +/// Parse only requested exported definitions. Unrelated intrinsics and helper +/// functions are controls; declarations alone are insufficient evidence. +pub(super) fn signatures( + ir: &str, + expected: &BTreeMap, +) -> Result> { + let mut found = BTreeMap::new(); + let mut header = String::new(); + for line in ir.lines() { + let line = line.trim(); + if line.starts_with("define ") { + ensure!(header.is_empty(), "unterminated LLVM function definition"); + header.push_str(line); + } else if !header.is_empty() { + header.push(' '); + header.push_str(line); + } else { + continue; + } + if !header.contains('{') { + continue; + } + let definition = std::mem::take(&mut header); + let Some((prefix, tail)) = definition.split_once('@') else { + continue; + }; + let Some((name, parameters)) = tail.split_once('(') else { + continue; + }; + let name = name.trim().trim_matches('"'); + let Some(expected_signature) = expected.get(name) else { + continue; + }; + ensure!( + !prefix + .split_whitespace() + .any(|word| matches!(word, "internal" | "private" | "available_externally")), + "libdevice symbol {name} is not an exported definition" + ); + let return_type = prefix + .split_whitespace() + .last() + .context("missing LLVM return type")?; + let return_type = llvm_type(return_type)?; + // Address-space parentheses can occur inside the parameter list. + let mut depth = 1; + let mut end = None; + for (index, byte) in parameters.bytes().enumerate() { + match byte { + b'(' => depth += 1, + b')' => depth -= 1, + _ => {} + } + if depth == 0 { + end = Some(index); + break; + } + } + let end = end.context("unterminated LLVM parameter list")?; + let parameters = ¶meters[..end]; + let arguments = if parameters.trim().is_empty() { + Vec::new() + } else { + parameters + .split(',') + .map(llvm_type) + .collect::>>()? + }; + let signature = format!("{return_type}({})", arguments.join(", ")); + ensure!( + &signature == expected_signature, + "libdevice signature mismatch for {name}: expected {expected_signature}, found {signature}" + ); + ensure!( + found.insert(name.into(), signature).is_none(), + "duplicate libdevice definition {name}" + ); + } + for symbol in expected.keys() { + if !found.contains_key(symbol) { + bail!("libdevice definition is missing: {symbol}"); + } + } + Ok(found) +} diff --git a/cuda-oxide/crates/cuda-intrinsics-gen/src/libdevice_math/mod.rs b/cuda-oxide/crates/cuda-intrinsics-gen/src/libdevice_math/mod.rs new file mode 100644 index 0000000000..c21399936b --- /dev/null +++ b/cuda-oxide/crates/cuda-intrinsics-gen/src/libdevice_math/mod.rs @@ -0,0 +1,408 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! Staged libm mappings. This family has bitcode evidence, not hardware ABI IDs. + +mod evidence; +mod render; +#[cfg(test)] +mod tests; + +use crate::util::{pretty_json, read_json, write_if_changed}; +use anyhow::{Context, Result, ensure}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::path::{Component, Path, PathBuf}; + +/// Read separately from the hardware shard index so hardware catalog hashes and +/// ABI IDs remain unchanged. `generate` reconciles this input explicitly. +pub const SHARD: &str = "overlay/libdevice_math.toml"; + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct Shard { + schema: u32, + family: String, + libdevice_math: Math, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct Math { + evidence: String, + libdevice_sha256: String, + widths: Vec, + #[serde(rename = "function")] + functions: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct Function { + name: String, + signature: Signature, + #[serde(default)] + libm: Vec, + #[serde(default)] + cmath: Option, + #[serde(default)] + placeholders: Vec, + #[serde(default)] + libdevice: Option, + #[serde(default)] + llvm: Option, + #[serde(default)] + lowering: Option, + #[serde(default)] + link: Link, +} + +#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq)] +enum Signature { + #[serde(rename = "(F) -> F")] + Unary, + #[serde(rename = "(F, F) -> F")] + Binary, + #[serde(rename = "(F, F, F) -> F")] + Ternary, + #[serde(rename = "(F) -> i32")] + IntegerResult, + #[serde(rename = "(F, i32) -> F")] + IntegerArgument, + #[serde(rename = "(F) -> (F, i32)")] + FloatIntegerPair, + #[serde(rename = "(F) -> (F, F)")] + FloatPair, + #[serde(rename = "(F, F) -> (F, i32)")] + RemainderPair, +} + +impl Signature { + fn rust(self, width: &str) -> String { + let generic = match self { + Self::Unary => "(F) -> F", + Self::Binary => "(F, F) -> F", + Self::Ternary => "(F, F, F) -> F", + Self::IntegerResult => "(F) -> i32", + Self::IntegerArgument => "(F, i32) -> F", + Self::FloatIntegerPair => "(F) -> (F, i32)", + Self::FloatPair => "(F) -> (F, F)", + Self::RemainderPair => "(F, F) -> (F, i32)", + }; + generic.replace('F', width) + } + + fn c_abi(self, width: &str) -> String { + let float = if width == "f32" { "float" } else { "double" }; + match self { + Self::Unary => format!("{float}({float})"), + Self::Binary => format!("{float}({float}, {float})"), + Self::Ternary => format!("{float}({float}, {float}, {float})"), + Self::IntegerResult => format!("i32({float})"), + Self::IntegerArgument => format!("{float}({float}, i32)"), + Self::FloatIntegerPair | Self::FloatPair => format!("{float}({float}, ptr)"), + Self::RemainderPair => format!("{float}({float}, {float}, ptr)"), + } + } +} + +#[derive(Debug, Clone, Copy, Default, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +enum Link { + #[default] + Always, + Libnvvm, + Never, +} + +impl Link { + fn as_str(self) -> &'static str { + match self { + Self::Always => "always", + Self::Libnvvm => "libnvvm", + Self::Never => "never", + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct Mapping { + libm: String, + signature: String, + placeholders: Vec, + cmath: Option, + libdevice: Option, + llvm: Option, + lowering: Option, + link: Link, + c_abi: String, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct Evidence { + schema: u32, + bitcode_sha256: String, + llvm_dis_version: String, + symbols: BTreeMap, +} + +fn identifier(value: &str) -> bool { + !value.is_empty() + && (value.as_bytes()[0].is_ascii_alphabetic() || value.starts_with('_')) + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') +} + +fn expand(template: &str, width: &str) -> Result { + let value = template + .replace("{width}", width) + .replace("{suffix}", if width == "f32" { "f" } else { "" }); + ensure!( + identifier(&value), + "invalid math name/template {template:?}" + ); + Ok(value) +} + +fn parse(bytes: &[u8]) -> Result { + let shard: Shard = toml::from_slice(bytes).context("parse libdevice_math shard")?; + ensure!(shard.schema == 1, "unsupported libdevice_math schema"); + ensure!( + shard.family == "libdevice_math", + "wrong libdevice_math family" + ); + let math = &shard.libdevice_math; + ensure!( + math.widths == ["f32", "f64"], + "libdevice_math widths must be f32, f64" + ); + ensure!( + !math.functions.is_empty(), + "libdevice_math has no functions" + ); + ensure!( + math.libdevice_sha256.len() == 64 + && math + .libdevice_sha256 + .bytes() + .all(|byte| byte.is_ascii_hexdigit()), + "invalid libdevice bitcode SHA-256" + ); + let path = Path::new(&math.evidence); + ensure!( + path.starts_with("evidence") + && path.extension().and_then(|part| part.to_str()) == Some("json") + && path + .components() + .all(|part| matches!(part, Component::Normal(_))), + "libdevice evidence must be a relative JSON path under evidence/" + ); + Ok(shard) +} + +fn read(root: &Path) -> Result { + let path = root.join("intrinsics").join(SHARD); + parse(&fs::read(&path).with_context(|| format!("read {}", path.display()))?) +} + +fn mappings(shard: &Shard) -> Result> { + let mut names = BTreeSet::new(); + let mut exports = BTreeSet::new(); + let mut cmath_paths = BTreeSet::new(); + let mut scalar_placeholders = BTreeSet::new(); + let mut result = Vec::new(); + for function in &shard.libdevice_math.functions { + ensure!(identifier(&function.name), "invalid math function name"); + ensure!( + names.insert(&function.name), + "duplicate math function {}", + function.name + ); + if let Some(lowering) = &function.lowering { + ensure!( + lowering + .strip_prefix("handwritten:") + .is_some_and(identifier), + "invalid handwritten math lowering {lowering:?}" + ); + } else { + ensure!( + !matches!( + function.signature, + Signature::FloatPair | Signature::FloatIntegerPair | Signature::RemainderPair + ), + "tuple-returning math requires an explicit handwritten adapter" + ); + } + let aliases = if function.libm.is_empty() { + vec![function.name.clone()] + } else { + function.libm.clone() + }; + for width in &shard.libdevice_math.widths { + let symbol = if let Some(symbol) = &function.libdevice { + Some(expand(symbol, width)?) + } else if function.lowering.is_none() { + Some(format!( + "__nv_{}{}", + function.name, + if width == "f32" { "f" } else { "" } + )) + } else { + None + }; + if let Some(symbol) = &symbol { + ensure!( + symbol.starts_with("__nv_"), + "math symbol must start with __nv_" + ); + } + let llvm = function + .llvm + .as_deref() + .map(|name| expand(name, width)) + .transpose()?; + ensure!( + (function.link == Link::Libnvvm) == llvm.is_some(), + "backend-dependent math must name its LLVM intrinsic" + ); + ensure!( + function.link != Link::Libnvvm || symbol.is_some(), + "backend-dependent math must name its libdevice ABI" + ); + ensure!( + function.link != Link::Never || (symbol.is_none() && function.lowering.is_some()), + "never-linked math must use a handwritten lowering without a libdevice call" + ); + let templates = if function.placeholders.is_empty() { + vec![format!("{}{{width}}", function.name)] + } else { + function.placeholders.clone() + }; + let placeholders = templates + .iter() + .map(|name| expand(name, width)) + .collect::>>()?; + if function.lowering.is_none() { + ensure!( + placeholders.len() == 1, + "plain math requires one placeholder" + ); + ensure!( + scalar_placeholders.insert((width, placeholders[0].clone())), + "duplicate scalar math placeholder {} for {width}", + placeholders[0] + ); + } + let cmath = function + .cmath + .as_deref() + .map(|name| expand(name, width).map(|name| format!("std::sys::cmath::{name}"))) + .transpose()?; + if let Some(path) = &cmath { + ensure!( + cmath_paths.insert(path.clone()), + "duplicate cmath spelling {path}" + ); + } + for alias in &aliases { + ensure!(identifier(alias), "invalid libm alias {alias:?}"); + let libm = format!("{alias}{}", if width == "f32" { "f" } else { "" }); + ensure!( + exports.insert(libm.clone()), + "duplicate libm spelling {libm}" + ); + result.push(Mapping { + libm, + signature: function.signature.rust(width), + placeholders: placeholders.clone(), + cmath: cmath.clone(), + libdevice: symbol.clone(), + llvm: llvm.clone(), + lowering: function.lowering.clone(), + link: function.link, + c_abi: function.signature.c_abi(width), + }); + } + } + } + result.sort_by(|left, right| left.libm.cmp(&right.libm)); + Ok(result) +} + +fn expected_symbols(rows: &[Mapping]) -> Result> { + let mut result = BTreeMap::new(); + for row in rows { + if let Some(symbol) = &row.libdevice + && let Some(previous) = result.insert(symbol.clone(), row.c_abi.clone()) + { + ensure!( + previous == row.c_abi, + "conflicting libdevice signatures for {symbol}" + ); + } + } + Ok(result) +} + +fn validate_evidence(shard: &Shard, rows: &[Mapping], evidence: &Evidence) -> Result<()> { + ensure!( + evidence.schema == 1, + "unsupported libdevice evidence schema" + ); + ensure!( + evidence.bitcode_sha256 == shard.libdevice_math.libdevice_sha256, + "libdevice bitcode hash does not match the shard pin" + ); + ensure!( + !evidence.llvm_dis_version.trim().is_empty(), + "missing llvm-dis provenance" + ); + for (symbol, signature) in expected_symbols(rows)? { + ensure!( + evidence.symbols.get(&symbol) == Some(&signature), + "libdevice evidence missing or signature mismatch for {symbol}: expected {signature}" + ); + } + Ok(()) +} + +pub fn outputs(root: &Path) -> Result> { + let shard = read(root)?; + let rows = mappings(&shard)?; + let evidence: Evidence = + read_json(&root.join("intrinsics").join(&shard.libdevice_math.evidence))?; + validate_evidence(&shard, &rows, &evidence)?; + Ok(render::outputs(&rows, &evidence.bitcode_sha256)) +} + +/// Capture reviewed signatures, or compare a local toolkit against the same +/// rows. Verification accepts a compatible newer toolkit without changing pins. +pub fn inspect(root: &Path, bitcode: &Path, llvm_dis: &Path, capture: bool) -> Result<()> { + let shard = read(root)?; + let rows = mappings(&shard)?; + let expected = expected_symbols(&rows)?; + let evidence = evidence::capture(bitcode, llvm_dis, &expected)?; + if capture { + validate_evidence(&shard, &rows, &evidence)?; + let path = root.join("intrinsics").join(&shard.libdevice_math.evidence); + write_if_changed(&path, &pretty_json(&evidence)?)?; + println!( + "captured {} libdevice signatures in {}", + expected.len(), + path.display() + ); + } else { + println!( + "verified {} libdevice signatures; bitcode SHA-256 {}", + expected.len(), + evidence.bitcode_sha256 + ); + } + Ok(()) +} diff --git a/cuda-oxide/crates/cuda-intrinsics-gen/src/libdevice_math/render.rs b/cuda-oxide/crates/cuda-intrinsics-gen/src/libdevice_math/render.rs new file mode 100644 index 0000000000..612dc00999 --- /dev/null +++ b/cuda-oxide/crates/cuda-intrinsics-gen/src/libdevice_math/render.rs @@ -0,0 +1,47 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use super::Mapping; +use std::collections::BTreeMap; +use std::fmt::Write; +use std::path::PathBuf; + +pub(super) fn outputs(rows: &[Mapping], bitcode_hash: &str) -> BTreeMap { + let mut rust = String::from( + "/*\n * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n * SPDX-License-Identifier: Apache-2.0\n */\n\n// @generated by cuda-intrinsics-gen from overlay/libdevice_math.toml. DO NOT EDIT.\n// Staged metadata; compiler dispatch is still handwritten.\n\n#[derive(Debug, Clone, Copy, PartialEq, Eq)]\npub struct MathMapping {\n pub libm: &'static str,\n pub signature: &'static str,\n pub placeholders: &'static [&'static str],\n pub cmath: Option<&'static str>,\n pub libdevice: Option<&'static str>,\n pub llvm: Option<&'static str>,\n pub lowering: Option<&'static str>,\n pub link: &'static str,\n}\n\npub const MAPPINGS: &[MathMapping] = &[\n", + ); + let mut reference = format!( + "\n\n\n\n# Staged libm mappings\n\n{} mapped f32/f64 libm exports. Compiler dispatch remains handwritten.\n\nLibdevice bitcode SHA-256: `{bitcode_hash}`.\n\n`libnvvm` links only on that backend; `always` links on both; `never` uses\na handwritten expansion. Aliases are separate exports sharing a mapping.\n\n| libm export | Rust signature | libdevice symbol | LLVM NVPTX alternative | Handwritten lowering | Linking |\n| --- | --- | --- | --- | --- | --- |\n", + rows.len() + ); + for row in rows { + writeln!(rust, "MathMapping {{ libm: {:?}, signature: {:?}, placeholders: &{:?}, cmath: {:?}, libdevice: {:?}, llvm: {:?}, lowering: {:?}, link: {:?} }},", row.libm, row.signature, row.placeholders, row.cmath, row.libdevice, row.llvm, row.lowering, row.link.as_str()).unwrap(); + writeln!( + reference, + "| `{}` | `{}` | {} | {} | {} | `{}` |", + row.libm, + row.signature, + code(row.libdevice.as_deref()), + code(row.llvm.as_deref()), + code(row.lowering.as_deref()), + row.link.as_str() + ) + .unwrap(); + } + rust.push_str("];\n"); + BTreeMap::from([ + ( + PathBuf::from("intrinsics/generated/libdevice_math.rs"), + rust, + ), + (PathBuf::from("intrinsics/libdevice-math.md"), reference), + ]) +} + +fn code(value: Option<&str>) -> String { + value + .map(|text| format!("`{text}`")) + .unwrap_or_else(|| " ".into()) +} diff --git a/cuda-oxide/crates/cuda-intrinsics-gen/src/libdevice_math/tests.rs b/cuda-oxide/crates/cuda-intrinsics-gen/src/libdevice_math/tests.rs new file mode 100644 index 0000000000..cf51aa8ae7 --- /dev/null +++ b/cuda-oxide/crates/cuda-intrinsics-gen/src/libdevice_math/tests.rs @@ -0,0 +1,400 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use super::*; + +mod generated { + include!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../intrinsics/generated/libdevice_math.rs" + )); +} + +fn root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../..") +} + +fn fixture(functions: &str) -> Shard { + parse(format!("schema = 1\nfamily = \"libdevice_math\"\n[libdevice_math]\nevidence = \"evidence/test.json\"\nlibdevice_sha256 = \"{}\"\nwidths = [\"f32\", \"f64\"]\n{functions}", "a".repeat(64)).as_bytes()).unwrap() +} + +#[test] +fn staged_snapshot_has_exactly_the_original_72_exports() { + let shard = read(&root()).unwrap(); + let rows = mappings(&shard).unwrap(); + assert_eq!(rows.len(), 72); + assert_eq!(expected_symbols(&rows).unwrap().len(), 64); + assert_eq!(generated::MAPPINGS.len(), rows.len()); + for (actual, row) in generated::MAPPINGS.iter().zip(&rows) { + assert_eq!(actual.libm, row.libm); + assert_eq!(actual.signature, row.signature); + assert_eq!( + actual.placeholders, + row.placeholders + .iter() + .map(String::as_str) + .collect::>() + ); + assert_eq!(actual.cmath, row.cmath.as_deref()); + assert_eq!(actual.libdevice, row.libdevice.as_deref()); + assert_eq!(actual.llvm, row.llvm.as_deref()); + assert_eq!(actual.lowering, row.lowering.as_deref()); + assert_eq!(actual.link, row.link.as_str()); + } +} + +fn quoted(text: &str) -> impl Iterator { + text.split('"').skip(1).step_by(2) +} + +fn function_body<'a>(source: &'a str, name: &str) -> &'a str { + let start = source.find(&format!("fn {name}(")).unwrap(); + let rest = &source[start..]; + let end = [rest.find("\n fn "), rest.find("\n}\n")] + .into_iter() + .flatten() + .min() + .unwrap(); + &rest[..end] +} + +fn variants(source: &str, name: &str, target: &str) -> BTreeMap { + let mut result = BTreeMap::new(); + for line in function_body(source, name).lines() { + let Some((left, right)) = line.split_once(target) else { + continue; + }; + for variant in left.split('|') { + if let Some(variant) = variant.trim().strip_prefix("Self::") { + result.insert( + variant.into(), + right.split([')', ',']).next().unwrap().trim().into(), + ); + } + } + } + result +} + +/// Compare staged rows against the live handwritten routes, not another copy +/// of an expected mapping list. Additional hand-landed functions may coexist +/// until the compiler-table migration, without making this PR depend on them. +#[test] +fn staged_rows_preserve_existing_import_placeholder_symbol_and_link_routes() { + let importer = + include_str!("../../../mir-importer/src/translator/terminator/intrinsics/float_math.rs"); + let dialect = include_str!("../../../dialect-mir/src/rust_intrinsics.rs"); + let lower = include_str!("../../../mir-lower/src/convert/ops/call.rs"); + let start = importer.find("fn from_libm_path(").unwrap(); + let end = start + + importer[start..] + .find("fn from_fast_intrinsic_path(") + .unwrap(); + let mut imports = BTreeMap::new(); + for line in importer[start..end].lines() { + if let Some((left, right)) = line.split_once("=> Some(Self::") { + for name in quoted(left) { + imports.insert(name, right.split(')').next().unwrap()); + } + } + } + let placeholders = variants(importer, "placeholder_callee", "=> rust_intrinsics::"); + let constants: BTreeMap<_, _> = dialect + .lines() + .filter_map(|line| { + let line = line.strip_prefix("pub const ")?; + let (name, _) = line.split_once(':')?; + Some((name, quoted(line).next()?)) + }) + .collect(); + let symbols = variants(lower, "libdevice_name", "=> Ok("); + let native = variants(lower, "llvm_intrinsic_name", "=> Ok(Some("); + let always = function_body(dialect, "is_libdevice_backed_placeholder"); + let conditional = function_body(dialect, "is_backend_dependent_libdevice_placeholder"); + let contains_constant = |body: &str, name: &str| { + body.split(|ch: char| !ch.is_ascii_alphanumeric() && ch != '_') + .any(|token| token == name) + }; + for row in mappings(&read(&root()).unwrap()).unwrap() { + if row.libm == "sincos" || row.libm == "sincosf" { + assert!(importer.contains(&format!("\"{}\" => Some(", row.libm))); + assert_eq!(row.placeholders.len(), 2); + assert_eq!(row.lowering.as_deref(), Some("handwritten:emit_sincos")); + continue; + } + let variant = imports[row.libm.as_str()]; + let constant = &placeholders[variant]; + assert_eq!( + row.placeholders, + [constants[constant.as_str()]], + "{}", + row.libm + ); + assert_eq!( + contains_constant(always, constant), + row.link == Link::Always, + "{}", + row.libm + ); + assert_eq!( + contains_constant(conditional, constant), + row.link == Link::Libnvvm, + "{}", + row.libm + ); + if variant == "Fabs" { + for symbol in [&row.libdevice, &row.llvm].into_iter().flatten() { + assert!(lower.contains(&format!("\"{symbol}\""))); + } + } else { + assert_eq!( + symbols.get(variant).map(|name| name.trim_matches('"')), + row.libdevice.as_deref(), + "{}", + row.libm + ); + assert_eq!( + native.get(variant).map(|name| name.trim_matches('"')), + row.llvm.as_deref(), + "{}", + row.libm + ); + } + if let Some(path) = &row.cmath { + assert!(importer.contains(&format!("\"{path}\" => Some(Self::{variant})"))); + } + } +} + +#[test] +fn aliases_and_handwritten_controls_keep_their_semantics() { + let rows = mappings(&read(&root()).unwrap()).unwrap(); + let find = |name: &str| rows.iter().find(|row| row.libm == name).unwrap(); + for (alias, original) in [("roundeven", "rint"), ("roundevenf", "rintf")] { + assert_eq!(find(alias).placeholders, find(original).placeholders); + assert_eq!(find(alias).libdevice, find(original).libdevice); + assert_eq!(find(alias).llvm, find(original).llvm); + } + for name in ["fmax", "fmaxf", "fmin", "fminf"] { + assert_eq!(find(name).link, Link::Never); + assert!(find(name).libdevice.is_none()); + assert!(find(name).llvm.is_none()); + } + assert_eq!(find("fabs").placeholders, find("fabsf").placeholders); + assert_eq!(find("floor").link, Link::Libnvvm); + assert_eq!(find("sqrt").link, Link::Always); +} + +#[test] +fn render_is_independent_of_row_order_and_check_detects_drift() { + let mut shard = read(&root()).unwrap(); + let before = render::outputs( + &mappings(&shard).unwrap(), + &shard.libdevice_math.libdevice_sha256, + ); + shard.libdevice_math.functions.reverse(); + let after = render::outputs( + &mappings(&shard).unwrap(), + &shard.libdevice_math.libdevice_sha256, + ); + assert_eq!(before, after); + for (path, source) in before { + let source = if path.extension().is_some_and(|extension| extension == "rs") { + crate::util::rustfmt_source(&source).unwrap() + } else { + source + }; + crate::util::check_contents(&root().join(&path), &source).unwrap(); + let mut stale = source.clone(); + stale.push_str("// drift\n"); + assert!(crate::util::check_contents(&root().join(&path), &stale).is_err()); + } +} + +#[test] +fn signature_classes_distinguish_integer_results_arguments_and_out_parameters() { + for (signature, c32, c64) in [ + (Signature::Unary, "float(float)", "double(double)"), + ( + Signature::Binary, + "float(float, float)", + "double(double, double)", + ), + ( + Signature::Ternary, + "float(float, float, float)", + "double(double, double, double)", + ), + (Signature::IntegerResult, "i32(float)", "i32(double)"), + ( + Signature::IntegerArgument, + "float(float, i32)", + "double(double, i32)", + ), + ( + Signature::FloatIntegerPair, + "float(float, ptr)", + "double(double, ptr)", + ), + ( + Signature::FloatPair, + "float(float, ptr)", + "double(double, ptr)", + ), + ( + Signature::RemainderPair, + "float(float, float, ptr)", + "double(double, double, ptr)", + ), + ] { + assert_eq!(signature.c_abi("f32"), c32); + assert_eq!(signature.c_abi("f64"), c64); + } + let adapter = fixture( + "[[libdevice_math.function]]\nname = \"frexp\"\nsignature = \"(F) -> (F, i32)\"\nlowering = \"handwritten:emit_frexp\"\nlibdevice = \"__nv_frexp{suffix}\"\n", + ); + let rows = mappings(&adapter).unwrap(); + assert_eq!( + expected_symbols(&rows).unwrap()["__nv_frexpf"], + "float(float, ptr)" + ); + assert_eq!(rows[0].signature, "(f64) -> (f64, i32)"); +} + +#[test] +fn malformed_rows_fail_closed() { + let base = "[[libdevice_math.function]]\nname = \"demo\"\nsignature = \"(F) -> F\"\n"; + for (suffix, expected) in [ + ( + "[[libdevice_math.function]]\nname = \"demo\"\nsignature = \"(F) -> F\"\n", + "duplicate math function", + ), + ("libm = [\"demo\", \"demo\"]\n", "duplicate libm spelling"), + ( + "placeholders = [\"bad{unknown}\"]\n", + "invalid math name/template", + ), + ("link = \"never\"\n", "never-linked math"), + ("link = \"libnvvm\"\n", "backend-dependent math"), + ("llvm = \"llvm_demo_{width}\"\n", "backend-dependent math"), + ( + "placeholders = [\"left{width}\", \"right{width}\"]\n", + "one placeholder", + ), + ( + "lowering = \"handwritten:emit_demo\"\nllvm = \"llvm_demo_{width}\"\nlink = \"libnvvm\"\n", + "libdevice ABI", + ), + ( + "[[libdevice_math.function]]\nname = \"other\"\nsignature = \"(F) -> F\"\nplaceholders = [\"demo{width}\"]\n", + "duplicate scalar math placeholder", + ), + ( + "libdevice = \"user_demo{suffix}\"\n", + "must start with __nv_", + ), + ( + "lowering = \"arbitrary_rust()\"\n", + "invalid handwritten math lowering", + ), + ] { + let shard = fixture(&format!("{base}{suffix}")); + let error = mappings(&shard).unwrap_err().to_string(); + assert!(error.contains(expected), "{error}"); + } + let tuple = fixture(&base.replace("(F) -> F", "(F) -> (F, i32)")); + assert!( + mappings(&tuple) + .unwrap_err() + .to_string() + .contains("handwritten adapter") + ); + let cmath = fixture(&format!( + "{base}cmath = \"demo{{suffix}}\"\n[[libdevice_math.function]]\nname = \"other\"\nsignature = \"(F) -> F\"\ncmath = \"demo{{suffix}}\"\n" + )); + assert!( + mappings(&cmath) + .unwrap_err() + .to_string() + .contains("duplicate cmath") + ); + let conflict = fixture(&format!( + "{base}libdevice = \"__nv_shared{{suffix}}\"\n[[libdevice_math.function]]\nname = \"other\"\nsignature = \"(F, F) -> F\"\nlibdevice = \"__nv_shared{{suffix}}\"\n" + )); + assert!( + expected_symbols(&mappings(&conflict).unwrap()) + .unwrap_err() + .to_string() + .contains("conflicting libdevice signatures") + ); +} + +#[test] +fn schema_and_evidence_pins_are_strict() { + let source = fs::read_to_string(root().join("intrinsics").join(SHARD)).unwrap(); + for changed in [ + source.replacen("schema = 1", "schema = 2", 1), + source.replacen("family = \"libdevice_math\"", "family = \"scalar_math\"", 1), + source.replacen("[\"f32\", \"f64\"]", "[\"f16\"]", 1), + source.replacen( + "evidence/libdevice_math/cuda-13.0.json", + "evidence/../outside.json", + 1, + ), + source.replacen("schema = 1", "schema = 1\nunknown = true", 1), + source.replacen("(F) -> F", "(F) -> bool", 1), + ] { + assert!(parse(changed.as_bytes()).is_err()); + } + let shard = read(&root()).unwrap(); + let rows = mappings(&shard).unwrap(); + let mut evidence: Evidence = read_json( + &root() + .join("intrinsics") + .join(&shard.libdevice_math.evidence), + ) + .unwrap(); + validate_evidence(&shard, &rows, &evidence).unwrap(); + evidence + .symbols + .insert("__nv_sinf".into(), "double(double)".into()); + assert!( + validate_evidence(&shard, &rows, &evidence) + .unwrap_err() + .to_string() + .contains("signature mismatch") + ); + evidence.bitcode_sha256 = "0".repeat(64); + assert!( + validate_evidence(&shard, &rows, &evidence) + .unwrap_err() + .to_string() + .contains("hash") + ); +} + +#[test] +fn llvm_evidence_requires_exported_definitions_and_exact_signatures() { + let expected = BTreeMap::from([ + ("__nv_demo".into(), "double(double, i32)".into()), + ("__nv_pairf".into(), "float(float, ptr)".into()), + ]); + let ir = "declare float @__nv_unrelated(float)\ndefine void @control() { ret void }\ndefine double @__nv_demo(\n double %x, i32 noundef %n) { ret double %x }\ndefine float @__nv_pairf(float %x, ptr nocapture %out) { ret float %x }\n"; + assert_eq!(evidence::signatures(ir, &expected).unwrap(), expected); + for changed in [ + ir.replace("define double @__nv_demo", "declare double @__nv_demo"), + ir.replace("i32 noundef %n", "i64 %n"), + ir.replace( + "define double @__nv_demo", + "define internal double @__nv_demo", + ), + ir.replace("ptr nocapture", "ptr addrspace(1) nocapture"), + format!("{ir}\ndefine double @__nv_demo(double %x, i32 %n) {{ ret double %x }}"), + ] { + assert!(evidence::signatures(&changed, &expected).is_err()); + } + let typed = ir.replace("ptr nocapture", "i32* nocapture"); + assert_eq!(evidence::signatures(&typed, &expected).unwrap(), expected); +} diff --git a/cuda-oxide/crates/cuda-intrinsics-gen/src/main.rs b/cuda-oxide/crates/cuda-intrinsics-gen/src/main.rs index 92b3c4c640..13361762a2 100644 --- a/cuda-oxide/crates/cuda-intrinsics-gen/src/main.rs +++ b/cuda-oxide/crates/cuda-intrinsics-gen/src/main.rs @@ -7,6 +7,7 @@ mod abi_history; mod coverage; mod extract; mod generate; +mod libdevice_math; mod model; mod probe; mod ptx; @@ -74,6 +75,21 @@ fn try_main() -> Result<()> { reject_extra(arguments)?; generate::run(&repo_root, true) } + "extract-libdevice" | "verify-libdevice" => { + let bitcode = take_option(&mut arguments, "--libdevice")? + .map(PathBuf::from) + .context("libdevice command requires --libdevice FILE")?; + let llvm_dis = take_option(&mut arguments, "--llvm-dis")? + .map(PathBuf::from) + .context("libdevice command requires --llvm-dis FILE")?; + reject_extra(arguments)?; + libdevice_math::inspect( + &repo_root, + &bitcode, + &llvm_dis, + command == "extract-libdevice", + ) + } "coverage" => { let family = take_option(&mut arguments, "--family")?; reject_extra(arguments)?; @@ -218,6 +234,8 @@ fn print_usage() { cuda-intrinsics-gen extract --llvm-src DIR --llvm-tblgen FILE [--repo-root DIR]\n \ cuda-intrinsics-gen generate [--repo-root DIR]\n \ cuda-intrinsics-gen check [--repo-root DIR]\n \ + cuda-intrinsics-gen extract-libdevice --libdevice FILE --llvm-dis FILE [--repo-root DIR]\n \ + cuda-intrinsics-gen verify-libdevice --libdevice FILE --llvm-dis FILE [--repo-root DIR]\n \ cuda-intrinsics-gen coverage [--family NAME] [--repo-root DIR]\n \ cuda-intrinsics-gen check-abi-history --base-ref REF [--repo-root DIR]\n \ cuda-intrinsics-gen probe [--all | --intrinsic ID] [--llc FILE] [--skip-terminal] [--per-target] [--repo-root DIR]\n \ diff --git a/cuda-oxide/intrinsics/evidence/libdevice_math/README.md b/cuda-oxide/intrinsics/evidence/libdevice_math/README.md new file mode 100644 index 0000000000..4ca16f6c8e --- /dev/null +++ b/cuda-oxide/intrinsics/evidence/libdevice_math/README.md @@ -0,0 +1,33 @@ + + + +# Libdevice math ABI capture + +`cuda-13.0.json` contains 64 distinct exported function signatures referenced by +the initial libdevice math shard. They were extracted from NVIDIA's CUDA +13.0.88 `libdevice.10.bc` with LLVM 23.1.1 from the pinned Rust 1.99.1 toolchain. + +Source: [NVIDIA nvidia-nvvm 13.0.88](https://pypi.org/project/nvidia-nvvm/13.0.88/), +Linux x86-64 wheel +`nvidia_nvvm-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl`. + +- Wheel SHA-256: `c5f41ffeb6466944a026dfa5317d7d85355c119bbec279205d22f1869d1054e0`. +- Bitcode member: `nvidia/cu13/nvvm/libdevice/libdevice.10.bc`. +- Bitcode SHA-256: `91334d6e12748f6cb5bbf0a1cd965a56bcd93dc4f496d2e5c5f8c6e523094356`. + +To reproduce the capture from matching local bitcode, run from `cuda-oxide/`: + +```bash +libdevice_llvm_dis="$(rustc --print sysroot)/lib/rustlib/$(rustc -vV | sed -n 's/^host: //p')/bin/llvm-dis" +cargo run -p cuda-intrinsics-gen --locked -- extract-libdevice \ + --libdevice /path/to/libdevice.10.bc --llvm-dis "$libdevice_llvm_dis" +cargo run -p cuda-intrinsics-gen --locked -- generate +cargo run -p cuda-intrinsics-gen --locked -- check +``` + +The extractor checks definitions, return types, operand types, and duplicate +symbols before writing. Only signatures required by the shard are retained. +Opaque LLVM pointers retain their address space, but do not encode a pointee +type; out-parameter semantics remain the handwritten adapter's responsibility. +Ordinary generation validates the capture's bitcode hash against the shard pin +and every referenced ABI against the captured signatures. diff --git a/cuda-oxide/intrinsics/evidence/libdevice_math/cuda-13.0.json b/cuda-oxide/intrinsics/evidence/libdevice_math/cuda-13.0.json new file mode 100644 index 0000000000..6c9e8b70f7 --- /dev/null +++ b/cuda-oxide/intrinsics/evidence/libdevice_math/cuda-13.0.json @@ -0,0 +1,71 @@ +{ + "schema": 1, + "bitcode_sha256": "91334d6e12748f6cb5bbf0a1cd965a56bcd93dc4f496d2e5c5f8c6e523094356", + "llvm_dis_version": "LLVM (http://llvm.org/):\n LLVM version 23.1.1-rust-1.99.1-stable\n Optimized build.", + "symbols": { + "__nv_acos": "double(double)", + "__nv_acosf": "float(float)", + "__nv_acosh": "double(double)", + "__nv_acoshf": "float(float)", + "__nv_asin": "double(double)", + "__nv_asinf": "float(float)", + "__nv_asinh": "double(double)", + "__nv_asinhf": "float(float)", + "__nv_atan": "double(double)", + "__nv_atan2": "double(double, double)", + "__nv_atan2f": "float(float, float)", + "__nv_atanf": "float(float)", + "__nv_atanh": "double(double)", + "__nv_atanhf": "float(float)", + "__nv_cbrt": "double(double)", + "__nv_cbrtf": "float(float)", + "__nv_ceil": "double(double)", + "__nv_ceilf": "float(float)", + "__nv_copysign": "double(double, double)", + "__nv_copysignf": "float(float, float)", + "__nv_cos": "double(double)", + "__nv_cosf": "float(float)", + "__nv_cosh": "double(double)", + "__nv_coshf": "float(float)", + "__nv_exp": "double(double)", + "__nv_exp2": "double(double)", + "__nv_exp2f": "float(float)", + "__nv_expf": "float(float)", + "__nv_expm1": "double(double)", + "__nv_expm1f": "float(float)", + "__nv_fabs": "double(double)", + "__nv_fabsf": "float(float)", + "__nv_floor": "double(double)", + "__nv_floorf": "float(float)", + "__nv_fma": "double(double, double, double)", + "__nv_fmaf": "float(float, float, float)", + "__nv_hypot": "double(double, double)", + "__nv_hypotf": "float(float, float)", + "__nv_log": "double(double)", + "__nv_log10": "double(double)", + "__nv_log10f": "float(float)", + "__nv_log1p": "double(double)", + "__nv_log1pf": "float(float)", + "__nv_log2": "double(double)", + "__nv_log2f": "float(float)", + "__nv_logf": "float(float)", + "__nv_pow": "double(double, double)", + "__nv_powf": "float(float, float)", + "__nv_rint": "double(double)", + "__nv_rintf": "float(float)", + "__nv_round": "double(double)", + "__nv_roundf": "float(float)", + "__nv_sin": "double(double)", + "__nv_sinf": "float(float)", + "__nv_sinh": "double(double)", + "__nv_sinhf": "float(float)", + "__nv_sqrt": "double(double)", + "__nv_sqrtf": "float(float)", + "__nv_tan": "double(double)", + "__nv_tanf": "float(float)", + "__nv_tanh": "double(double)", + "__nv_tanhf": "float(float)", + "__nv_trunc": "double(double)", + "__nv_truncf": "float(float)" + } +} diff --git a/cuda-oxide/intrinsics/generated/libdevice_math.rs b/cuda-oxide/intrinsics/generated/libdevice_math.rs new file mode 100644 index 0000000000..aedf6eebed --- /dev/null +++ b/cuda-oxide/intrinsics/generated/libdevice_math.rs @@ -0,0 +1,742 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +// @generated by cuda-intrinsics-gen from overlay/libdevice_math.toml. DO NOT EDIT. +// Staged metadata; compiler dispatch is still handwritten. + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MathMapping { + pub libm: &'static str, + pub signature: &'static str, + pub placeholders: &'static [&'static str], + pub cmath: Option<&'static str>, + pub libdevice: Option<&'static str>, + pub llvm: Option<&'static str>, + pub lowering: Option<&'static str>, + pub link: &'static str, +} + +pub const MAPPINGS: &[MathMapping] = &[ + MathMapping { + libm: "acos", + signature: "(f64) -> f64", + placeholders: &["acosf64"], + cmath: Some("std::sys::cmath::acos"), + libdevice: Some("__nv_acos"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "acosf", + signature: "(f32) -> f32", + placeholders: &["acosf32"], + cmath: Some("std::sys::cmath::acosf"), + libdevice: Some("__nv_acosf"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "acosh", + signature: "(f64) -> f64", + placeholders: &["acoshf64"], + cmath: Some("std::sys::cmath::acosh"), + libdevice: Some("__nv_acosh"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "acoshf", + signature: "(f32) -> f32", + placeholders: &["acoshf32"], + cmath: Some("std::sys::cmath::acoshf"), + libdevice: Some("__nv_acoshf"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "asin", + signature: "(f64) -> f64", + placeholders: &["asinf64"], + cmath: Some("std::sys::cmath::asin"), + libdevice: Some("__nv_asin"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "asinf", + signature: "(f32) -> f32", + placeholders: &["asinf32"], + cmath: Some("std::sys::cmath::asinf"), + libdevice: Some("__nv_asinf"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "asinh", + signature: "(f64) -> f64", + placeholders: &["asinhf64"], + cmath: Some("std::sys::cmath::asinh"), + libdevice: Some("__nv_asinh"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "asinhf", + signature: "(f32) -> f32", + placeholders: &["asinhf32"], + cmath: Some("std::sys::cmath::asinhf"), + libdevice: Some("__nv_asinhf"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "atan", + signature: "(f64) -> f64", + placeholders: &["atanf64"], + cmath: Some("std::sys::cmath::atan"), + libdevice: Some("__nv_atan"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "atan2", + signature: "(f64, f64) -> f64", + placeholders: &["atan2f64"], + cmath: Some("std::sys::cmath::atan2"), + libdevice: Some("__nv_atan2"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "atan2f", + signature: "(f32, f32) -> f32", + placeholders: &["atan2f32"], + cmath: Some("std::sys::cmath::atan2f"), + libdevice: Some("__nv_atan2f"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "atanf", + signature: "(f32) -> f32", + placeholders: &["atanf32"], + cmath: Some("std::sys::cmath::atanf"), + libdevice: Some("__nv_atanf"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "atanh", + signature: "(f64) -> f64", + placeholders: &["atanhf64"], + cmath: Some("std::sys::cmath::atanh"), + libdevice: Some("__nv_atanh"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "atanhf", + signature: "(f32) -> f32", + placeholders: &["atanhf32"], + cmath: Some("std::sys::cmath::atanhf"), + libdevice: Some("__nv_atanhf"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "cbrt", + signature: "(f64) -> f64", + placeholders: &["cbrtf64"], + cmath: Some("std::sys::cmath::cbrt"), + libdevice: Some("__nv_cbrt"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "cbrtf", + signature: "(f32) -> f32", + placeholders: &["cbrtf32"], + cmath: Some("std::sys::cmath::cbrtf"), + libdevice: Some("__nv_cbrtf"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "ceil", + signature: "(f64) -> f64", + placeholders: &["ceilf64"], + cmath: None, + libdevice: Some("__nv_ceil"), + llvm: Some("llvm_ceil_f64"), + lowering: None, + link: "libnvvm", + }, + MathMapping { + libm: "ceilf", + signature: "(f32) -> f32", + placeholders: &["ceilf32"], + cmath: None, + libdevice: Some("__nv_ceilf"), + llvm: Some("llvm_ceil_f32"), + lowering: None, + link: "libnvvm", + }, + MathMapping { + libm: "copysign", + signature: "(f64, f64) -> f64", + placeholders: &["copysignf64"], + cmath: None, + libdevice: Some("__nv_copysign"), + llvm: Some("llvm_copysign_f64"), + lowering: None, + link: "libnvvm", + }, + MathMapping { + libm: "copysignf", + signature: "(f32, f32) -> f32", + placeholders: &["copysignf32"], + cmath: None, + libdevice: Some("__nv_copysignf"), + llvm: Some("llvm_copysign_f32"), + lowering: None, + link: "libnvvm", + }, + MathMapping { + libm: "cos", + signature: "(f64) -> f64", + placeholders: &["cosf64"], + cmath: Some("std::sys::cmath::cos"), + libdevice: Some("__nv_cos"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "cosf", + signature: "(f32) -> f32", + placeholders: &["cosf32"], + cmath: Some("std::sys::cmath::cosf"), + libdevice: Some("__nv_cosf"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "cosh", + signature: "(f64) -> f64", + placeholders: &["coshf64"], + cmath: Some("std::sys::cmath::cosh"), + libdevice: Some("__nv_cosh"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "coshf", + signature: "(f32) -> f32", + placeholders: &["coshf32"], + cmath: Some("std::sys::cmath::coshf"), + libdevice: Some("__nv_coshf"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "exp", + signature: "(f64) -> f64", + placeholders: &["expf64"], + cmath: None, + libdevice: Some("__nv_exp"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "exp2", + signature: "(f64) -> f64", + placeholders: &["exp2f64"], + cmath: None, + libdevice: Some("__nv_exp2"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "exp2f", + signature: "(f32) -> f32", + placeholders: &["exp2f32"], + cmath: None, + libdevice: Some("__nv_exp2f"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "expf", + signature: "(f32) -> f32", + placeholders: &["expf32"], + cmath: None, + libdevice: Some("__nv_expf"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "expm1", + signature: "(f64) -> f64", + placeholders: &["expm1f64"], + cmath: Some("std::sys::cmath::expm1"), + libdevice: Some("__nv_expm1"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "expm1f", + signature: "(f32) -> f32", + placeholders: &["expm1f32"], + cmath: Some("std::sys::cmath::expm1f"), + libdevice: Some("__nv_expm1f"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "fabs", + signature: "(f64) -> f64", + placeholders: &["fabs"], + cmath: None, + libdevice: Some("__nv_fabs"), + llvm: Some("llvm_fabs_f64"), + lowering: None, + link: "libnvvm", + }, + MathMapping { + libm: "fabsf", + signature: "(f32) -> f32", + placeholders: &["fabs"], + cmath: None, + libdevice: Some("__nv_fabsf"), + llvm: Some("llvm_fabs_f32"), + lowering: None, + link: "libnvvm", + }, + MathMapping { + libm: "floor", + signature: "(f64) -> f64", + placeholders: &["floorf64"], + cmath: None, + libdevice: Some("__nv_floor"), + llvm: Some("llvm_floor_f64"), + lowering: None, + link: "libnvvm", + }, + MathMapping { + libm: "floorf", + signature: "(f32) -> f32", + placeholders: &["floorf32"], + cmath: None, + libdevice: Some("__nv_floorf"), + llvm: Some("llvm_floor_f32"), + lowering: None, + link: "libnvvm", + }, + MathMapping { + libm: "fma", + signature: "(f64, f64, f64) -> f64", + placeholders: &["fmaf64"], + cmath: None, + libdevice: Some("__nv_fma"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "fmaf", + signature: "(f32, f32, f32) -> f32", + placeholders: &["fmaf32"], + cmath: None, + libdevice: Some("__nv_fmaf"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "fmax", + signature: "(f64, f64) -> f64", + placeholders: &["maximum_number_nsz_f64"], + cmath: None, + libdevice: None, + llvm: None, + lowering: Some("handwritten:lower_float_minmax_nsz"), + link: "never", + }, + MathMapping { + libm: "fmaxf", + signature: "(f32, f32) -> f32", + placeholders: &["maximum_number_nsz_f32"], + cmath: None, + libdevice: None, + llvm: None, + lowering: Some("handwritten:lower_float_minmax_nsz"), + link: "never", + }, + MathMapping { + libm: "fmin", + signature: "(f64, f64) -> f64", + placeholders: &["minimum_number_nsz_f64"], + cmath: None, + libdevice: None, + llvm: None, + lowering: Some("handwritten:lower_float_minmax_nsz"), + link: "never", + }, + MathMapping { + libm: "fminf", + signature: "(f32, f32) -> f32", + placeholders: &["minimum_number_nsz_f32"], + cmath: None, + libdevice: None, + llvm: None, + lowering: Some("handwritten:lower_float_minmax_nsz"), + link: "never", + }, + MathMapping { + libm: "hypot", + signature: "(f64, f64) -> f64", + placeholders: &["hypotf64"], + cmath: Some("std::sys::cmath::hypot"), + libdevice: Some("__nv_hypot"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "hypotf", + signature: "(f32, f32) -> f32", + placeholders: &["hypotf32"], + cmath: Some("std::sys::cmath::hypotf"), + libdevice: Some("__nv_hypotf"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "log", + signature: "(f64) -> f64", + placeholders: &["logf64"], + cmath: None, + libdevice: Some("__nv_log"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "log10", + signature: "(f64) -> f64", + placeholders: &["log10f64"], + cmath: None, + libdevice: Some("__nv_log10"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "log10f", + signature: "(f32) -> f32", + placeholders: &["log10f32"], + cmath: None, + libdevice: Some("__nv_log10f"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "log1p", + signature: "(f64) -> f64", + placeholders: &["log1pf64"], + cmath: Some("std::sys::cmath::log1p"), + libdevice: Some("__nv_log1p"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "log1pf", + signature: "(f32) -> f32", + placeholders: &["log1pf32"], + cmath: Some("std::sys::cmath::log1pf"), + libdevice: Some("__nv_log1pf"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "log2", + signature: "(f64) -> f64", + placeholders: &["log2f64"], + cmath: None, + libdevice: Some("__nv_log2"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "log2f", + signature: "(f32) -> f32", + placeholders: &["log2f32"], + cmath: None, + libdevice: Some("__nv_log2f"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "logf", + signature: "(f32) -> f32", + placeholders: &["logf32"], + cmath: None, + libdevice: Some("__nv_logf"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "pow", + signature: "(f64, f64) -> f64", + placeholders: &["powf64"], + cmath: None, + libdevice: Some("__nv_pow"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "powf", + signature: "(f32, f32) -> f32", + placeholders: &["powf32"], + cmath: None, + libdevice: Some("__nv_powf"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "rint", + signature: "(f64) -> f64", + placeholders: &["round_ties_even_f64"], + cmath: None, + libdevice: Some("__nv_rint"), + llvm: Some("llvm_roundeven_f64"), + lowering: None, + link: "libnvvm", + }, + MathMapping { + libm: "rintf", + signature: "(f32) -> f32", + placeholders: &["round_ties_even_f32"], + cmath: None, + libdevice: Some("__nv_rintf"), + llvm: Some("llvm_roundeven_f32"), + lowering: None, + link: "libnvvm", + }, + MathMapping { + libm: "round", + signature: "(f64) -> f64", + placeholders: &["roundf64"], + cmath: None, + libdevice: Some("__nv_round"), + llvm: Some("llvm_round_f64"), + lowering: None, + link: "libnvvm", + }, + MathMapping { + libm: "roundeven", + signature: "(f64) -> f64", + placeholders: &["round_ties_even_f64"], + cmath: None, + libdevice: Some("__nv_rint"), + llvm: Some("llvm_roundeven_f64"), + lowering: None, + link: "libnvvm", + }, + MathMapping { + libm: "roundevenf", + signature: "(f32) -> f32", + placeholders: &["round_ties_even_f32"], + cmath: None, + libdevice: Some("__nv_rintf"), + llvm: Some("llvm_roundeven_f32"), + lowering: None, + link: "libnvvm", + }, + MathMapping { + libm: "roundf", + signature: "(f32) -> f32", + placeholders: &["roundf32"], + cmath: None, + libdevice: Some("__nv_roundf"), + llvm: Some("llvm_round_f32"), + lowering: None, + link: "libnvvm", + }, + MathMapping { + libm: "sin", + signature: "(f64) -> f64", + placeholders: &["sinf64"], + cmath: Some("std::sys::cmath::sin"), + libdevice: Some("__nv_sin"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "sincos", + signature: "(f64) -> (f64, f64)", + placeholders: &["sinf64", "cosf64"], + cmath: None, + libdevice: None, + llvm: None, + lowering: Some("handwritten:emit_sincos"), + link: "always", + }, + MathMapping { + libm: "sincosf", + signature: "(f32) -> (f32, f32)", + placeholders: &["sinf32", "cosf32"], + cmath: None, + libdevice: None, + llvm: None, + lowering: Some("handwritten:emit_sincos"), + link: "always", + }, + MathMapping { + libm: "sinf", + signature: "(f32) -> f32", + placeholders: &["sinf32"], + cmath: Some("std::sys::cmath::sinf"), + libdevice: Some("__nv_sinf"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "sinh", + signature: "(f64) -> f64", + placeholders: &["sinhf64"], + cmath: Some("std::sys::cmath::sinh"), + libdevice: Some("__nv_sinh"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "sinhf", + signature: "(f32) -> f32", + placeholders: &["sinhf32"], + cmath: Some("std::sys::cmath::sinhf"), + libdevice: Some("__nv_sinhf"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "sqrt", + signature: "(f64) -> f64", + placeholders: &["sqrtf64"], + cmath: None, + libdevice: Some("__nv_sqrt"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "sqrtf", + signature: "(f32) -> f32", + placeholders: &["sqrtf32"], + cmath: None, + libdevice: Some("__nv_sqrtf"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "tan", + signature: "(f64) -> f64", + placeholders: &["tanf64"], + cmath: Some("std::sys::cmath::tan"), + libdevice: Some("__nv_tan"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "tanf", + signature: "(f32) -> f32", + placeholders: &["tanf32"], + cmath: Some("std::sys::cmath::tanf"), + libdevice: Some("__nv_tanf"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "tanh", + signature: "(f64) -> f64", + placeholders: &["tanhf64"], + cmath: Some("std::sys::cmath::tanh"), + libdevice: Some("__nv_tanh"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "tanhf", + signature: "(f32) -> f32", + placeholders: &["tanhf32"], + cmath: Some("std::sys::cmath::tanhf"), + libdevice: Some("__nv_tanhf"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: "trunc", + signature: "(f64) -> f64", + placeholders: &["truncf64"], + cmath: None, + libdevice: Some("__nv_trunc"), + llvm: Some("llvm_trunc_f64"), + lowering: None, + link: "libnvvm", + }, + MathMapping { + libm: "truncf", + signature: "(f32) -> f32", + placeholders: &["truncf32"], + cmath: None, + libdevice: Some("__nv_truncf"), + llvm: Some("llvm_trunc_f32"), + lowering: None, + link: "libnvvm", + }, +]; diff --git a/cuda-oxide/intrinsics/libdevice-math.md b/cuda-oxide/intrinsics/libdevice-math.md new file mode 100644 index 0000000000..c6ddf489c9 --- /dev/null +++ b/cuda-oxide/intrinsics/libdevice-math.md @@ -0,0 +1,87 @@ + + + + +# Staged libm mappings + +72 mapped f32/f64 libm exports. Compiler dispatch remains handwritten. + +Libdevice bitcode SHA-256: `91334d6e12748f6cb5bbf0a1cd965a56bcd93dc4f496d2e5c5f8c6e523094356`. + +`libnvvm` links only on that backend; `always` links on both; `never` uses +a handwritten expansion. Aliases are separate exports sharing a mapping. + +| libm export | Rust signature | libdevice symbol | LLVM NVPTX alternative | Handwritten lowering | Linking | +| --- | --- | --- | --- | --- | --- | +| `acos` | `(f64) -> f64` | `__nv_acos` | | | `always` | +| `acosf` | `(f32) -> f32` | `__nv_acosf` | | | `always` | +| `acosh` | `(f64) -> f64` | `__nv_acosh` | | | `always` | +| `acoshf` | `(f32) -> f32` | `__nv_acoshf` | | | `always` | +| `asin` | `(f64) -> f64` | `__nv_asin` | | | `always` | +| `asinf` | `(f32) -> f32` | `__nv_asinf` | | | `always` | +| `asinh` | `(f64) -> f64` | `__nv_asinh` | | | `always` | +| `asinhf` | `(f32) -> f32` | `__nv_asinhf` | | | `always` | +| `atan` | `(f64) -> f64` | `__nv_atan` | | | `always` | +| `atan2` | `(f64, f64) -> f64` | `__nv_atan2` | | | `always` | +| `atan2f` | `(f32, f32) -> f32` | `__nv_atan2f` | | | `always` | +| `atanf` | `(f32) -> f32` | `__nv_atanf` | | | `always` | +| `atanh` | `(f64) -> f64` | `__nv_atanh` | | | `always` | +| `atanhf` | `(f32) -> f32` | `__nv_atanhf` | | | `always` | +| `cbrt` | `(f64) -> f64` | `__nv_cbrt` | | | `always` | +| `cbrtf` | `(f32) -> f32` | `__nv_cbrtf` | | | `always` | +| `ceil` | `(f64) -> f64` | `__nv_ceil` | `llvm_ceil_f64` | | `libnvvm` | +| `ceilf` | `(f32) -> f32` | `__nv_ceilf` | `llvm_ceil_f32` | | `libnvvm` | +| `copysign` | `(f64, f64) -> f64` | `__nv_copysign` | `llvm_copysign_f64` | | `libnvvm` | +| `copysignf` | `(f32, f32) -> f32` | `__nv_copysignf` | `llvm_copysign_f32` | | `libnvvm` | +| `cos` | `(f64) -> f64` | `__nv_cos` | | | `always` | +| `cosf` | `(f32) -> f32` | `__nv_cosf` | | | `always` | +| `cosh` | `(f64) -> f64` | `__nv_cosh` | | | `always` | +| `coshf` | `(f32) -> f32` | `__nv_coshf` | | | `always` | +| `exp` | `(f64) -> f64` | `__nv_exp` | | | `always` | +| `exp2` | `(f64) -> f64` | `__nv_exp2` | | | `always` | +| `exp2f` | `(f32) -> f32` | `__nv_exp2f` | | | `always` | +| `expf` | `(f32) -> f32` | `__nv_expf` | | | `always` | +| `expm1` | `(f64) -> f64` | `__nv_expm1` | | | `always` | +| `expm1f` | `(f32) -> f32` | `__nv_expm1f` | | | `always` | +| `fabs` | `(f64) -> f64` | `__nv_fabs` | `llvm_fabs_f64` | | `libnvvm` | +| `fabsf` | `(f32) -> f32` | `__nv_fabsf` | `llvm_fabs_f32` | | `libnvvm` | +| `floor` | `(f64) -> f64` | `__nv_floor` | `llvm_floor_f64` | | `libnvvm` | +| `floorf` | `(f32) -> f32` | `__nv_floorf` | `llvm_floor_f32` | | `libnvvm` | +| `fma` | `(f64, f64, f64) -> f64` | `__nv_fma` | | | `always` | +| `fmaf` | `(f32, f32, f32) -> f32` | `__nv_fmaf` | | | `always` | +| `fmax` | `(f64, f64) -> f64` | | | `handwritten:lower_float_minmax_nsz` | `never` | +| `fmaxf` | `(f32, f32) -> f32` | | | `handwritten:lower_float_minmax_nsz` | `never` | +| `fmin` | `(f64, f64) -> f64` | | | `handwritten:lower_float_minmax_nsz` | `never` | +| `fminf` | `(f32, f32) -> f32` | | | `handwritten:lower_float_minmax_nsz` | `never` | +| `hypot` | `(f64, f64) -> f64` | `__nv_hypot` | | | `always` | +| `hypotf` | `(f32, f32) -> f32` | `__nv_hypotf` | | | `always` | +| `log` | `(f64) -> f64` | `__nv_log` | | | `always` | +| `log10` | `(f64) -> f64` | `__nv_log10` | | | `always` | +| `log10f` | `(f32) -> f32` | `__nv_log10f` | | | `always` | +| `log1p` | `(f64) -> f64` | `__nv_log1p` | | | `always` | +| `log1pf` | `(f32) -> f32` | `__nv_log1pf` | | | `always` | +| `log2` | `(f64) -> f64` | `__nv_log2` | | | `always` | +| `log2f` | `(f32) -> f32` | `__nv_log2f` | | | `always` | +| `logf` | `(f32) -> f32` | `__nv_logf` | | | `always` | +| `pow` | `(f64, f64) -> f64` | `__nv_pow` | | | `always` | +| `powf` | `(f32, f32) -> f32` | `__nv_powf` | | | `always` | +| `rint` | `(f64) -> f64` | `__nv_rint` | `llvm_roundeven_f64` | | `libnvvm` | +| `rintf` | `(f32) -> f32` | `__nv_rintf` | `llvm_roundeven_f32` | | `libnvvm` | +| `round` | `(f64) -> f64` | `__nv_round` | `llvm_round_f64` | | `libnvvm` | +| `roundeven` | `(f64) -> f64` | `__nv_rint` | `llvm_roundeven_f64` | | `libnvvm` | +| `roundevenf` | `(f32) -> f32` | `__nv_rintf` | `llvm_roundeven_f32` | | `libnvvm` | +| `roundf` | `(f32) -> f32` | `__nv_roundf` | `llvm_round_f32` | | `libnvvm` | +| `sin` | `(f64) -> f64` | `__nv_sin` | | | `always` | +| `sincos` | `(f64) -> (f64, f64)` | | | `handwritten:emit_sincos` | `always` | +| `sincosf` | `(f32) -> (f32, f32)` | | | `handwritten:emit_sincos` | `always` | +| `sinf` | `(f32) -> f32` | `__nv_sinf` | | | `always` | +| `sinh` | `(f64) -> f64` | `__nv_sinh` | | | `always` | +| `sinhf` | `(f32) -> f32` | `__nv_sinhf` | | | `always` | +| `sqrt` | `(f64) -> f64` | `__nv_sqrt` | | | `always` | +| `sqrtf` | `(f32) -> f32` | `__nv_sqrtf` | | | `always` | +| `tan` | `(f64) -> f64` | `__nv_tan` | | | `always` | +| `tanf` | `(f32) -> f32` | `__nv_tanf` | | | `always` | +| `tanh` | `(f64) -> f64` | `__nv_tanh` | | | `always` | +| `tanhf` | `(f32) -> f32` | `__nv_tanhf` | | | `always` | +| `trunc` | `(f64) -> f64` | `__nv_trunc` | `llvm_trunc_f64` | | `libnvvm` | +| `truncf` | `(f32) -> f32` | `__nv_truncf` | `llvm_trunc_f32` | | `libnvvm` | diff --git a/cuda-oxide/intrinsics/overlay/libdevice_math.toml b/cuda-oxide/intrinsics/overlay/libdevice_math.toml new file mode 100644 index 0000000000..ceb976e8d4 --- /dev/null +++ b/cuda-oxide/intrinsics/overlay/libdevice_math.toml @@ -0,0 +1,194 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Separate libdevice evidence domain; not part of the hardware ABI shard index. +# PR3 stages the existing 72 libm exports. Compiler tables are migrated later. +schema = 1 +family = "libdevice_math" + +[libdevice_math] +evidence = "evidence/libdevice_math/cuda-13.0.json" +libdevice_sha256 = "91334d6e12748f6cb5bbf0a1cd965a56bcd93dc4f496d2e5c5f8c6e523094356" +widths = ["f32", "f64"] + +[[libdevice_math.function]] +name = "acos" +signature = "(F) -> F" +cmath = "acos{suffix}" + +[[libdevice_math.function]] +name = "acosh" +signature = "(F) -> F" +cmath = "acosh{suffix}" + +[[libdevice_math.function]] +name = "asin" +signature = "(F) -> F" +cmath = "asin{suffix}" + +[[libdevice_math.function]] +name = "asinh" +signature = "(F) -> F" +cmath = "asinh{suffix}" + +[[libdevice_math.function]] +name = "atan" +signature = "(F) -> F" +cmath = "atan{suffix}" + +[[libdevice_math.function]] +name = "atan2" +signature = "(F, F) -> F" +cmath = "atan2{suffix}" + +[[libdevice_math.function]] +name = "atanh" +signature = "(F) -> F" +cmath = "atanh{suffix}" + +[[libdevice_math.function]] +name = "cbrt" +signature = "(F) -> F" +cmath = "cbrt{suffix}" + +[[libdevice_math.function]] +name = "ceil" +signature = "(F) -> F" +llvm = "llvm_ceil_{width}" +link = "libnvvm" + +[[libdevice_math.function]] +name = "copysign" +signature = "(F, F) -> F" +llvm = "llvm_copysign_{width}" +link = "libnvvm" + +[[libdevice_math.function]] +name = "cos" +signature = "(F) -> F" +cmath = "cos{suffix}" + +[[libdevice_math.function]] +name = "cosh" +signature = "(F) -> F" +cmath = "cosh{suffix}" + +[[libdevice_math.function]] +name = "exp" +signature = "(F) -> F" + +[[libdevice_math.function]] +name = "exp2" +signature = "(F) -> F" + +[[libdevice_math.function]] +name = "expm1" +signature = "(F) -> F" +cmath = "expm1{suffix}" + +[[libdevice_math.function]] +name = "fabs" +signature = "(F) -> F" +placeholders = ["fabs"] +llvm = "llvm_fabs_{width}" +link = "libnvvm" + +[[libdevice_math.function]] +name = "floor" +signature = "(F) -> F" +llvm = "llvm_floor_{width}" +link = "libnvvm" + +[[libdevice_math.function]] +name = "fma" +signature = "(F, F, F) -> F" + +[[libdevice_math.function]] +name = "fmax" +signature = "(F, F) -> F" +placeholders = ["maximum_number_nsz_{width}"] +lowering = "handwritten:lower_float_minmax_nsz" +link = "never" + +[[libdevice_math.function]] +name = "fmin" +signature = "(F, F) -> F" +placeholders = ["minimum_number_nsz_{width}"] +lowering = "handwritten:lower_float_minmax_nsz" +link = "never" + +[[libdevice_math.function]] +name = "hypot" +signature = "(F, F) -> F" +cmath = "hypot{suffix}" + +[[libdevice_math.function]] +name = "log" +signature = "(F) -> F" + +[[libdevice_math.function]] +name = "log10" +signature = "(F) -> F" + +[[libdevice_math.function]] +name = "log1p" +signature = "(F) -> F" +cmath = "log1p{suffix}" + +[[libdevice_math.function]] +name = "log2" +signature = "(F) -> F" + +[[libdevice_math.function]] +name = "pow" +signature = "(F, F) -> F" + +[[libdevice_math.function]] +name = "rint" +signature = "(F) -> F" +libm = ["rint", "roundeven"] +placeholders = ["round_ties_even_{width}"] +llvm = "llvm_roundeven_{width}" +link = "libnvvm" + +[[libdevice_math.function]] +name = "round" +signature = "(F) -> F" +llvm = "llvm_round_{width}" +link = "libnvvm" + +[[libdevice_math.function]] +name = "sin" +signature = "(F) -> F" +cmath = "sin{suffix}" + +[[libdevice_math.function]] +name = "sincos" +signature = "(F) -> (F, F)" +placeholders = ["sin{width}", "cos{width}"] +lowering = "handwritten:emit_sincos" + +[[libdevice_math.function]] +name = "sinh" +signature = "(F) -> F" +cmath = "sinh{suffix}" + +[[libdevice_math.function]] +name = "sqrt" +signature = "(F) -> F" + +[[libdevice_math.function]] +name = "tan" +signature = "(F) -> F" +cmath = "tan{suffix}" + +[[libdevice_math.function]] +name = "tanh" +signature = "(F) -> F" +cmath = "tanh{suffix}" + +[[libdevice_math.function]] +name = "trunc" +signature = "(F) -> F" +llvm = "llvm_trunc_{width}" +link = "libnvvm" From 5068a87304970c9dd53314b8a8a8c73a3f09198f Mon Sep 17 00:00:00 2001 From: Raul Estrada Date: Sun, 25 Oct 2026 01:39:05 -0600 Subject: [PATCH 2/4] feat: lower tgamma, ilogb and ldexp through libdevice Signed-off-by: Raul Estrada --- .../crates/dialect-mir/src/rust_intrinsics.rs | 25 + .../terminator/intrinsics/float_math.rs | 77 +- .../crates/mir-lower/src/convert/ops/call.rs | 129 +++ .../tests/lowering_test/math_conversions.rs | 261 ++++++ .../examples/math_special/Cargo.lock | 761 ++++++++++++++++++ .../examples/math_special/Cargo.toml | 16 + .../examples/math_special/README.md | 70 ++ .../examples/math_special/src/main.rs | 233 ++++++ .../rustc-codegen-cuda/src/collector.rs | 28 +- 9 files changed, 1596 insertions(+), 4 deletions(-) create mode 100644 cuda-oxide/crates/rustc-codegen-cuda/examples/math_special/Cargo.lock create mode 100644 cuda-oxide/crates/rustc-codegen-cuda/examples/math_special/Cargo.toml create mode 100644 cuda-oxide/crates/rustc-codegen-cuda/examples/math_special/README.md create mode 100644 cuda-oxide/crates/rustc-codegen-cuda/examples/math_special/src/main.rs diff --git a/cuda-oxide/crates/dialect-mir/src/rust_intrinsics.rs b/cuda-oxide/crates/dialect-mir/src/rust_intrinsics.rs index 43d7241f52..4d4b5387ea 100644 --- a/cuda-oxide/crates/dialect-mir/src/rust_intrinsics.rs +++ b/cuda-oxide/crates/dialect-mir/src/rust_intrinsics.rs @@ -211,6 +211,19 @@ pub const CALLEE_HYPOT_F32: &str = placeholder!("hypotf32"); /// Placeholder call used for `f64::hypot` / `std::sys::cmath::hypot` (binary). pub const CALLEE_HYPOT_F64: &str = placeholder!("hypotf64"); +/// Placeholder call used for `libm::tgammaf`. +pub const CALLEE_TGAMMA_F32: &str = placeholder!("tgammaf32"); +/// Placeholder call used for `libm::tgamma`. +pub const CALLEE_TGAMMA_F64: &str = placeholder!("tgammaf64"); +/// Placeholder call used for `libm::ilogbf`. +pub const CALLEE_ILOGB_F32: &str = placeholder!("ilogbf32"); +/// Placeholder call used for `libm::ilogb`. +pub const CALLEE_ILOGB_F64: &str = placeholder!("ilogbf64"); +/// Placeholder call used for `libm::ldexpf`. +pub const CALLEE_LDEXP_F32: &str = placeholder!("ldexpf32"); +/// Placeholder call used for `libm::ldexp`. +pub const CALLEE_LDEXP_F64: &str = placeholder!("ldexpf64"); + /// Placeholder call used for `core::intrinsics::fadd_fast` (generic over float type). /// /// Lowered to `llvm.fadd` with explicit `fast` fast-math flags. The `f*_fast` intrinsics @@ -350,6 +363,12 @@ pub fn is_libdevice_backed_placeholder(callee: &str) -> bool { | CALLEE_LOG1P_F64 | CALLEE_HYPOT_F32 | CALLEE_HYPOT_F64 + | CALLEE_TGAMMA_F32 + | CALLEE_TGAMMA_F64 + | CALLEE_ILOGB_F32 + | CALLEE_ILOGB_F64 + | CALLEE_LDEXP_F32 + | CALLEE_LDEXP_F64 ) } @@ -452,6 +471,12 @@ mod tests { CALLEE_LOG1P_F64, CALLEE_HYPOT_F32, CALLEE_HYPOT_F64, + CALLEE_TGAMMA_F32, + CALLEE_TGAMMA_F64, + CALLEE_ILOGB_F32, + CALLEE_ILOGB_F64, + CALLEE_LDEXP_F32, + CALLEE_LDEXP_F64, ]; /// Every placeholder that is libdevice-backed only under the libNVVM diff --git a/cuda-oxide/crates/mir-importer/src/translator/terminator/intrinsics/float_math.rs b/cuda-oxide/crates/mir-importer/src/translator/terminator/intrinsics/float_math.rs index 77140b8dcb..3eb562bed4 100644 --- a/cuda-oxide/crates/mir-importer/src/translator/terminator/intrinsics/float_math.rs +++ b/cuda-oxide/crates/mir-importer/src/translator/terminator/intrinsics/float_math.rs @@ -16,7 +16,7 @@ use pliron::location::Location; use pliron::operation::Operation; use rustc_public::mir; -/// Floating-point math intrinsic from libcore. +/// Floating-point math intrinsic from core, std, or libm. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum RustFloatMathIntrinsic { /// `core::intrinsics::sqrtf32`. @@ -161,6 +161,18 @@ pub enum RustFloatMathIntrinsic { HypotF32, /// `f64::hypot` / `std::sys::cmath::hypot` (binary). HypotF64, + /// `libm::tgammaf`. + TgammaF32, + /// `libm::tgamma`. + TgammaF64, + /// `libm::ilogbf`. + IlogbF32, + /// `libm::ilogb`. + IlogbF64, + /// `libm::ldexpf`. + LdexpF32, + /// `libm::ldexp`. + LdexpF64, /// Generic `core::intrinsics::fadd_fast` (lowered to `llvm.fadd` + fast-math). FaddFast, /// Generic `core::intrinsics::fsub_fast` (lowered to `llvm.fsub` + fast-math). @@ -341,6 +353,8 @@ impl RustFloatMathIntrinsic { "std::sys::cmath::log1p" => Some(Self::Log1pF64), "std::sys::cmath::hypotf" => Some(Self::HypotF32), "std::sys::cmath::hypot" => Some(Self::HypotF64), + "std::sys::cmath::tgammaf" => Some(Self::TgammaF32), + "std::sys::cmath::tgamma" => Some(Self::TgammaF64), other => Self::from_libm_path(other).or_else(|| Self::from_fast_intrinsic_path(other)), } } @@ -428,6 +442,12 @@ impl RustFloatMathIntrinsic { "log1p" => Some(Self::Log1pF64), "hypotf" => Some(Self::HypotF32), "hypot" => Some(Self::HypotF64), + "tgammaf" => Some(Self::TgammaF32), + "tgamma" => Some(Self::TgammaF64), + "ilogbf" => Some(Self::IlogbF32), + "ilogb" => Some(Self::IlogbF64), + "ldexpf" | "scalbnf" => Some(Self::LdexpF32), + "ldexp" | "scalbn" => Some(Self::LdexpF64), _ => None, } } @@ -523,6 +543,12 @@ impl RustFloatMathIntrinsic { Self::Log1pF64 => rust_intrinsics::CALLEE_LOG1P_F64, Self::HypotF32 => rust_intrinsics::CALLEE_HYPOT_F32, Self::HypotF64 => rust_intrinsics::CALLEE_HYPOT_F64, + Self::TgammaF32 => rust_intrinsics::CALLEE_TGAMMA_F32, + Self::TgammaF64 => rust_intrinsics::CALLEE_TGAMMA_F64, + Self::IlogbF32 => rust_intrinsics::CALLEE_ILOGB_F32, + Self::IlogbF64 => rust_intrinsics::CALLEE_ILOGB_F64, + Self::LdexpF32 => rust_intrinsics::CALLEE_LDEXP_F32, + Self::LdexpF64 => rust_intrinsics::CALLEE_LDEXP_F64, Self::FaddFast => rust_intrinsics::CALLEE_FADD_FAST, Self::FsubFast => rust_intrinsics::CALLEE_FSUB_FAST, Self::FmulFast => rust_intrinsics::CALLEE_FMUL_FAST, @@ -736,6 +762,55 @@ mod tests { use super::*; use dialect_mir::rust_intrinsics; + #[test] + fn special_math_paths_resolve_to_typed_placeholders() { + use RustFloatMathIntrinsic as I; + for (name, intrinsic, callee) in [ + ("tgammaf", I::TgammaF32, rust_intrinsics::CALLEE_TGAMMA_F32), + ("tgamma", I::TgammaF64, rust_intrinsics::CALLEE_TGAMMA_F64), + ("ilogbf", I::IlogbF32, rust_intrinsics::CALLEE_ILOGB_F32), + ("ilogb", I::IlogbF64, rust_intrinsics::CALLEE_ILOGB_F64), + ("ldexpf", I::LdexpF32, rust_intrinsics::CALLEE_LDEXP_F32), + ("ldexp", I::LdexpF64, rust_intrinsics::CALLEE_LDEXP_F64), + ("scalbnf", I::LdexpF32, rust_intrinsics::CALLEE_LDEXP_F32), + ("scalbn", I::LdexpF64, rust_intrinsics::CALLEE_LDEXP_F64), + ] { + for path in [ + format!("libm::{name}"), + format!("libm::math::{name}::{name}"), + ] { + assert_eq!(I::from_core_path(&path), Some(intrinsic), "{path}"); + assert_eq!(intrinsic.placeholder_callee(), callee); + } + for path in [ + format!("my_app::libm_compat::{name}"), + format!("my_app::libm::{name}"), + format!("libm_lookalike::{name}"), + format!("std::sys::cmath::{name}_unsupported"), + ] { + assert_eq!(I::from_core_path(&path), None, "{path}"); + } + } + assert_eq!( + I::from_core_path("std::sys::cmath::tgammaf"), + Some(I::TgammaF32) + ); + assert_eq!( + I::from_core_path("std::sys::cmath::tgamma"), + Some(I::TgammaF64) + ); + // Existing scalar and tuple dispatch must keep their identities. + assert_eq!(I::from_core_path("libm::hypot"), Some(I::HypotF64)); + assert_eq!( + I::from_core_path("core::intrinsics::powif32"), + Some(I::PowiF32) + ); + assert_eq!(libm_sincos_is_f64("libm::sincosf"), Some(false)); + for name in ["frexp", "modf", "remquo", "lgamma_r", "tgammaf128"] { + assert_eq!(I::from_core_path(&format!("libm::{name}")), None); + } + } + #[test] fn generic_float_intrinsics_resolve_only_the_declared_float_type() { use RustFloatMathIntrinsic as I; diff --git a/cuda-oxide/crates/mir-lower/src/convert/ops/call.rs b/cuda-oxide/crates/mir-lower/src/convert/ops/call.rs index bc19168b0c..b18e43b12c 100644 --- a/cuda-oxide/crates/mir-lower/src/convert/ops/call.rs +++ b/cuda-oxide/crates/mir-lower/src/convert/ops/call.rs @@ -262,6 +262,12 @@ enum RustFloatMathIntrinsic { Log1pF64, HypotF32, HypotF64, + TgammaF32, + TgammaF64, + IlogbF32, + IlogbF64, + LdexpF32, + LdexpF64, FaddFast, FsubFast, FmulFast, @@ -344,6 +350,12 @@ impl RustFloatMathIntrinsic { rust_intrinsics::CALLEE_LOG1P_F64 => Some(Self::Log1pF64), rust_intrinsics::CALLEE_HYPOT_F32 => Some(Self::HypotF32), rust_intrinsics::CALLEE_HYPOT_F64 => Some(Self::HypotF64), + rust_intrinsics::CALLEE_TGAMMA_F32 => Some(Self::TgammaF32), + rust_intrinsics::CALLEE_TGAMMA_F64 => Some(Self::TgammaF64), + rust_intrinsics::CALLEE_ILOGB_F32 => Some(Self::IlogbF32), + rust_intrinsics::CALLEE_ILOGB_F64 => Some(Self::IlogbF64), + rust_intrinsics::CALLEE_LDEXP_F32 => Some(Self::LdexpF32), + rust_intrinsics::CALLEE_LDEXP_F64 => Some(Self::LdexpF64), rust_intrinsics::CALLEE_FADD_FAST => Some(Self::FaddFast), rust_intrinsics::CALLEE_FSUB_FAST => Some(Self::FsubFast), rust_intrinsics::CALLEE_FMUL_FAST => Some(Self::FmulFast), @@ -448,6 +460,12 @@ impl RustFloatMathIntrinsic { Self::Log1pF64 => Ok("__nv_log1p"), Self::HypotF32 => Ok("__nv_hypotf"), Self::HypotF64 => Ok("__nv_hypot"), + Self::TgammaF32 => Ok("__nv_tgammaf"), + Self::TgammaF64 => Ok("__nv_tgamma"), + Self::IlogbF32 => Ok("__nv_ilogbf"), + Self::IlogbF64 => Ok("__nv_ilogb"), + Self::LdexpF32 => Ok("__nv_ldexpf"), + Self::LdexpF64 => Ok("__nv_ldexp"), Self::FaddFast | Self::FsubFast | Self::FmulFast | Self::FdivFast | Self::FremFast => { // The `f*_fast` intrinsics lower directly to LLVM `fadd`/`fsub`/ // `fmul`/`fdiv`/`frem` with fast-math flags, not to a libdevice @@ -479,6 +497,8 @@ impl RustFloatMathIntrinsic { | Self::Atan2F64 | Self::HypotF32 | Self::HypotF64 + | Self::LdexpF32 + | Self::LdexpF64 | Self::FaddFast | Self::FsubFast | Self::FmulFast @@ -1300,6 +1320,35 @@ fn convert_rust_float_math_intrinsic( let result_mir_ty = op.deref(ctx).get_result(0).get_type(ctx); let result_ty = convert_type(ctx, result_mir_ty).map_err(anyhow_to_pliron)?; + // These libdevice signatures mix integer and float types. Validate the + // complete signature before declaring the callee: deriving it solely from + // a malformed placeholder would silently declare the wrong libdevice ABI. + let special_float_ty: Option = match intrinsic { + RustFloatMathIntrinsic::TgammaF32 + | RustFloatMathIntrinsic::IlogbF32 + | RustFloatMathIntrinsic::LdexpF32 => Some(FP32Type::get(ctx).into()), + RustFloatMathIntrinsic::TgammaF64 + | RustFloatMathIntrinsic::IlogbF64 + | RustFloatMathIntrinsic::LdexpF64 => Some(FP64Type::get(ctx).into()), + _ => None, + }; + if let Some(float_ty) = special_float_ty { + let i32_ty: TypeHandle = IntegerType::get(ctx, 32, Signedness::Signless).into(); + let expected_result = match intrinsic { + RustFloatMathIntrinsic::IlogbF32 | RustFloatMathIntrinsic::IlogbF64 => i32_ty, + _ => float_ty, + }; + if result_ty != expected_result + || args[0].get_type(ctx) != float_ty + || (matches!( + intrinsic, + RustFloatMathIntrinsic::LdexpF32 | RustFloatMathIntrinsic::LdexpF64 + ) && args[1].get_type(ctx) != i32_ty) + { + return pliron::input_err!(loc, "Rust special math intrinsic signature mismatch"); + } + } + // The rounding and sign intrinsics have direct LLVM equivalents that the // NVPTX backend selects without a libdevice call; everything else (and // every op under the libNVVM backend) stays on libdevice. See @@ -2005,6 +2054,86 @@ fn resolve_device_extern_symbol(callee_name: &str) -> String { mod tests { use super::*; + #[test] + fn special_math_symbols_and_arities_match_on_both_backends() { + use RustFloatMathIntrinsic as I; + for backend in [ + crate::IntrinsicBackend::LlvmNvptx, + crate::IntrinsicBackend::LibNvvm, + ] { + let mut ctx = Context::new(); + crate::context::set_lowering_options( + &mut ctx, + crate::LoweringOptions { + intrinsic_backend: backend, + ..Default::default() + }, + ); + let f32_ty = FP32Type::get(&ctx).into(); + let f64_ty = FP64Type::get(&ctx).into(); + let i32_ty = IntegerType::get(&ctx, 32, Signedness::Signless).into(); + for (callee, intrinsic, result_ty, symbol, arity) in [ + ( + rust_intrinsics::CALLEE_TGAMMA_F32, + I::TgammaF32, + f32_ty, + "__nv_tgammaf", + 1, + ), + ( + rust_intrinsics::CALLEE_TGAMMA_F64, + I::TgammaF64, + f64_ty, + "__nv_tgamma", + 1, + ), + ( + rust_intrinsics::CALLEE_ILOGB_F32, + I::IlogbF32, + i32_ty, + "__nv_ilogbf", + 1, + ), + ( + rust_intrinsics::CALLEE_ILOGB_F64, + I::IlogbF64, + i32_ty, + "__nv_ilogb", + 1, + ), + ( + rust_intrinsics::CALLEE_LDEXP_F32, + I::LdexpF32, + f32_ty, + "__nv_ldexpf", + 2, + ), + ( + rust_intrinsics::CALLEE_LDEXP_F64, + I::LdexpF64, + f64_ty, + "__nv_ldexp", + 2, + ), + ] { + assert_eq!(I::from_placeholder_callee(callee), Some(intrinsic)); + assert_eq!(intrinsic.arg_count(), arity); + assert_eq!( + float_math_intrinsic_symbol( + &ctx, + intrinsic, + result_ty, + pliron::location::Location::Unknown, + ) + .unwrap(), + symbol + ); + assert!(rust_intrinsics::is_libdevice_backed_placeholder(callee)); + assert!(!rust_intrinsics::is_backend_dependent_libdevice_placeholder(callee)); + } + } + } + #[test] fn no_fma_policy_removes_only_contract_from_fast_float_intrinsics() { let mut ctx = Context::new(); diff --git a/cuda-oxide/crates/mir-lower/tests/lowering_test/math_conversions.rs b/cuda-oxide/crates/mir-lower/tests/lowering_test/math_conversions.rs index 95fbf7bd31..940af0bf53 100644 --- a/cuda-oxide/crates/mir-lower/tests/lowering_test/math_conversions.rs +++ b/cuda-oxide/crates/mir-lower/tests/lowering_test/math_conversions.rs @@ -15,6 +15,267 @@ use pliron::operation::Operation; use crate::common::{append_return, build_test_kernel, lowered_kernel_body, make_test_ctx}; +#[test] +fn test_special_math_mixed_signatures_on_both_backends() -> Result<(), anyhow::Error> { + use dialect_mir::rust_intrinsics as ri; + use pliron::builtin::attributes::StringAttr; + use pliron::builtin::types::{FP32Type, FP64Type, IntegerType, Signedness}; + use pliron::r#type::{TypeHandle, Typed}; + + for backend in [ + mir_lower::IntrinsicBackend::LlvmNvptx, + mir_lower::IntrinsicBackend::LibNvvm, + ] { + let mut ctx = make_test_ctx(); + let f32_ty: TypeHandle = FP32Type::get(&ctx).into(); + let f64_ty: TypeHandle = FP64Type::get(&ctx).into(); + let i32_mir: TypeHandle = IntegerType::get(&ctx, 32, Signedness::Signed).into(); + let i32_llvm: TypeHandle = IntegerType::get(&ctx, 32, Signedness::Signless).into(); + let (module_ptr, entry) = build_test_kernel(&mut ctx, vec![f32_ty, f64_ty, i32_mir]); + let x32 = entry.deref(&ctx).get_argument(0); + let x64 = entry.deref(&ctx).get_argument(1); + let exponent = entry.deref(&ctx).get_argument(2); + let cases = [ + ( + ri::CALLEE_TGAMMA_F32, + "__nv_tgammaf", + f32_ty, + vec![x32], + vec![f32_ty], + ), + ( + ri::CALLEE_TGAMMA_F64, + "__nv_tgamma", + f64_ty, + vec![x64], + vec![f64_ty], + ), + ( + ri::CALLEE_ILOGB_F32, + "__nv_ilogbf", + i32_mir, + vec![x32], + vec![f32_ty], + ), + ( + ri::CALLEE_ILOGB_F64, + "__nv_ilogb", + i32_mir, + vec![x64], + vec![f64_ty], + ), + ( + ri::CALLEE_LDEXP_F32, + "__nv_ldexpf", + f32_ty, + vec![x32, exponent], + vec![f32_ty, i32_llvm], + ), + ( + ri::CALLEE_LDEXP_F64, + "__nv_ldexp", + f64_ty, + vec![x64, exponent], + vec![f64_ty, i32_llvm], + ), + // Control: the pre-existing mixed float/integer powi path. + ( + ri::CALLEE_POWI_F32, + "__nv_powif", + f32_ty, + vec![x32, exponent], + vec![f32_ty, i32_llvm], + ), + ( + ri::CALLEE_POWI_F64, + "__nv_powi", + f64_ty, + vec![x64, exponent], + vec![f64_ty, i32_llvm], + ), + ]; + for (callee, _, result_ty, operands, _) in &cases { + let call = Operation::new( + &mut ctx, + mir::MirCallOp::get_concrete_op_info(), + vec![*result_ty], + operands.clone(), + vec![], + 0, + ); + mir::MirCallOp::new(call).set_attr_callee(&ctx, StringAttr::new((*callee).to_string())); + call.insert_at_back(entry, &ctx); + } + append_return(&mut ctx, entry); + mir_lower::lower_mir_to_llvm_with_options( + &mut ctx, + module_ptr, + mir_lower::LoweringOptions { + intrinsic_backend: backend, + ..Default::default() + }, + ) + .map_err(|error| anyhow::anyhow!("{error}"))?; + + let calls = lowered_kernel_body(&ctx, module_ptr) + .into_iter() + .filter_map(|op| Operation::get_op::(op, &ctx)) + .collect::>(); + assert_eq!(calls.len(), cases.len()); + for call in calls { + let CallOpCallable::Direct(symbol) = call.callee(&ctx) else { + panic!("special math must lower to a direct call"); + }; + let name = symbol.to_string(); + let (_, _, mir_result, _, expected_args) = cases + .iter() + .find(|(_, symbol, _, _, _)| *symbol == name) + .expect("unexpected libdevice symbol"); + let expected_result = if *mir_result == i32_mir { + i32_llvm + } else { + *mir_result + }; + let op = call.get_operation().deref(&ctx); + assert_eq!(op.get_num_results(), 1, "{name}"); + assert_eq!(op.get_result(0).get_type(&ctx), expected_result, "{name}"); + assert_eq!( + op.operands() + .map(|arg| arg.get_type(&ctx)) + .collect::>(), + *expected_args, + "{name}" + ); + } + let module = Operation::get_op::(module_ptr, &ctx).unwrap(); + let ir = llvm_export::export::export_module_to_string(&ctx, &module) + .expect("mixed-signature math must export to LLVM IR"); + for declaration in [ + "declare float @__nv_tgammaf(float", + "declare double @__nv_tgamma(double", + "declare i32 @__nv_ilogbf(float", + "declare i32 @__nv_ilogb(double", + "declare float @__nv_ldexpf(float", + "declare double @__nv_ldexp(double", + ] { + assert!(ir.contains(declaration), "missing {declaration}:\n{ir}"); + } + assert!(!ir.contains(ri::PLACEHOLDER_PREFIX), "{ir}"); + } + Ok(()) +} + +#[test] +fn test_special_math_lowering_rejects_bad_arity_and_types() { + use dialect_mir::rust_intrinsics as ri; + use pliron::builtin::attributes::StringAttr; + use pliron::builtin::types::{FP32Type, FP64Type, IntegerType, Signedness}; + use pliron::r#type::TypeHandle; + + for backend in [ + mir_lower::IntrinsicBackend::LlvmNvptx, + mir_lower::IntrinsicBackend::LibNvvm, + ] { + for bad_case in 0..10 { + let mut ctx = make_test_ctx(); + let f32_ty: TypeHandle = FP32Type::get(&ctx).into(); + let f64_ty: TypeHandle = FP64Type::get(&ctx).into(); + let i32_ty: TypeHandle = IntegerType::get(&ctx, 32, Signedness::Signed).into(); + let i64_ty: TypeHandle = IntegerType::get(&ctx, 64, Signedness::Signed).into(); + let (module_ptr, entry) = + build_test_kernel(&mut ctx, vec![f32_ty, f64_ty, i32_ty, i64_ty]); + let x32 = entry.deref(&ctx).get_argument(0); + let x64 = entry.deref(&ctx).get_argument(1); + let exponent = entry.deref(&ctx).get_argument(2); + let wide_exponent = entry.deref(&ctx).get_argument(3); + let (callee, results, args, diagnostic) = match bad_case { + 0 => ( + ri::CALLEE_TGAMMA_F32, + vec![f32_ty], + vec![], + "requires 1 operand(s)", + ), + 1 => ( + ri::CALLEE_LDEXP_F64, + vec![f64_ty], + vec![x64], + "requires 2 operand(s)", + ), + 2 => ( + ri::CALLEE_ILOGB_F32, + vec![f32_ty], + vec![x32], + "signature mismatch", + ), + 3 => ( + ri::CALLEE_LDEXP_F32, + vec![f32_ty], + vec![x32, x32], + "signature mismatch", + ), + 4 => ( + ri::CALLEE_LDEXP_F64, + vec![f64_ty], + vec![x64, wide_exponent], + "signature mismatch", + ), + 5 => ( + ri::CALLEE_TGAMMA_F32, + vec![f32_ty], + vec![x64], + "signature mismatch", + ), + 6 => ( + ri::CALLEE_TGAMMA_F64, + vec![f32_ty], + vec![x64], + "signature mismatch", + ), + 7 => ( + ri::CALLEE_ILOGB_F64, + vec![i64_ty], + vec![x64], + "signature mismatch", + ), + 8 => ( + ri::CALLEE_LDEXP_F32, + vec![f64_ty], + vec![x32, exponent], + "signature mismatch", + ), + _ => ( + ri::CALLEE_TGAMMA_F64, + vec![], + vec![x64], + "must have one result", + ), + }; + let call = Operation::new( + &mut ctx, + mir::MirCallOp::get_concrete_op_info(), + results, + args, + vec![], + 0, + ); + mir::MirCallOp::new(call).set_attr_callee(&ctx, StringAttr::new(callee.to_string())); + call.insert_at_back(entry, &ctx); + append_return(&mut ctx, entry); + let error = mir_lower::lower_mir_to_llvm_with_options( + &mut ctx, + module_ptr, + mir_lower::LoweringOptions { + intrinsic_backend: backend, + ..Default::default() + }, + ) + .expect_err("malformed special math calls must fail") + .to_string(); + assert!(error.contains(diagnostic), "case {bad_case}: {error}"); + } + } +} + #[test] fn test_fast_float_intrinsics_lower_to_explicit_fast_binops() -> Result<(), anyhow::Error> { use dialect_mir::rust_intrinsics; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/math_special/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/math_special/Cargo.lock new file mode 100644 index 0000000000..71036a0499 --- /dev/null +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/math_special/Cargo.lock @@ -0,0 +1,761 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn", + "which", +] + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "cuda-artifact-finalizer" +version = "0.2.1" +dependencies = [ + "libnvvm-sys", + "nvjitlink-sys", + "serde", + "sha2", + "thiserror", +] + +[[package]] +name = "cuda-bindings" +version = "0.5.0" +dependencies = [ + "bindgen", + "libloading", + "prettyplease", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "cuda-core" +version = "0.5.0" +dependencies = [ + "anyhow", + "cuda-bindings", + "cuda-core-derive", + "half", + "oxide-artifacts", +] + +[[package]] +name = "cuda-core-derive" +version = "0.5.0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "cuda-device" +version = "0.2.1" +dependencies = [ + "cuda-macros", +] + +[[package]] +name = "cuda-host" +version = "0.2.1" +dependencies = [ + "cuda-artifact-finalizer", + "cuda-core", + "cuda-device", + "cuda-macros", + "half", + "libc", + "oxide-artifacts", + "ptx-parse", + "sha2", + "thiserror", +] + +[[package]] +name = "cuda-macros" +version = "0.2.1" +dependencies = [ + "oxide-tile-export", + "proc-macro2", + "quote", + "reserved-oxide-symbols", + "syn", +] + +[[package]] +name = "cuda-target-spec" +version = "0.1.0" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libnvvm-sys" +version = "0.2.1" +dependencies = [ + "cuda-target-spec", + "libloading", + "thiserror", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "math_special" +version = "0.1.0" +dependencies = [ + "cuda-core", + "cuda-device", + "cuda-host", + "libm", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nvjitlink-sys" +version = "0.2.1" +dependencies = [ + "libloading", + "thiserror", +] + +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "crc32fast", + "hashbrown 0.15.5", + "indexmap", + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "oxide-artifacts" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" +dependencies = [ + "object", +] + +[[package]] +name = "oxide-tile-export" +version = "0.2.1" +dependencies = [ + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "ptx-parse" +version = "0.1.0" + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reserved-oxide-symbols" +version = "0.2.1" + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.59.0", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/math_special/Cargo.toml b/cuda-oxide/crates/rustc-codegen-cuda/examples/math_special/Cargo.toml new file mode 100644 index 0000000000..6e87dfe5a4 --- /dev/null +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/math_special/Cargo.toml @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "math_special" +version = "0.1.0" +edition = "2024" +license = "Apache-2.0" + +[workspace] + +[dependencies] +cuda-device = { path = "../../../cuda-device" } +cuda-host = { path = "../../../cuda-host" } +cuda-core = { path = "../../../../../cuda-core" } +libm = "=0.2.16" diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/math_special/README.md b/cuda-oxide/crates/rustc-codegen-cuda/examples/math_special/README.md new file mode 100644 index 0000000000..a21149f0ab --- /dev/null +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/math_special/README.md @@ -0,0 +1,70 @@ + + + +# Special scalar math + +Exercises the first scalar additions for issue #1224: + +| Rust call | f32 libdevice symbol | f64 libdevice symbol | +| --- | --- | --- | +| `libm::tgamma{f}` and `f32/f64::gamma` | `__nv_tgammaf` | `__nv_tgamma` | +| `libm::ilogb{f}` | `__nv_ilogbf` | `__nv_ilogb` | +| `libm::ldexp{f}`, `libm::scalbn{f}` | `__nv_ldexpf` | `__nv_ldexp` | + +`ilogb` returns an `i32`; the exponent argument of `ldexp`/`scalbn` is +an `i32`. `float_gamma` is enabled to exercise the standard-library cmath +shim as well as the `libm` entry points. Use the repository's pinned toolchain +and its existing `RUSTC_BOOTSTRAP` configuration. + +Before these mappings are added, the standard gamma path is rejected by the +collector and the libm paths have no dedicated libdevice lowering. Capture the +baseline diagnostic on the unmodified tree; no local baseline/GPU run is claimed +by this source package. + +The example checks 264 results: 22 inputs, six checks per input, and two widths. +It compares gamma to host libm within a relative tolerance, compares integer +logarithms exactly, and compares scaled values bit for bit. The inputs include +negative non-integers, both exponent signs, subnormals, scaling overflow and +underflow. A user function with a libm-like module name must retain its own +body. `ilogb` inputs are finite and nonzero; its domain-error sentinels are +outside this example's comparison. + +The lowering tests independently verify all six libdevice signatures on both +backends, reject malformed calls, and retain `powi` as a mixed-signature control. +Importer tests check public/internal libm spellings, aliases, cmath gamma, +unrelated names, and functions deferred to later PRs. + +## Validation + +Run from `cuda-oxide/` after copying the files into the existing checkout: + +```bash +cargo oxide fmt +cargo oxide fmt --check +git diff --check +cargo test -p dialect-mir -p mir-importer -p mir-lower --all-targets +cargo clippy -p dialect-mir -p mir-importer -p mir-lower --all-targets -- -D warnings +(cd crates/rustc-codegen-cuda && cargo test --lib) +(cd crates/rustc-codegen-cuda && cargo clippy --all-targets -- -D warnings) +(cd crates/rustc-codegen-cuda/examples/math_special && cargo clippy --locked --all-targets -- -D warnings) +cargo oxide build math_special +cargo oxide run math_special +cargo oxide run libm_math +cargo oxide run math_hyperbolic +bash scripts/check-example-smoketest-contract.sh +bash scripts/sync-example-locks.sh --check +``` + +`math_special` must print `SUCCESS: math_special passed 264 checks across f32 and f64`. +GPU commands require a working CUDA toolkit, driver and device. The example is +a separate workspace; the parent workspace's tests do not build it. + +Before marking the PR ready, run the repository gate from the repository root: + +```bash +just -f cuda-oxide/Justfile check +``` + +This package has undergone static source and archive checks only. Rust, Cargo, +and CUDA were unavailable in the preparation environment; compile, clippy and +GPU validation are pending execution on a configured checkout. diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/math_special/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/math_special/src/main.rs new file mode 100644 index 0000000000..89c688e14f --- /dev/null +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/math_special/src/main.rs @@ -0,0 +1,233 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! tgamma, ilogb, ldexp and scalbn in f32/f64 device code. +//! Run with `cargo oxide run math_special`. +#![feature(float_gamma)] +// The width-generic host macro casts both f32 and f64 results to f64. +#![allow(clippy::too_many_arguments, clippy::unnecessary_cast)] + +use cuda_core::simt::LaunchConfig; +use cuda_core::{CudaContext, DeviceBuffer}; +use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; + +// An unrelated function with a libm-like path must retain its own body. +mod libm_lookalike { + #[inline(never)] + pub fn tgammaf(x: f32) -> f32 { + x + 1000.0 + } + + #[inline(never)] + pub fn tgamma(x: f64) -> f64 { + x + 1000.0 + } +} + +#[cuda_module] +mod kernels { + use super::*; + + #[kernel] + pub fn special_f32( + xs: &[f32], + exponents: &[i32], + mut gamma: DisjointSlice, + mut std_gamma: DisjointSlice, + mut logarithm: DisjointSlice, + mut scaled: DisjointSlice, + mut alias: DisjointSlice, + mut control: DisjointSlice, + ) { + let i = thread::index_1d().get(); + if i < xs.len() { + let x = xs[i]; + if let Some(out) = gamma.get_mut(thread::index_1d()) { + *out = libm::tgammaf(x); + } + if let Some(out) = std_gamma.get_mut(thread::index_1d()) { + *out = x.gamma(); + } + if let Some(out) = logarithm.get_mut(thread::index_1d()) { + *out = libm::ilogbf(x); + } + if let Some(out) = scaled.get_mut(thread::index_1d()) { + *out = libm::ldexpf(x, exponents[i]); + } + if let Some(out) = alias.get_mut(thread::index_1d()) { + *out = libm::scalbnf(x, exponents[i]); + } + if let Some(out) = control.get_mut(thread::index_1d()) { + *out = libm_lookalike::tgammaf(x); + } + } + } + + #[kernel] + pub fn special_f64( + xs: &[f64], + exponents: &[i32], + mut gamma: DisjointSlice, + mut std_gamma: DisjointSlice, + mut logarithm: DisjointSlice, + mut scaled: DisjointSlice, + mut alias: DisjointSlice, + mut control: DisjointSlice, + ) { + let i = thread::index_1d().get(); + if i < xs.len() { + let x = xs[i]; + if let Some(out) = gamma.get_mut(thread::index_1d()) { + *out = libm::tgamma(x); + } + if let Some(out) = std_gamma.get_mut(thread::index_1d()) { + *out = x.gamma(); + } + if let Some(out) = logarithm.get_mut(thread::index_1d()) { + *out = libm::ilogb(x); + } + if let Some(out) = scaled.get_mut(thread::index_1d()) { + *out = libm::ldexp(x, exponents[i]); + } + if let Some(out) = alias.get_mut(thread::index_1d()) { + *out = libm::scalbn(x, exponents[i]); + } + if let Some(out) = control.get_mut(thread::index_1d()) { + *out = libm_lookalike::tgamma(x); + } + } + } +} + +fn close(got: f64, expected: f64, rtol: f64) -> bool { + if expected.is_nan() { + got.is_nan() + } else if expected.is_infinite() || expected == 0.0 { + got.to_bits() == expected.to_bits() + } else { + got.is_finite() && (got - expected).abs() <= rtol * expected.abs() + } +} + +fn main() -> Result<(), Box> { + let ctx = CudaContext::new(0)?; + let stream = ctx.default_stream(); + let module = kernels::load(&ctx)?; + let mut failures = 0usize; + let mut checks = 0usize; + + macro_rules! check_width { + ($ty:ty, $kernel:ident, $gamma:ident, $ilogb:ident, $ldexp:ident, $rtol:expr, $extra:expr) => {{ + // Nonzero finite inputs keep ilogb away from C's domain-error + // sentinels. Both gamma signs and both exponent signs are covered. + let mut xs: Vec<$ty> = vec![ + -3.5, -2.5, -1.25, -0.5, 0.125, 0.5, 0.75, 1.0, 1.5, 2.0, 3.0, 4.5, 6.0, 8.0, 10.0, + 0.33, + ]; + let mut exponents = vec![-12, -4, -1, 0, 1, 4, 8, 24, -24, 100, -100, 3, 2, -3, 5, -5]; + // Width-specific subnormal inputs and scaling overflow/underflow. + for (x, exponent) in $extra { + xs.push(x); + exponents.push(exponent); + } + let n = xs.len(); + let inputs = DeviceBuffer::from_host(&stream, &xs)?; + let powers = DeviceBuffer::from_host(&stream, &exponents)?; + let mut gamma = DeviceBuffer::<$ty>::zeroed(&stream, n)?; + let mut std_gamma = DeviceBuffer::<$ty>::zeroed(&stream, n)?; + let mut logarithm = DeviceBuffer::::zeroed(&stream, n)?; + let mut scaled = DeviceBuffer::<$ty>::zeroed(&stream, n)?; + let mut alias = DeviceBuffer::<$ty>::zeroed(&stream, n)?; + let mut control = DeviceBuffer::<$ty>::zeroed(&stream, n)?; + // SAFETY: every buffer has n elements; the kernel bounds-checks i. + unsafe { + module.$kernel( + &stream, + LaunchConfig::for_num_elems(n as u32), + &inputs, + &powers, + &mut gamma, + &mut std_gamma, + &mut logarithm, + &mut scaled, + &mut alias, + &mut control, + )?; + } + let gamma = gamma.to_host_vec(&stream)?; + let std_gamma = std_gamma.to_host_vec(&stream)?; + let logarithm = logarithm.to_host_vec(&stream)?; + let scaled = scaled.to_host_vec(&stream)?; + let alias = alias.to_host_vec(&stream)?; + let control = control.to_host_vec(&stream)?; + for (i, &x) in xs.iter().enumerate() { + let expected_gamma = libm::$gamma(x); + let expected_logarithm = libm::$ilogb(x); + let expected_scaled = libm::$ldexp(x, exponents[i]); + for (name, valid) in [ + ( + "libm gamma", + close(gamma[i] as f64, expected_gamma as f64, $rtol), + ), + ( + "std gamma", + close(std_gamma[i] as f64, expected_gamma as f64, $rtol), + ), + ("ilogb", logarithm[i] == expected_logarithm), + ("ldexp", scaled[i].to_bits() == expected_scaled.to_bits()), + ("scalbn", alias[i].to_bits() == expected_scaled.to_bits()), + ("control", control[i].to_bits() == (x + 1000.0).to_bits()), + ] { + checks += 1; + if !valid { + failures += 1; + eprintln!( + "{} {name}[{i}]: x={x:?}, exponent={}", + stringify!($ty), + exponents[i] + ); + } + } + } + }}; + } + check_width!( + f32, + special_f32, + tgammaf, + ilogbf, + ldexpf, + 4e-6, + [ + (f32::from_bits(1), 149), + (f32::MIN_POSITIVE, -1), + (1.0, -149), + (-1.0, -150), + (1.0, 128), + (-1.0, 128), + ] + ); + check_width!( + f64, + special_f64, + tgamma, + ilogb, + ldexp, + 2e-13, + [ + (f64::from_bits(1), 1074), + (f64::MIN_POSITIVE, -1), + (1.0, -1074), + (-1.0, -1075), + (1.0, 1024), + (-1.0, 1024), + ] + ); + if failures != 0 { + return Err(format!("FAILED: {failures} of {checks} checks failed").into()); + } + println!("SUCCESS: math_special passed {checks} checks across f32 and f64"); + Ok(()) +} diff --git a/cuda-oxide/crates/rustc-codegen-cuda/src/collector.rs b/cuda-oxide/crates/rustc-codegen-cuda/src/collector.rs index 078099ebaf..239f34060b 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/src/collector.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/src/collector.rs @@ -776,6 +776,8 @@ fn is_intrinsic_lowered_cmath_shim(fn_path: &str) -> bool { | "std::sys::cmath::log1p" | "std::sys::cmath::hypotf" | "std::sys::cmath::hypot" + | "std::sys::cmath::tgammaf" + | "std::sys::cmath::tgamma" ) } @@ -795,9 +797,9 @@ fn is_intrinsic_lowered_cmath_shim(fn_path: &str) -> bool { /// /// Nothing in these wrappers touches the OS or the heap, so collect them like /// any other reachable pure function. Every stable method's shim is in -/// `is_intrinsic_lowered_cmath_shim`; the unstable `gamma`/`erf` family and -/// `f128` still stop one level deeper, at their shim, with the guard naming -/// that shim. +/// `is_intrinsic_lowered_cmath_shim`; `gamma` is also intercepted for f32/f64. +/// Unsupported methods such as `erf`, and f128 shims, still stop one level +/// deeper with the guard naming that shim. fn is_std_float_inherent_method(fn_path: &str) -> bool { ["f16", "f32", "f64", "f128"] .iter() @@ -3212,6 +3214,26 @@ mod tests { }; use rustc_middle::mir::BasicBlock; + #[test] + fn special_math_cmath_shims_are_an_exact_allowlist() { + for path in ["std::sys::cmath::tgammaf", "std::sys::cmath::tgamma"] { + assert!(super::is_intrinsic_lowered_cmath_shim(path), "{path}"); + } + for path in [ + "user::std::sys::cmath::tgamma", + "std::sys::cmath::tgammaf128", + "std::sys::cmath::lgamma_r", + "std::sys::cmath::frexp", + "std::sys::cmath::erf", + "std::io::stdio::_print", + ] { + assert!(!super::is_intrinsic_lowered_cmath_shim(path), "{path}"); + } + assert!(super::is_intrinsic_lowered_cmath_shim( + "std::sys::cmath::hypot" + )); + } + #[test] fn std_float_wrappers_are_collected_not_forbidden() { for ty in ["f16", "f32", "f64", "f128"] { From c430870333dd30c76199843b8f8c0e804632d1ab Mon Sep 17 00:00:00 2001 From: Raul Estrada Date: Sun, 25 Oct 2026 17:53:48 -0600 Subject: [PATCH 3/4] feat: add libdevice frexp tuple adapter Signed-off-by: Raul Estrada --- .../crates/dialect-mir/src/rust_intrinsics.rs | 9 + .../terminator/intrinsics/float_math.rs | 46 +++- .../crates/mir-lower/src/convert/ops/call.rs | 185 ++++++++++++- .../tests/lowering_test/math_conversions.rs | 259 ++++++++++++++++++ .../examples/math_special/README.md | 94 +++---- .../examples/math_special/src/main.rs | 245 ++++++++++++++++- 6 files changed, 785 insertions(+), 53 deletions(-) diff --git a/cuda-oxide/crates/dialect-mir/src/rust_intrinsics.rs b/cuda-oxide/crates/dialect-mir/src/rust_intrinsics.rs index 4d4b5387ea..9b640c64ea 100644 --- a/cuda-oxide/crates/dialect-mir/src/rust_intrinsics.rs +++ b/cuda-oxide/crates/dialect-mir/src/rust_intrinsics.rs @@ -224,6 +224,11 @@ pub const CALLEE_LDEXP_F32: &str = placeholder!("ldexpf32"); /// Placeholder call used for `libm::ldexp`. pub const CALLEE_LDEXP_F64: &str = placeholder!("ldexpf64"); +/// Placeholder call used for `libm::frexpf`, returning `(f32, i32)`. +pub const CALLEE_FREXP_F32: &str = placeholder!("frexpf32"); +/// Placeholder call used for `libm::frexp`, returning `(f64, i32)`. +pub const CALLEE_FREXP_F64: &str = placeholder!("frexpf64"); + /// Placeholder call used for `core::intrinsics::fadd_fast` (generic over float type). /// /// Lowered to `llvm.fadd` with explicit `fast` fast-math flags. The `f*_fast` intrinsics @@ -369,6 +374,8 @@ pub fn is_libdevice_backed_placeholder(callee: &str) -> bool { | CALLEE_ILOGB_F64 | CALLEE_LDEXP_F32 | CALLEE_LDEXP_F64 + | CALLEE_FREXP_F32 + | CALLEE_FREXP_F64 ) } @@ -477,6 +484,8 @@ mod tests { CALLEE_ILOGB_F64, CALLEE_LDEXP_F32, CALLEE_LDEXP_F64, + CALLEE_FREXP_F32, + CALLEE_FREXP_F64, ]; /// Every placeholder that is libdevice-backed only under the libNVVM diff --git a/cuda-oxide/crates/mir-importer/src/translator/terminator/intrinsics/float_math.rs b/cuda-oxide/crates/mir-importer/src/translator/terminator/intrinsics/float_math.rs index 3eb562bed4..92bbb5cef0 100644 --- a/cuda-oxide/crates/mir-importer/src/translator/terminator/intrinsics/float_math.rs +++ b/cuda-oxide/crates/mir-importer/src/translator/terminator/intrinsics/float_math.rs @@ -173,6 +173,10 @@ pub enum RustFloatMathIntrinsic { LdexpF32, /// `libm::ldexp`. LdexpF64, + /// `libm::frexpf`, returning `(f32, i32)`. + FrexpF32, + /// `libm::frexp`, returning `(f64, i32)`. + FrexpF64, /// Generic `core::intrinsics::fadd_fast` (lowered to `llvm.fadd` + fast-math). FaddFast, /// Generic `core::intrinsics::fsub_fast` (lowered to `llvm.fsub` + fast-math). @@ -448,6 +452,8 @@ impl RustFloatMathIntrinsic { "ilogb" => Some(Self::IlogbF64), "ldexpf" | "scalbnf" => Some(Self::LdexpF32), "ldexp" | "scalbn" => Some(Self::LdexpF64), + "frexpf" => Some(Self::FrexpF32), + "frexp" => Some(Self::FrexpF64), _ => None, } } @@ -549,6 +555,8 @@ impl RustFloatMathIntrinsic { Self::IlogbF64 => rust_intrinsics::CALLEE_ILOGB_F64, Self::LdexpF32 => rust_intrinsics::CALLEE_LDEXP_F32, Self::LdexpF64 => rust_intrinsics::CALLEE_LDEXP_F64, + Self::FrexpF32 => rust_intrinsics::CALLEE_FREXP_F32, + Self::FrexpF64 => rust_intrinsics::CALLEE_FREXP_F64, Self::FaddFast => rust_intrinsics::CALLEE_FADD_FAST, Self::FsubFast => rust_intrinsics::CALLEE_FSUB_FAST, Self::FmulFast => rust_intrinsics::CALLEE_FMUL_FAST, @@ -568,8 +576,8 @@ pub fn is_libm_path(name: &str) -> bool { } /// Recognize `libm::sincosf` / `libm::sincos` (glam's `nostd-libm` lowering of -/// `f32::sin_cos`). These return a `(sin, cos)` tuple, so they do not fit the -/// scalar `RustFloatMathIntrinsic` dispatch; [`emit_sincos`] handles them. +/// `f32::sin_cos`). [`emit_sincos`] handles these separately by emitting +/// two scalar placeholders and constructing their tuple in the importer. /// Returns `Some(is_f64)` when `name` is a libm sincos function. pub fn libm_sincos_is_f64(name: &str) -> Option { if !is_libm_path(name) { @@ -762,6 +770,38 @@ mod tests { use super::*; use dialect_mir::rust_intrinsics; + #[test] + fn frexp_paths_resolve_only_libm_entry_points() { + use RustFloatMathIntrinsic as I; + for (name, variant, placeholder) in [ + ("frexpf", I::FrexpF32, rust_intrinsics::CALLEE_FREXP_F32), + ("frexp", I::FrexpF64, rust_intrinsics::CALLEE_FREXP_F64), + ] { + for path in [ + format!("libm::{name}"), + format!("libm::math::{name}::{name}"), + ] { + assert_eq!(I::from_core_path(&path), Some(variant), "{path}"); + assert_eq!(variant.placeholder_callee(), placeholder); + assert!(rust_intrinsics::is_libdevice_backed_placeholder( + placeholder + )); + } + for root in [ + "user::libm", + "libm_lookalike", + "std::sys::cmath", + "core::intrinsics", + ] { + let path = format!("{root}::{name}"); + assert_eq!(I::from_core_path(&path), None, "{path}"); + } + } + assert_eq!(I::from_core_path("libm::sincos"), None); + assert_eq!(libm_sincos_is_f64("libm::sincos"), Some(true)); + assert_eq!(I::from_core_path("libm::ldexp"), Some(I::LdexpF64)); + } + #[test] fn special_math_paths_resolve_to_typed_placeholders() { use RustFloatMathIntrinsic as I; @@ -806,7 +846,7 @@ mod tests { Some(I::PowiF32) ); assert_eq!(libm_sincos_is_f64("libm::sincosf"), Some(false)); - for name in ["frexp", "modf", "remquo", "lgamma_r", "tgammaf128"] { + for name in ["modf", "remquo", "lgamma_r", "tgammaf128"] { assert_eq!(I::from_core_path(&format!("libm::{name}")), None); } } diff --git a/cuda-oxide/crates/mir-lower/src/convert/ops/call.rs b/cuda-oxide/crates/mir-lower/src/convert/ops/call.rs index b18e43b12c..8e98b2031b 100644 --- a/cuda-oxide/crates/mir-lower/src/convert/ops/call.rs +++ b/cuda-oxide/crates/mir-lower/src/convert/ops/call.rs @@ -87,7 +87,7 @@ use pliron::builtin::type_interfaces::FunctionTypeInterface; use pliron::builtin::types::{FP32Type, FP64Type, IntegerType, Signedness}; use pliron::context::{Context, Ptr}; use pliron::irbuild::dialect_conversion::{DialectConversionRewriter, OperandsInfo}; -use pliron::irbuild::inserter::Inserter; +use pliron::irbuild::inserter::{Inserter, OpInsertionPoint}; use pliron::irbuild::rewriter::Rewriter; use pliron::linked_list::ContainsLinkedList; use pliron::op::Op; @@ -268,6 +268,8 @@ enum RustFloatMathIntrinsic { IlogbF64, LdexpF32, LdexpF64, + FrexpF32, + FrexpF64, FaddFast, FsubFast, FmulFast, @@ -356,6 +358,8 @@ impl RustFloatMathIntrinsic { rust_intrinsics::CALLEE_ILOGB_F64 => Some(Self::IlogbF64), rust_intrinsics::CALLEE_LDEXP_F32 => Some(Self::LdexpF32), rust_intrinsics::CALLEE_LDEXP_F64 => Some(Self::LdexpF64), + rust_intrinsics::CALLEE_FREXP_F32 => Some(Self::FrexpF32), + rust_intrinsics::CALLEE_FREXP_F64 => Some(Self::FrexpF64), rust_intrinsics::CALLEE_FADD_FAST => Some(Self::FaddFast), rust_intrinsics::CALLEE_FSUB_FAST => Some(Self::FsubFast), rust_intrinsics::CALLEE_FMUL_FAST => Some(Self::FmulFast), @@ -466,6 +470,8 @@ impl RustFloatMathIntrinsic { Self::IlogbF64 => Ok("__nv_ilogb"), Self::LdexpF32 => Ok("__nv_ldexpf"), Self::LdexpF64 => Ok("__nv_ldexp"), + Self::FrexpF32 => Ok("__nv_frexpf"), + Self::FrexpF64 => Ok("__nv_frexp"), Self::FaddFast | Self::FsubFast | Self::FmulFast | Self::FdivFast | Self::FremFast => { // The `f*_fast` intrinsics lower directly to LLVM `fadd`/`fsub`/ // `fmul`/`fdiv`/`frem` with fast-math flags, not to a libdevice @@ -480,7 +486,7 @@ impl RustFloatMathIntrinsic { } } - /// Number of operands expected by the libdevice function. + /// Number of operands expected by the Rust placeholder. fn arg_count(self) -> usize { match self { Self::PowiF32 @@ -1317,6 +1323,13 @@ fn convert_rust_float_math_intrinsic( return lower_float_minmax_nsz(ctx, rewriter, op, &args, predicate); } + if matches!( + intrinsic, + RustFloatMathIntrinsic::FrexpF32 | RustFloatMathIntrinsic::FrexpF64 + ) { + return convert_rust_frexp(ctx, rewriter, op, intrinsic, args[0]); + } + let result_mir_ty = op.deref(ctx).get_result(0).get_type(ctx); let result_ty = convert_type(ctx, result_mir_ty).map_err(anyhow_to_pliron)?; @@ -1377,6 +1390,174 @@ fn convert_rust_float_math_intrinsic( Ok(()) } +/// Emit a libdevice scalar return plus one out-parameter. The caller supplies +/// the initial out value and handles its Rust tuple and semantic adjustments. +/// Keeping storage and ABI construction here lets modf/remquo share the path. +fn emit_libdevice_out_param_call( + ctx: &mut Context, + rewriter: &mut DialectConversionRewriter, + op: Ptr, + symbol: &str, + args: &[Value], + scalar_ty: TypeHandle, + initial_out: Value, +) -> Result<(Value, Value)> { + let loc = op.deref(ctx).loc().clone(); + let block = op.deref(ctx).get_parent_block().ok_or_else(|| { + pliron::input_error!( + loc.clone(), + "libdevice out-parameter call has no parent block" + ) + })?; + let parent = helpers::get_parent_func(ctx, block).map_err(anyhow_to_pliron)?; + let entry = parent + .deref(ctx) + .get_region(0) + .deref(ctx) + .iter(ctx) + .next() + .ok_or_else(|| { + pliron::input_error!( + loc.clone(), + "libdevice out-parameter call has no entry block" + ) + })?; + let first = entry.deref(ctx).iter(ctx).next().ok_or_else(|| { + pliron::input_error!( + loc.clone(), + "libdevice out-parameter call has an empty entry block" + ) + })?; + // A call inside a loop must reuse a static slot, rather than allocate + // additional stack storage on every iteration. Its size constant must + // dominate it too; initialization stays at the actual call site. + let one_attr = IntegerAttr::new( + IntegerType::get(ctx, 32, Signedness::Signless), + APInt::from_u64(1, NonZeroUsize::new(32).unwrap()), + ); + let one = llvm::ConstantOp::new(ctx, Box::new(one_attr)); + let saved_ip = rewriter.get_insertion_point(); + rewriter.set_insertion_point(OpInsertionPoint::BeforeOperation(first)); + rewriter.insert_operation(ctx, one.get_operation()); + let out_ty = initial_out.get_type(ctx); + let one_value = one.get_operation().deref(ctx).get_result(0); + let slot = llvm::AllocaOp::new(ctx, out_ty, one_value, 0); + rewriter.insert_operation(ctx, slot.get_operation()); + rewriter.set_insertion_point(saved_ip); + let ptr = slot.get_operation().deref(ctx).get_result(0); + let initialize = llvm::StoreOp::new(ctx, initial_out, ptr); + rewriter.insert_operation(ctx, initialize.get_operation()); + let mut call_args = args.to_vec(); + call_args.push(ptr); + let arg_types = call_args.iter().map(|arg| arg.get_type(ctx)).collect(); + let func_ty = llvm_types::FuncType::get(ctx, scalar_ty, arg_types, false); + helpers::ensure_intrinsic_declared(ctx, block, symbol, func_ty).map_err(|error| { + pliron::input_error!(loc.clone(), "Failed to declare {symbol}: {error}") + })?; + let symbol = symbol + .try_into() + .map_err(|error| pliron::input_error!(loc.clone(), "Invalid libdevice name: {error:?}"))?; + let call = llvm::CallOp::new(ctx, CallOpCallable::Direct(symbol), func_ty, call_args); + crate::convert::preserve_location(ctx, op, call.get_operation()); + rewriter.insert_operation(ctx, call.get_operation()); + let out = llvm::LoadOp::new(ctx, ptr, out_ty); + rewriter.insert_operation(ctx, out.get_operation()); + Ok(( + call.get_operation().deref(ctx).get_result(0), + out.get_operation().deref(ctx).get_result(0), + )) +} + +/// Adapt libdevice's `(F, i32*) -> F` ABI to libm's `(F) -> (F, i32)`. +fn convert_rust_frexp( + ctx: &mut Context, + rewriter: &mut DialectConversionRewriter, + op: Ptr, + intrinsic: RustFloatMathIntrinsic, + x: Value, +) -> Result<()> { + let loc = op.deref(ctx).loc().clone(); + let (float_ty, width, exponent_mask): (TypeHandle, u32, u64) = match intrinsic { + RustFloatMathIntrinsic::FrexpF32 => (FP32Type::get(ctx).into(), 32, 0x7f80_0000), + RustFloatMathIntrinsic::FrexpF64 => (FP64Type::get(ctx).into(), 64, 0x7ff0_0000_0000_0000), + _ => unreachable!("frexp converter requires a frexp placeholder"), + }; + let result_mir_ty = op.deref(ctx).get_result(0).get_type(ctx); + let layout = { + let ty = result_mir_ty.deref(ctx); + let Some(tuple) = ty.downcast_ref::() else { + return pliron::input_err!(loc, "Rust frexp intrinsic requires a (float, i32) result"); + }; + let fields = tuple.get_types(); + let i32_mir: TypeHandle = IntegerType::get(ctx, 32, Signedness::Signed).into(); + if fields != [float_ty, i32_mir] || x.get_type(ctx) != float_ty { + return pliron::input_err!(loc, "Rust frexp intrinsic signature mismatch"); + } + StructLayoutInfo::of_tuple(tuple) + }; + let map = build_struct_slot_map(ctx, &layout).map_err(anyhow_to_pliron)?; + let mut constant = |ctx: &mut Context, value: u64, bits: u32| { + let ty = IntegerType::get(ctx, bits, Signedness::Signless); + let attr = IntegerAttr::new( + ty, + APInt::from_u64(value, NonZeroUsize::new(bits as usize).unwrap()), + ); + let constant = llvm::ConstantOp::new(ctx, Box::new(attr)); + rewriter.insert_operation(ctx, constant.get_operation()); + constant.get_operation().deref(ctx).get_result(0) + }; + let zero = constant(ctx, 0, 32); + let mask = constant(ctx, exponent_mask, width); + let symbol = intrinsic.libdevice_name(ctx, float_ty, loc.clone())?; + let (fraction, exponent) = + emit_libdevice_out_param_call(ctx, rewriter, op, symbol, &[x], float_ty, zero)?; + // libm returns (x, 0) for infinities and NaNs; libdevice leaves the + // exponent unspecified. Bit tests preserve NaN payloads without floating + // comparisons or fast-math assumptions. The select preserves x itself. + let bits_ty = IntegerType::get(ctx, width, Signedness::Signless).into(); + let bits = llvm::BitcastOp::new(ctx, x, bits_ty); + rewriter.insert_operation(ctx, bits.get_operation()); + let bits_value = bits.get_operation().deref(ctx).get_result(0); + let masked = llvm::AndOp::new(ctx, bits_value, mask); + rewriter.insert_operation(ctx, masked.get_operation()); + let masked_value = masked.get_operation().deref(ctx).get_result(0); + let non_finite = llvm::ICmpOp::new( + ctx, + llvm_export::attributes::ICmpPredicateAttr::EQ, + masked_value, + mask, + ); + rewriter.insert_operation(ctx, non_finite.get_operation()); + let predicate = non_finite.get_operation().deref(ctx).get_result(0); + let fraction = llvm::SelectOp::new(ctx, predicate, x, fraction); + rewriter.insert_operation(ctx, fraction.get_operation()); + let exponent = llvm::SelectOp::new(ctx, predicate, zero, exponent); + rewriter.insert_operation(ctx, exponent.get_operation()); + let fields = [ + fraction.get_operation().deref(ctx).get_result(0), + exponent.get_operation().deref(ctx).get_result(0), + ]; + let undef = llvm::UndefOp::new(ctx, map.llvm_struct_ty); + rewriter.insert_operation(ctx, undef.get_operation()); + let mut value = undef.get_operation().deref(ctx).get_result(0); + let mut last = undef.get_operation(); + for (field, operand) in fields.into_iter().enumerate() { + let slot = map.decl_to_llvm[field].ok_or_else(|| { + pliron::input_error!( + loc.clone(), + "Rust frexp result field has no LLVM storage slot" + ) + })?; + let insert = llvm::InsertValueOp::new(ctx, value, operand, vec![slot]); + rewriter.insert_operation(ctx, insert.get_operation()); + value = insert.get_operation().deref(ctx).get_result(0); + last = insert.get_operation(); + } + crate::convert::preserve_location(ctx, op, last); + rewriter.replace_operation(ctx, op, last); + Ok(()) +} + /// Resolve the callee symbol a float-math intrinsic lowers to under the /// active intrinsic backend. /// diff --git a/cuda-oxide/crates/mir-lower/tests/lowering_test/math_conversions.rs b/cuda-oxide/crates/mir-lower/tests/lowering_test/math_conversions.rs index 940af0bf53..0c7e406400 100644 --- a/cuda-oxide/crates/mir-lower/tests/lowering_test/math_conversions.rs +++ b/cuda-oxide/crates/mir-lower/tests/lowering_test/math_conversions.rs @@ -15,6 +15,265 @@ use pliron::operation::Operation; use crate::common::{append_return, build_test_kernel, lowered_kernel_body, make_test_ctx}; +#[test] +fn test_frexp_out_parameter_lowering_on_both_backends() -> Result<(), anyhow::Error> { + use dialect_mir::rust_intrinsics as ri; + use dialect_mir::types::MirTupleType; + use pliron::basic_block::BasicBlock; + use pliron::builtin::attributes::StringAttr; + use pliron::builtin::types::{FP32Type, FP64Type, IntegerType, Signedness}; + use pliron::r#type::{TypeHandle, Typed}; + + for backend in [ + mir_lower::IntrinsicBackend::LlvmNvptx, + mir_lower::IntrinsicBackend::LibNvvm, + ] { + for wide in [false, true] { + // The reordered case also carries real padding. Tuple packing + // must use the declared-field map rather than hardcoded slots. + for reordered in [false, true] { + let mut ctx = make_test_ctx(); + let float_ty: TypeHandle = if wide { + FP64Type::get(&ctx).into() + } else { + FP32Type::get(&ctx).into() + }; + let i32_mir: TypeHandle = IntegerType::get(&ctx, 32, Signedness::Signed).into(); + let tuple_ty: TypeHandle = if reordered { + MirTupleType::get_with_layout( + &mut ctx, + vec![float_ty, i32_mir], + vec![1, 0], + vec![if wide { 8 } else { 4 }, 0], + if wide { 16 } else { 8 }, + if wide { 8 } else { 4 }, + ) + .into() + } else { + MirTupleType::get(&mut ctx, vec![float_ty, i32_mir]).into() + }; + let (module_ptr, entry) = build_test_kernel(&mut ctx, vec![float_ty]); + let x = entry.deref(&ctx).get_argument(0); + let parent = entry.deref(&ctx).get_parent_op(&ctx).unwrap(); + let region = parent.deref(&ctx).get_region(0); + let body = BasicBlock::new(&mut ctx, None, vec![]); + body.insert_at_back(region, &ctx); + Operation::new( + &mut ctx, + mir::MirGotoOp::get_concrete_op_info(), + vec![], + vec![], + vec![body], + 0, + ) + .insert_at_back(entry, &ctx); + let placeholder = if wide { + ri::CALLEE_FREXP_F64 + } else { + ri::CALLEE_FREXP_F32 + }; + let call = Operation::new( + &mut ctx, + mir::MirCallOp::get_concrete_op_info(), + vec![tuple_ty], + vec![x], + vec![], + 0, + ); + mir::MirCallOp::new(call) + .set_attr_callee(&ctx, StringAttr::new(placeholder.to_string())); + call.insert_at_back(body, &ctx); + append_return(&mut ctx, body); + mir_lower::lower_mir_to_llvm_with_options( + &mut ctx, + module_ptr, + mir_lower::LoweringOptions { + intrinsic_backend: backend, + ..Default::default() + }, + ) + .map_err(|error| anyhow::anyhow!("{error}"))?; + pliron::operation::verify_operation(module_ptr, &ctx) + .map_err(|error| anyhow::anyhow!("{error}"))?; + let module_block = module_ptr + .deref(&ctx) + .get_region(0) + .deref(&ctx) + .iter(&ctx) + .next() + .unwrap(); + let func = module_block + .deref(&ctx) + .iter(&ctx) + .find_map(|op| { + let func = Operation::get_op::(op, &ctx)?; + (func.get_symbol_name(&ctx).to_string() == "kernel_func").then_some(func) + }) + .unwrap(); + let blocks = func + .get_operation() + .deref(&ctx) + .get_region(0) + .deref(&ctx) + .iter(&ctx) + .collect::>(); + let operations = blocks + .iter() + .flat_map(|block| block.deref(&ctx).iter(&ctx)) + .collect::>(); + let allocas = operations + .iter() + .filter(|&&op| Operation::get_op::(op, &ctx).is_some()) + .copied() + .collect::>(); + assert_eq!(allocas.len(), 1); + assert_eq!( + allocas[0].deref(&ctx).get_parent_block(), + Some(blocks[0]), + "out slot must be static even for a call outside the entry block" + ); + let calls = operations + .iter() + .filter_map(|&op| Operation::get_op::(op, &ctx)) + .collect::>(); + assert_eq!(calls.len(), 1); + let call = &calls[0]; + let CallOpCallable::Direct(symbol) = call.callee(&ctx) else { + panic!("expected a direct libdevice call") + }; + assert_eq!( + symbol.to_string(), + if wide { "__nv_frexp" } else { "__nv_frexpf" } + ); + let call_op = call.get_operation().deref(&ctx); + assert_eq!(call_op.get_num_operands(), 2); + assert_eq!(call_op.get_result(0).get_type(&ctx), float_ty); + let out_ptr = call_op.get_operand(1); + assert_eq!(out_ptr, allocas[0].deref(&ctx).get_result(0)); + assert!( + operations + .iter() + .any(|&op| Operation::get_op::(op, &ctx).is_some() + && op.deref(&ctx).get_operand(1) == out_ptr) + ); + assert!( + operations + .iter() + .any(|&op| Operation::get_op::(op, &ctx).is_some() + && op.deref(&ctx).get_operand(0) == out_ptr) + ); + assert_eq!( + operations + .iter() + .filter(|&&op| Operation::get_op::(op, &ctx).is_some()) + .count(), + 2, + "non-finite fraction and exponent must both be normalized" + ); + let module = Operation::get_op::(module_ptr, &ctx).unwrap(); + let ir = llvm_export::export::export_module_to_string(&ctx, &module) + .expect("frexp module must export"); + assert!( + ir.contains(if wide { + "declare double @__nv_frexp(double, ptr)" + } else { + "declare float @__nv_frexpf(float, ptr)" + }), + "{ir}" + ); + assert!( + ir.contains(if wide { "and i64" } else { "and i32" }), + "{ir}" + ); + assert!(!ir.contains(ri::PLACEHOLDER_PREFIX), "{ir}"); + } + } + } + Ok(()) +} + +#[test] +fn test_frexp_rejects_malformed_calls() { + use dialect_mir::rust_intrinsics as ri; + use dialect_mir::types::MirTupleType; + use pliron::builtin::attributes::StringAttr; + use pliron::builtin::types::{FP32Type, FP64Type, IntegerType, Signedness}; + use pliron::r#type::TypeHandle; + + for backend in [ + mir_lower::IntrinsicBackend::LlvmNvptx, + mir_lower::IntrinsicBackend::LibNvvm, + ] { + for wide in [false, true] { + for invalid in 0..7 { + let mut ctx = make_test_ctx(); + let float_ty: TypeHandle = if wide { + FP64Type::get(&ctx).into() + } else { + FP32Type::get(&ctx).into() + }; + let other_ty: TypeHandle = if wide { + FP32Type::get(&ctx).into() + } else { + FP64Type::get(&ctx).into() + }; + let i32_ty: TypeHandle = IntegerType::get(&ctx, 32, Signedness::Signed).into(); + let i64_ty: TypeHandle = IntegerType::get(&ctx, 64, Signedness::Signed).into(); + let (module_ptr, entry) = build_test_kernel(&mut ctx, vec![float_ty, other_ty]); + let x = entry.deref(&ctx).get_argument(0); + let other = entry.deref(&ctx).get_argument(1); + let fields = match invalid { + 3 => vec![float_ty, i64_ty], + 4 => vec![other_ty, i32_ty], + 5 => vec![float_ty], + _ => vec![float_ty, i32_ty], + }; + let tuple: TypeHandle = MirTupleType::get(&mut ctx, fields).into(); + let result = if invalid == 2 { float_ty } else { tuple }; + let args = match invalid { + 0 => vec![], + 1 => vec![x, x], + 6 => vec![other], + _ => vec![x], + }; + let placeholder = if wide { + ri::CALLEE_FREXP_F64 + } else { + ri::CALLEE_FREXP_F32 + }; + let call = Operation::new( + &mut ctx, + mir::MirCallOp::get_concrete_op_info(), + vec![result], + args, + vec![], + 0, + ); + mir::MirCallOp::new(call) + .set_attr_callee(&ctx, StringAttr::new(placeholder.to_string())); + call.insert_at_back(entry, &ctx); + append_return(&mut ctx, entry); + let error = mir_lower::lower_mir_to_llvm_with_options( + &mut ctx, + module_ptr, + mir_lower::LoweringOptions { + intrinsic_backend: backend, + ..Default::default() + }, + ) + .expect_err("malformed frexp ABI must fail") + .to_string(); + let expected = match invalid { + 0 | 1 => "requires 1 operand(s)", + 2 => "requires a (float, i32) result", + _ => "signature mismatch", + }; + assert!(error.contains(expected), "case {invalid}: {error}"); + } + } + } +} + #[test] fn test_special_math_mixed_signatures_on_both_backends() -> Result<(), anyhow::Error> { use dialect_mir::rust_intrinsics as ri; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/math_special/README.md b/cuda-oxide/crates/rustc-codegen-cuda/examples/math_special/README.md index a21149f0ab..9fe14eaf72 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/math_special/README.md +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/math_special/README.md @@ -1,42 +1,49 @@ -# Special scalar math - -Exercises the first scalar additions for issue #1224: - -| Rust call | f32 libdevice symbol | f64 libdevice symbol | -| --- | --- | --- | -| `libm::tgamma{f}` and `f32/f64::gamma` | `__nv_tgammaf` | `__nv_tgamma` | -| `libm::ilogb{f}` | `__nv_ilogbf` | `__nv_ilogb` | -| `libm::ldexp{f}`, `libm::scalbn{f}` | `__nv_ldexpf` | `__nv_ldexp` | - -`ilogb` returns an `i32`; the exponent argument of `ldexp`/`scalbn` is -an `i32`. `float_gamma` is enabled to exercise the standard-library cmath -shim as well as the `libm` entry points. Use the repository's pinned toolchain -and its existing `RUSTC_BOOTSTRAP` configuration. - -Before these mappings are added, the standard gamma path is rejected by the -collector and the libm paths have no dedicated libdevice lowering. Capture the -baseline diagnostic on the unmodified tree; no local baseline/GPU run is claimed -by this source package. - -The example checks 264 results: 22 inputs, six checks per input, and two widths. -It compares gamma to host libm within a relative tolerance, compares integer -logarithms exactly, and compares scaled values bit for bit. The inputs include -negative non-integers, both exponent signs, subnormals, scaling overflow and -underflow. A user function with a libm-like module name must retain its own -body. `ilogb` inputs are finite and nonzero; its domain-error sentinels are -outside this example's comparison. - -The lowering tests independently verify all six libdevice signatures on both -backends, reject malformed calls, and retain `powi` as a mixed-signature control. -Importer tests check public/internal libm spellings, aliases, cmath gamma, -unrelated names, and functions deferred to later PRs. +# Special math + +The scalar tests cover `tgamma`, standard-library `gamma()`, `ilogb`, and +`ldexp`/`scalbn` for f32/f64. The frexp tests additionally exercise +`libm::frexpf(f32) -> (f32, i32)` and `libm::frexp(f64) -> (f64, i32)`. + +Libdevice returns a scalar fraction and writes an exponent through a pointer. +The lowering adapts this ABI to Rust's tuple using a function-local slot, +a libdevice call, a load and tuple construction. The slot lives in the function's +entry block so loop iterations reuse static storage. Tuple fields use the +existing layout map, including reordered fields and padding. + +For infinities and NaNs, libm returns `(x, 0)`, whereas libdevice leaves the +exponent unspecified. The adapter initializes the exponent slot and explicitly +selects the original input and zero exponent for non-finite inputs. Integer +bit tests preserve NaN payloads and do not introduce floating-point comparisons. + +References: + +- [libm frexp implementation](https://github.com/rust-lang/libm/blob/master/libm/src/math/frexp.rs) +- [libm frexpf implementation](https://github.com/rust-lang/libm/blob/master/libm/src/math/frexpf.rs) +- [NVIDIA libdevice frexp contract](https://docs.nvidia.com/cuda/libdevice-users-guide/libdevice_users_guide/function_device__functions__decls_8h_1addcff9bac6691e5d398a2f8e684ae441.html) + +## Coverage + +The example retains the 264 scalar checks and adds 512 frexp checks, for an +expected total of 776. Each width has 32 frexp inputs: finite values, signed +zeros, normal limits, subnormal limits, infinities and positive/negative quiet +and signaling NaNs with payloads. Four routes check both tuple components: + +- A device-function boundary returning the tuple. +- Assignment into a nested tuple field. +- A call inside a loop. +- An unrelated user function with a libm-like name, which must retain its body. + +The host compares both frexp components exactly, including fraction bits. +Lowering tests cover both intrinsic backends, an out-of-entry-block call, +reordered tuple layouts, and rejected arities and types. Importer tests cover +public/internal libm spellings and unrelated names. ## Validation -Run from `cuda-oxide/` after copying the files into the existing checkout: +Run from `cuda-oxide/` using the pinned toolchain: ```bash cargo oxide fmt @@ -45,26 +52,19 @@ git diff --check cargo test -p dialect-mir -p mir-importer -p mir-lower --all-targets cargo clippy -p dialect-mir -p mir-importer -p mir-lower --all-targets -- -D warnings (cd crates/rustc-codegen-cuda && cargo test --lib) -(cd crates/rustc-codegen-cuda && cargo clippy --all-targets -- -D warnings) (cd crates/rustc-codegen-cuda/examples/math_special && cargo clippy --locked --all-targets -- -D warnings) -cargo oxide build math_special cargo oxide run math_special cargo oxide run libm_math cargo oxide run math_hyperbolic -bash scripts/check-example-smoketest-contract.sh -bash scripts/sync-example-locks.sh --check +just check ``` -`math_special` must print `SUCCESS: math_special passed 264 checks across f32 and f64`. -GPU commands require a working CUDA toolkit, driver and device. The example is -a separate workspace; the parent workspace's tests do not build it. +Expected output from the extended example: -Before marking the PR ready, run the repository gate from the repository root: - -```bash -just -f cuda-oxide/Justfile check +```text +SUCCESS: math_special passed 776 checks across f32 and f64 ``` -This package has undergone static source and archive checks only. Rust, Cargo, -and CUDA were unavailable in the preparation environment; compile, clippy and -GPU validation are pending execution on a configured checkout. +The tuple-returning helpers are marked `#[inline(never)]` to exercise the +device-function ABI in the optimized run, together with libdevice linking. +`cargo oxide run` uses the release profile and does not accept `--profile`. diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/math_special/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/math_special/src/main.rs index 89c688e14f..867691e554 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/math_special/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/math_special/src/main.rs @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -//! tgamma, ilogb, ldexp and scalbn in f32/f64 device code. +//! Scalar math and tuple-returning frexp in f32/f64 device code. //! Run with `cargo oxide run math_special`. #![feature(float_gamma)] // The width-generic host macro casts both f32 and f64 results to f64. @@ -24,6 +24,29 @@ mod libm_lookalike { pub fn tgamma(x: f64) -> f64 { x + 1000.0 } + + #[inline(never)] + pub fn frexpf(x: f32) -> (f32, i32) { + (x, 12345) + } + + #[inline(never)] + pub fn frexp(x: f64) -> (f64, i32) { + (x, 12345) + } +} + +// Exercise an ordinary device-function boundary with a tuple return. +mod frexp_helpers { + #[inline(never)] + pub fn split_f32(x: f32) -> (f32, i32) { + libm::frexpf(x) + } + + #[inline(never)] + pub fn split_f64(x: f64) -> (f64, i32) { + libm::frexp(x) + } } #[cuda_module] @@ -99,6 +122,108 @@ mod kernels { } } } + + #[kernel] + pub fn frexp_f32( + xs: &[f32], + mut direct_fraction: DisjointSlice, + mut direct_exponent: DisjointSlice, + mut projected_fraction: DisjointSlice, + mut projected_exponent: DisjointSlice, + mut loop_fraction: DisjointSlice, + mut loop_exponent: DisjointSlice, + mut control_fraction: DisjointSlice, + mut control_exponent: DisjointSlice, + ) { + let i = thread::index_1d().get(); + if i < xs.len() { + let x = xs[i]; + let direct = frexp_helpers::split_f32(x); + let mut projected = (17_u32, (x, 0_i32)); + projected.1 = libm::frexpf(projected.1.0); + let mut repeated = (0.0, 0_i32); + for step in 0..3 { + let input = if step == 1 { -x } else { x }; + repeated = libm::frexpf(input); + } + let control = libm_lookalike::frexpf(x); + if let Some(out) = direct_fraction.get_mut(thread::index_1d()) { + *out = direct.0; + } + if let Some(out) = direct_exponent.get_mut(thread::index_1d()) { + *out = direct.1; + } + if let Some(out) = projected_fraction.get_mut(thread::index_1d()) { + *out = projected.1.0; + } + if let Some(out) = projected_exponent.get_mut(thread::index_1d()) { + *out = projected.1.1; + } + if let Some(out) = loop_fraction.get_mut(thread::index_1d()) { + *out = repeated.0; + } + if let Some(out) = loop_exponent.get_mut(thread::index_1d()) { + *out = repeated.1; + } + if let Some(out) = control_fraction.get_mut(thread::index_1d()) { + *out = control.0; + } + if let Some(out) = control_exponent.get_mut(thread::index_1d()) { + *out = control.1; + } + } + } + + #[kernel] + pub fn frexp_f64( + xs: &[f64], + mut direct_fraction: DisjointSlice, + mut direct_exponent: DisjointSlice, + mut projected_fraction: DisjointSlice, + mut projected_exponent: DisjointSlice, + mut loop_fraction: DisjointSlice, + mut loop_exponent: DisjointSlice, + mut control_fraction: DisjointSlice, + mut control_exponent: DisjointSlice, + ) { + let i = thread::index_1d().get(); + if i < xs.len() { + let x = xs[i]; + let direct = frexp_helpers::split_f64(x); + let mut projected = (17_u32, (x, 0_i32)); + projected.1 = libm::frexp(projected.1.0); + let mut repeated = (0.0, 0_i32); + for step in 0..3 { + let input = if step == 1 { -x } else { x }; + repeated = libm::frexp(input); + } + let control = libm_lookalike::frexp(x); + if let Some(out) = direct_fraction.get_mut(thread::index_1d()) { + *out = direct.0; + } + if let Some(out) = direct_exponent.get_mut(thread::index_1d()) { + *out = direct.1; + } + if let Some(out) = projected_fraction.get_mut(thread::index_1d()) { + *out = projected.1.0; + } + if let Some(out) = projected_exponent.get_mut(thread::index_1d()) { + *out = projected.1.1; + } + if let Some(out) = loop_fraction.get_mut(thread::index_1d()) { + *out = repeated.0; + } + if let Some(out) = loop_exponent.get_mut(thread::index_1d()) { + *out = repeated.1; + } + if let Some(out) = control_fraction.get_mut(thread::index_1d()) { + *out = control.0; + } + if let Some(out) = control_exponent.get_mut(thread::index_1d()) { + *out = control.1; + } + } + } } fn close(got: f64, expected: f64, rtol: f64) -> bool { @@ -225,6 +350,124 @@ fn main() -> Result<(), Box> { (-1.0, 1024), ] ); + macro_rules! check_frexp_width { + ($ty:ty, $kernel:ident, $frexp:ident, $extra:expr) => {{ + let mut xs: Vec<$ty> = vec![ + 0.0, -0.0, 0.5, -0.5, 1.0, -1.0, 1.5, -1.5, 2.0, -2.0, 3.25, -3.25, 0.75, -0.75, + 42.0, -42.0, 0.125, -0.125, + ]; + xs.extend($extra); + let n = xs.len(); + let inputs = DeviceBuffer::from_host(&stream, &xs)?; + let mut direct_fraction = DeviceBuffer::<$ty>::zeroed(&stream, n)?; + let mut direct_exponent = DeviceBuffer::::zeroed(&stream, n)?; + let mut projected_fraction = DeviceBuffer::<$ty>::zeroed(&stream, n)?; + let mut projected_exponent = DeviceBuffer::::zeroed(&stream, n)?; + let mut loop_fraction = DeviceBuffer::<$ty>::zeroed(&stream, n)?; + let mut loop_exponent = DeviceBuffer::::zeroed(&stream, n)?; + let mut control_fraction = DeviceBuffer::<$ty>::zeroed(&stream, n)?; + let mut control_exponent = DeviceBuffer::::zeroed(&stream, n)?; + // SAFETY: every buffer contains n elements and each output is + // written only by its matching thread, with an input bounds check. + unsafe { + module.$kernel( + &stream, + LaunchConfig::for_num_elems(n as u32), + &inputs, + &mut direct_fraction, + &mut direct_exponent, + &mut projected_fraction, + &mut projected_exponent, + &mut loop_fraction, + &mut loop_exponent, + &mut control_fraction, + &mut control_exponent, + )?; + } + let fractions = [ + direct_fraction.to_host_vec(&stream)?, + projected_fraction.to_host_vec(&stream)?, + loop_fraction.to_host_vec(&stream)?, + control_fraction.to_host_vec(&stream)?, + ]; + let exponents = [ + direct_exponent.to_host_vec(&stream)?, + projected_exponent.to_host_vec(&stream)?, + loop_exponent.to_host_vec(&stream)?, + control_exponent.to_host_vec(&stream)?, + ]; + for (i, &x) in xs.iter().enumerate() { + let expected = libm::$frexp(x); + for (route, name) in ["tuple return", "projected tuple", "loop", "user control"] + .iter() + .enumerate() + { + let (fraction, exponent) = if route == 3 { (x, 12345) } else { expected }; + checks += 2; + if fractions[route][i].to_bits() != fraction.to_bits() { + failures += 1; + eprintln!( + "{} frexp {name}[{i}] fraction: got={:#x} expected={:#x}", + stringify!($ty), + fractions[route][i].to_bits(), + fraction.to_bits() + ); + } + if exponents[route][i] != exponent { + failures += 1; + eprintln!( + "{} frexp {name}[{i}] exponent: got={} expected={exponent}", + stringify!($ty), + exponents[route][i] + ); + } + } + } + }}; + } + check_frexp_width!( + f32, + frexp_f32, + frexpf, + [ + f32::MIN_POSITIVE, + -f32::MIN_POSITIVE, + f32::MAX, + -f32::MAX, + f32::from_bits(1), + f32::from_bits(0x8000_0001), + f32::from_bits(0x007f_ffff), + f32::from_bits(0x807f_ffff), + f32::INFINITY, + f32::NEG_INFINITY, + f32::from_bits(0x7fc1_2345), + f32::from_bits(0xffc1_2345), + f32::from_bits(0x7f81_2345), + f32::from_bits(0xff81_2345), + ] + ); + check_frexp_width!( + f64, + frexp_f64, + frexp, + [ + f64::MIN_POSITIVE, + -f64::MIN_POSITIVE, + f64::MAX, + -f64::MAX, + f64::from_bits(1), + f64::from_bits(0x8000_0000_0000_0001), + f64::from_bits(0x000f_ffff_ffff_ffff), + f64::from_bits(0x800f_ffff_ffff_ffff), + f64::INFINITY, + f64::NEG_INFINITY, + f64::from_bits(0x7ff8_0000_0001_2345), + f64::from_bits(0xfff8_0000_0001_2345), + f64::from_bits(0x7ff0_0000_0001_2345), + f64::from_bits(0xfff0_0000_0001_2345), + ] + ); + if failures != 0 { return Err(format!("FAILED: {failures} of {checks} checks failed").into()); } From 080f5d66267a2bb159cf8f513da69acd2f38ea31 Mon Sep 17 00:00:00 2001 From: Raul Estrada Date: Sun, 25 Oct 2026 22:25:23 -0600 Subject: [PATCH 4/4] refactor: generate compiler libdevice math dispatch Signed-off-by: Raul Estrada --- .../crates/cuda-intrinsics-gen/README.md | 114 +- .../cuda-intrinsics-gen/src/generate.rs | 1 + .../src/libdevice_math/compiler.rs | 179 +++ .../libdevice_math/migration-baseline.json | 216 ++++ .../src/libdevice_math/mod.rs | 247 +++- .../src/libdevice_math/render.rs | 18 +- .../src/libdevice_math/tests.rs | 321 +++-- .../src/generated/libdevice_math.rs | 1103 +++++++++++++++++ .../crates/dialect-mir/src/rust_intrinsics.rs | 270 +--- .../terminator/intrinsics/float_math.rs | 635 +--------- .../src/translator/terminator/mod.rs | 9 +- .../crates/mir-lower/src/convert/ops/call.rs | 426 +------ .../rustc-codegen-cuda/src/collector.rs | 48 +- .../evidence/libdevice_math/README.md | 5 +- .../evidence/libdevice_math/cuda-13.0.json | 10 + .../intrinsics/generated/libdevice_math.rs | 695 +++++++++-- cuda-oxide/intrinsics/libdevice-math.md | 22 +- .../intrinsics/overlay/libdevice_math.toml | 68 +- 18 files changed, 2871 insertions(+), 1516 deletions(-) create mode 100644 cuda-oxide/crates/cuda-intrinsics-gen/src/libdevice_math/compiler.rs create mode 100644 cuda-oxide/crates/cuda-intrinsics-gen/src/libdevice_math/migration-baseline.json create mode 100644 cuda-oxide/crates/dialect-mir/src/generated/libdevice_math.rs diff --git a/cuda-oxide/crates/cuda-intrinsics-gen/README.md b/cuda-oxide/crates/cuda-intrinsics-gen/README.md index 02524a79df..1857ab9a86 100644 --- a/cuda-oxide/crates/cuda-intrinsics-gen/README.md +++ b/cuda-oxide/crates/cuda-intrinsics-gen/README.md @@ -2,9 +2,10 @@ Extractor and deterministic source generator for cuda-oxide's intrinsics. -It reads `intrinsics/catalog.json` and writes every generated file in the -intrinsic pipeline. `MANAGED_GENERATED_ROOTS` in `src/generate.rs` lists the -ten directories it owns, where `generate` writes outputs and removes stale +It reads `intrinsics/catalog.json` and the libdevice math shard, and writes every +generated file in the intrinsic pipeline. `MANAGED_GENERATED_ROOTS` in +`src/generate.rs` lists the +eleven directories it owns, where `generate` writes outputs and removes stale marker-bearing files: - `cuda-oxide/crates/cuda-intrinsics/src/generated/`: the raw intrinsic declarations; @@ -17,11 +18,12 @@ marker-bearing files: requirements), `cuda-oxide/crates/mir-lower/src/convert` (conversion interfaces), and `cuda-oxide/crates/rustc-codegen-cuda/src` (collector predicates); - `intrinsics/probes/`: the per-intrinsic PTX probe evidence. -- `intrinsics/generated/`: staged libm-to-libdevice mapping metadata. +- `intrinsics/generated/`: libm-to-libdevice mapping metadata; +- `crates/dialect-mir/src/generated/`: shared compiler math dispatch. Generated files carry a `// @generated by cuda-intrinsics-gen ... DO NOT EDIT.` banner; edit the -catalog and regenerate instead. +catalog or shard and regenerate instead. ``` cuda-intrinsics-gen extract --intrinsics-json FILE --nvptx-json FILE @@ -44,55 +46,81 @@ by each route, and Generation shells out to `rustfmt`, which the pinned toolchain provides. -## Staged libdevice math family - -`intrinsics/overlay/libdevice_math.toml` records the 72 existing f32/f64 libm -exports, including aliases and handwritten `fmin`/`fmax` and `sincos` routes. -The renderer emits `intrinsics/generated/libdevice_math.rs` and -`intrinsics/libdevice-math.md`. The Rust metadata is compiled by generator tests; -compiler dispatch and its handwritten tables are migrated in a subsequent PR. - -This input is read explicitly by the libdevice family, separately from the -hardware shard index in `intrinsics/overlay.toml`. Its evidence comes from -`libdevice.10.bc`, so it does not receive hardware ABI IDs or change the hardware -catalog hash. The overlay reconciliation guard recognizes this separate reader; -unlisted hardware shards remain errors. Normal `generate` and `check` process -both domains, and stale metadata under `intrinsics/generated/` is reconciled. - -Rows name a Rust signature class, optional libm aliases and cmath spelling, -placeholder suffixes, libdevice symbol, optional native LLVM alternative, -link classification, and optional `handwritten:NAME` handler. `{width}` expands -to `f32`/`f64`; `{suffix}` expands to `f`/the empty string. Unknown fields, -duplicate names, invalid templates, ambiguous ABI signatures, and incompatible -link classifications fail before outputs are written. Tuple signatures require -an explicit handwritten handler. - -Generation checks every referenced C symbol and signature against the pinned -capture in `intrinsics/evidence/libdevice_math/cuda-13.0.json`. Capture provenance -and reproduction instructions are recorded beside it. The cache lets ordinary -`generate` and `check` run without a CUDA installation. - -To verify the same ABI against a locally installed toolkit, use the pinned -toolchain's `llvm-tools` component: +## Libdevice math family + +`intrinsics/overlay/libdevice_math.toml` is the schema-2 source of compiler math +dispatch. It preserves all 82 existing f32/f64 libm exports, including the +hand-landed tgamma, ilogb, ldexp/scalbn, and frexp routes, plus the core-only +powi and fmuladd intrinsics. This migration adds no new math exports. + +The renderer writes three outputs: + +- `crates/dialect-mir/src/generated/libdevice_math.rs`: the shared math enum, + placeholder constants, compiler-path and alias dispatch, operand counts, + libdevice/native LLVM symbols, exact cmath interception, and link requirements; +- `intrinsics/generated/libdevice_math.rs`: mapping metadata used by generator + tests; +- `intrinsics/libdevice-math.md`: the mapping reference. + +The importer, MIR lowerer, and rustc collector consume the shared generated +dispatch instead of maintaining their own copies. Type-aware generic core +intrinsics and the semantic expansions for frexp, sincos, min/max, and fast +arithmetic remain handwritten. For example, before and after regeneration, +`libm::math::scalbn::scalbnf` resolves to `LdexpF32`, emits the same ldexp +placeholder, and lowers to `__nv_ldexpf` with an i32 exponent. + +`generate` owns and reconciles all three outputs; `check` rejects drift in any +of them, including the compiler dispatch. A regression fixture records the 214 +scalar compiler paths before migration and compares them against the compiled +consumer's identities, placeholders, symbols, and link classification. Separate +controls cover sincos, generic fabs, native LLVM routes, and unknown names. + +This input is read separately from the hardware shard index in +`intrinsics/overlay.toml`. Its evidence comes from `libdevice.10.bc`, so it does +not receive hardware ABI IDs or change the hardware catalog hash. Ordinary +unindexed hardware shards remain errors. + +Rows name a Rust signature class, optional libm aliases, core intrinsic names, +compatibility paths, cmath spelling, placeholder suffixes, libdevice symbol, +native LLVM alternative, link classification, and handwritten handler. By +default, a row exports its name from libm; `libm = []` explicitly describes a +core-only operation. `{width}` expands to `f32`/`f64`, `{Width}` and `{WIDTH}` to +`F32`/`F64`, and `{suffix}` to `f`/the empty string. Variant and constant overrides +preserve existing compiler identities such as `PowfF32` and the generic `Fabs`. + +Unknown fields, duplicate paths/identities/placeholders, invalid templates, +ambiguous C ABIs, incompatible link classifications, and unsupported handwritten +handlers fail before outputs are written. Tuple signatures require a supported +handwritten adapter. + +Generation checks all 74 referenced C symbols and signatures against the pinned +capture in `intrinsics/evidence/libdevice_math/cuda-13.0.json`. Its provenance +and reproduction instructions are recorded beside it. Ordinary `generate` and +`check` do not need a CUDA installation. + +To verify the same ABI against a locally installed toolkit: ```bash libdevice_llvm_dis="$(rustc --print sysroot)/lib/rustlib/$(rustc -vV | sed -n 's/^host: //p')/bin/llvm-dis" -cargo run -p cuda-intrinsics-gen -- verify-libdevice \ +cargo run -p cuda-intrinsics-gen --locked -- verify-libdevice \ --libdevice "${CUDA_PATH:-/usr/local/cuda}/nvvm/libdevice/libdevice.10.bc" \ --llvm-dis "$libdevice_llvm_dis" ``` Verification accepts compatible newer toolkit bitcode and reports its hash. -To refresh evidence, first review and update the bitcode hash pin in the shard, -then run `extract-libdevice` with the same arguments, followed by `generate` and -`check`. Extraction hashes and disassembles the same bytes, requires exported -definitions with exact return/argument types, and writes the capture only after -all referenced signatures pass. +To refresh evidence, review and update the shard's bitcode hash pin, then run +`extract-libdevice` with the same arguments, followed by `generate` and `check`. +Extraction hashes and disassembles the same bytes, requires exported definitions +with exact return/argument types, and writes only after all referenced signatures +pass. Opaque pointers do not encode their pointee type; out-parameter semantics +remain the handwritten adapter's responsibility. Validation from `cuda-oxide/`: ```bash -cargo test -p cuda-intrinsics-gen --locked --all-targets -cargo clippy -p cuda-intrinsics-gen --locked --all-targets -- -D warnings +cargo test -p cuda-intrinsics-gen -p dialect-mir -p mir-importer -p mir-lower --locked --all-targets +cargo clippy -p cuda-intrinsics-gen -p dialect-mir -p mir-importer -p mir-lower --locked --all-targets -- -D warnings cargo run -p cuda-intrinsics-gen --locked -- check +cargo test --manifest-path crates/rustc-codegen-cuda/Cargo.toml --locked --lib +just check ``` diff --git a/cuda-oxide/crates/cuda-intrinsics-gen/src/generate.rs b/cuda-oxide/crates/cuda-intrinsics-gen/src/generate.rs index 4ecfab910d..8029d4c758 100644 --- a/cuda-oxide/crates/cuda-intrinsics-gen/src/generate.rs +++ b/cuda-oxide/crates/cuda-intrinsics-gen/src/generate.rs @@ -23,6 +23,7 @@ const MANAGED_GENERATED_ROOTS: &[&str] = &[ "crates/rustc-codegen-cuda/src", "intrinsics/probes", "intrinsics/generated", + "crates/dialect-mir/src/generated", ]; const GENERATED_MARKER: &[u8] = b"@generated by cuda-intrinsics-gen"; diff --git a/cuda-oxide/crates/cuda-intrinsics-gen/src/libdevice_math/compiler.rs b/cuda-oxide/crates/cuda-intrinsics-gen/src/libdevice_math/compiler.rs new file mode 100644 index 0000000000..c4ffcc0d4b --- /dev/null +++ b/cuda-oxide/crates/cuda-intrinsics-gen/src/libdevice_math/compiler.rs @@ -0,0 +1,179 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! Render the shared dispatch used by the importer, lowerer, and collector. + +use super::{Link, Mapping}; +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt::Write; + +// These polymorphic arithmetic operations have handwritten semantic lowerings, +// no libm aliases, and no libdevice ABI. They are controls, not shard rows. +pub(super) const FAST: &[(&str, &str, &str)] = &[ + ("FaddFast", "CALLEE_FADD_FAST", "fadd_fast"), + ("FsubFast", "CALLEE_FSUB_FAST", "fsub_fast"), + ("FmulFast", "CALLEE_FMUL_FAST", "fmul_fast"), + ("FdivFast", "CALLEE_FDIV_FAST", "fdiv_fast"), + ("FremFast", "CALLEE_FREM_FAST", "frem_fast"), +]; + +pub(super) fn render(rows: &[Mapping]) -> String { + let mut out = String::from( + "/*\n * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n * SPDX-License-Identifier: Apache-2.0\n */\n\n// @generated by cuda-intrinsics-gen from overlay/libdevice_math.toml. DO NOT EDIT.\n\n", + ); + let mut constants = BTreeMap::new(); + let mut variants: BTreeMap<&str, BTreeMap<&str, &Mapping>> = BTreeMap::new(); + let mut paths = BTreeMap::new(); + let mut exports = BTreeMap::new(); + for row in rows { + let Some(variant) = row.variant.as_deref() else { + continue; + }; + let constant = row.constant.as_deref().unwrap(); + constants.insert(constant, &row.placeholders[0]); + variants.entry(variant).or_default().insert(&row.width, row); + for path in &row.paths { + paths.insert(path, variant); + } + if let Some(libm) = &row.libm { + exports.insert(libm, variant); + } + } + for (constant, suffix) in constants { + writeln!(out, "/// Internal math placeholder generated from the libdevice shard.\npub const {constant}: &str = placeholder!({suffix:?});").unwrap(); + } + out.push_str("\n/// Shared math identity. Semantic expansions remain in the importer and lowerer.\n#[derive(Clone, Copy, Debug, PartialEq, Eq)]\npub enum RustFloatMathIntrinsic {\n"); + for variant in variants + .keys() + .copied() + .chain(FAST.iter().map(|entry| entry.0)) + { + writeln!(out, "{variant},").unwrap(); + } + out.push_str("}\n\n/// Whether a math placeholder needs libdevice under the selected backend.\n#[derive(Clone, Copy, Debug, PartialEq, Eq)]\npub enum MathLink { Always, Libnvvm, Never }\n\nimpl RustFloatMathIntrinsic {\n"); + + out.push_str("/// Recognize exact core, cmath, or libm compiler paths.\npub fn from_core_path(name: &str) -> Option { match name {\n"); + for (path, variant) in paths { + writeln!(out, "{path:?} => Some(Self::{variant}),").unwrap(); + } + for (variant, _, name) in FAST { + writeln!( + out, + "\"core::intrinsics::{name}\" | \"std::intrinsics::{name}\" => Some(Self::{variant})," + ) + .unwrap(); + } + out.push_str("_ => Self::from_libm_path(name),\n} }\n\n/// Resolve aliases only within the libm crate namespace.\npub fn from_libm_path(name: &str) -> Option {\nif !is_libm_path(name) { return None; }\nmatch name.rsplit(\"::\").next().unwrap_or(name) {\n"); + for (export, variant) in exports { + writeln!(out, "{export:?} => Some(Self::{variant}),").unwrap(); + } + out.push_str("_ => None,\n} }\n\n/// Return the placeholder emitted for this math identity.\npub fn placeholder_callee(self) -> &'static str { match self {\n"); + for (variant, widths) in &variants { + writeln!( + out, + "Self::{variant} => {},", + widths.values().next().unwrap().constant.as_deref().unwrap() + ) + .unwrap(); + } + for (variant, constant, _) in FAST { + writeln!(out, "Self::{variant} => {constant},").unwrap(); + } + out.push_str("} }\n\n/// Recover an identity from an exact known placeholder.\npub fn from_placeholder_callee(callee: &str) -> Option { match callee {\n"); + for (variant, widths) in &variants { + writeln!( + out, + "{} => Some(Self::{variant}),", + widths.values().next().unwrap().constant.as_deref().unwrap() + ) + .unwrap(); + } + for (variant, constant, _) in FAST { + writeln!(out, "{constant} => Some(Self::{variant}),").unwrap(); + } + out.push_str("_ => None,\n} }\n\n/// Operand count of the Rust placeholder, before C ABI adaptation.\npub fn arg_count(self) -> usize { match self {\n"); + for (variant, widths) in &variants { + let signature = &widths.values().next().unwrap().signature; + let arity = signature.split(" -> ").next().unwrap().matches(',').count() + 1; + writeln!(out, "Self::{variant} => {arity},").unwrap(); + } + for (variant, _, _) in FAST { + writeln!(out, "Self::{variant} => 2,").unwrap(); + } + out.push_str("} }\n\n/// Classify libdevice linking without accepting unknown placeholder prefixes.\npub fn link_requirement(self) -> MathLink { match self {\n"); + for (variant, widths) in &variants { + let link = match widths.values().next().unwrap().link { + Link::Always => "Always", + Link::Libnvvm => "Libnvvm", + Link::Never => "Never", + }; + writeln!(out, "Self::{variant} => MathLink::{link},").unwrap(); + } + for (variant, _, _) in FAST { + writeln!(out, "Self::{variant} => MathLink::Never,").unwrap(); + } + out.push_str("} }\n"); + for (method, llvm) in [("libdevice_symbol", false), ("llvm_symbol", true)] { + writeln!(out, "\n/// Resolve the symbol; only generic placeholders consult the concrete width.\npub fn {method}(self, is_f64: bool) -> Option<&'static str> {{ match self {{").unwrap(); + for (variant, widths) in &variants { + if widths.len() == 2 { + writeln!( + out, + "Self::{variant} => if is_f64 {{ {:?} }} else {{ {:?} }},", + symbol(widths["f64"], llvm), + symbol(widths["f32"], llvm) + ) + .unwrap(); + } else { + writeln!( + out, + "Self::{variant} => {:?},", + symbol(widths.values().next().unwrap(), llvm) + ) + .unwrap(); + } + } + for (variant, _, _) in FAST { + writeln!(out, "Self::{variant} => None,").unwrap(); + } + out.push_str("} }\n"); + } + out.push_str("}\n\n/// Whether the path belongs to the libm crate, rather than a user namespace.\npub fn is_libm_path(name: &str) -> bool { name.split(\"::\").next() == Some(\"libm\") }\n\n/// Exact std shims intercepted before the collector's forbidden-std guard.\npub fn is_intrinsic_lowered_cmath_shim(name: &str) -> bool { matches!(name,\n"); + let cmath: BTreeSet<_> = rows.iter().filter_map(|row| row.cmath.as_deref()).collect(); + if cmath.is_empty() { + out.push_str("_ if false"); + } else { + out.push_str( + &cmath + .into_iter() + .map(|path| format!("{path:?}")) + .collect::>() + .join(" | "), + ); + } + out.push_str(") }\n\n/// Sincos retains its importer tuple expansion into two scalar placeholders.\npub fn libm_sincos_is_f64(name: &str) -> Option {\nif !is_libm_path(name) { return None; }\nmatch name.rsplit(\"::\").next().unwrap_or(name) {\n"); + for row in rows + .iter() + .filter(|row| row.lowering.as_deref() == Some("handwritten:emit_sincos")) + { + writeln!( + out, + "{:?} => Some({}),", + row.libm.as_deref().unwrap(), + row.width == "f64" + ) + .unwrap(); + } + out.push_str("_ => None,\n} }\n"); + out +} + +fn symbol(row: &Mapping, llvm: bool) -> Option<&str> { + if llvm { + row.llvm.as_deref() + } else { + row.libdevice.as_deref() + } +} diff --git a/cuda-oxide/crates/cuda-intrinsics-gen/src/libdevice_math/migration-baseline.json b/cuda-oxide/crates/cuda-intrinsics-gen/src/libdevice_math/migration-baseline.json new file mode 100644 index 0000000000..3f185b35c5 --- /dev/null +++ b/cuda-oxide/crates/cuda-intrinsics-gen/src/libdevice_math/migration-baseline.json @@ -0,0 +1,216 @@ +[ + {"path":"core::intrinsics::ceilf32","variant":"CeilF32","placeholder":"ceilf32","libdevice":"__nv_ceilf","llvm":"llvm_ceil_f32","link":"libnvvm"}, + {"path":"core::intrinsics::ceilf64","variant":"CeilF64","placeholder":"ceilf64","libdevice":"__nv_ceil","llvm":"llvm_ceil_f64","link":"libnvvm"}, + {"path":"core::intrinsics::copysignf32","variant":"CopysignF32","placeholder":"copysignf32","libdevice":"__nv_copysignf","llvm":"llvm_copysign_f32","link":"libnvvm"}, + {"path":"core::intrinsics::copysignf64","variant":"CopysignF64","placeholder":"copysignf64","libdevice":"__nv_copysign","llvm":"llvm_copysign_f64","link":"libnvvm"}, + {"path":"core::intrinsics::cosf32","variant":"CosF32","placeholder":"cosf32","libdevice":"__nv_cosf","llvm":null,"link":"always"}, + {"path":"core::intrinsics::cosf64","variant":"CosF64","placeholder":"cosf64","libdevice":"__nv_cos","llvm":null,"link":"always"}, + {"path":"core::intrinsics::exp2f32","variant":"Exp2F32","placeholder":"exp2f32","libdevice":"__nv_exp2f","llvm":null,"link":"always"}, + {"path":"core::intrinsics::exp2f64","variant":"Exp2F64","placeholder":"exp2f64","libdevice":"__nv_exp2","llvm":null,"link":"always"}, + {"path":"core::intrinsics::expf32","variant":"ExpF32","placeholder":"expf32","libdevice":"__nv_expf","llvm":null,"link":"always"}, + {"path":"core::intrinsics::expf64","variant":"ExpF64","placeholder":"expf64","libdevice":"__nv_exp","llvm":null,"link":"always"}, + {"path":"core::intrinsics::fabs","variant":"Fabs","placeholder":"fabs","libdevice":"__nv_fabs","llvm":"llvm_fabs_f64","link":"libnvvm"}, + {"path":"core::intrinsics::fadd_fast","variant":"FaddFast","placeholder":"fadd_fast","libdevice":null,"llvm":null,"link":"never"}, + {"path":"core::intrinsics::fdiv_fast","variant":"FdivFast","placeholder":"fdiv_fast","libdevice":null,"llvm":null,"link":"never"}, + {"path":"core::intrinsics::floorf32","variant":"FloorF32","placeholder":"floorf32","libdevice":"__nv_floorf","llvm":"llvm_floor_f32","link":"libnvvm"}, + {"path":"core::intrinsics::floorf64","variant":"FloorF64","placeholder":"floorf64","libdevice":"__nv_floor","llvm":"llvm_floor_f64","link":"libnvvm"}, + {"path":"core::intrinsics::fmaf32","variant":"FmaF32","placeholder":"fmaf32","libdevice":"__nv_fmaf","llvm":null,"link":"always"}, + {"path":"core::intrinsics::fmaf64","variant":"FmaF64","placeholder":"fmaf64","libdevice":"__nv_fma","llvm":null,"link":"always"}, + {"path":"core::intrinsics::fmul_fast","variant":"FmulFast","placeholder":"fmul_fast","libdevice":null,"llvm":null,"link":"never"}, + {"path":"core::intrinsics::fmuladdf32","variant":"FmuladdF32","placeholder":"fmuladdf32","libdevice":"__nv_fmaf","llvm":null,"link":"always"}, + {"path":"core::intrinsics::fmuladdf64","variant":"FmuladdF64","placeholder":"fmuladdf64","libdevice":"__nv_fma","llvm":null,"link":"always"}, + {"path":"core::intrinsics::frem_fast","variant":"FremFast","placeholder":"frem_fast","libdevice":null,"llvm":null,"link":"never"}, + {"path":"core::intrinsics::fsub_fast","variant":"FsubFast","placeholder":"fsub_fast","libdevice":null,"llvm":null,"link":"never"}, + {"path":"core::intrinsics::log10f32","variant":"Log10F32","placeholder":"log10f32","libdevice":"__nv_log10f","llvm":null,"link":"always"}, + {"path":"core::intrinsics::log10f64","variant":"Log10F64","placeholder":"log10f64","libdevice":"__nv_log10","llvm":null,"link":"always"}, + {"path":"core::intrinsics::log2f32","variant":"Log2F32","placeholder":"log2f32","libdevice":"__nv_log2f","llvm":null,"link":"always"}, + {"path":"core::intrinsics::log2f64","variant":"Log2F64","placeholder":"log2f64","libdevice":"__nv_log2","llvm":null,"link":"always"}, + {"path":"core::intrinsics::logf32","variant":"LogF32","placeholder":"logf32","libdevice":"__nv_logf","llvm":null,"link":"always"}, + {"path":"core::intrinsics::logf64","variant":"LogF64","placeholder":"logf64","libdevice":"__nv_log","llvm":null,"link":"always"}, + {"path":"core::intrinsics::maximum_number_nsz_f32","variant":"MaxNumNszF32","placeholder":"maximum_number_nsz_f32","libdevice":null,"llvm":null,"link":"never"}, + {"path":"core::intrinsics::maximum_number_nsz_f64","variant":"MaxNumNszF64","placeholder":"maximum_number_nsz_f64","libdevice":null,"llvm":null,"link":"never"}, + {"path":"core::intrinsics::minimum_number_nsz_f32","variant":"MinNumNszF32","placeholder":"minimum_number_nsz_f32","libdevice":null,"llvm":null,"link":"never"}, + {"path":"core::intrinsics::minimum_number_nsz_f64","variant":"MinNumNszF64","placeholder":"minimum_number_nsz_f64","libdevice":null,"llvm":null,"link":"never"}, + {"path":"core::intrinsics::powf32","variant":"PowfF32","placeholder":"powf32","libdevice":"__nv_powf","llvm":null,"link":"always"}, + {"path":"core::intrinsics::powf64","variant":"PowfF64","placeholder":"powf64","libdevice":"__nv_pow","llvm":null,"link":"always"}, + {"path":"core::intrinsics::powif32","variant":"PowiF32","placeholder":"powif32","libdevice":"__nv_powif","llvm":null,"link":"always"}, + {"path":"core::intrinsics::powif64","variant":"PowiF64","placeholder":"powif64","libdevice":"__nv_powi","llvm":null,"link":"always"}, + {"path":"core::intrinsics::round_ties_even_f32","variant":"RoundevenF32","placeholder":"round_ties_even_f32","libdevice":"__nv_rintf","llvm":"llvm_roundeven_f32","link":"libnvvm"}, + {"path":"core::intrinsics::round_ties_even_f64","variant":"RoundevenF64","placeholder":"round_ties_even_f64","libdevice":"__nv_rint","llvm":"llvm_roundeven_f64","link":"libnvvm"}, + {"path":"core::intrinsics::roundf32","variant":"RoundF32","placeholder":"roundf32","libdevice":"__nv_roundf","llvm":"llvm_round_f32","link":"libnvvm"}, + {"path":"core::intrinsics::roundf64","variant":"RoundF64","placeholder":"roundf64","libdevice":"__nv_round","llvm":"llvm_round_f64","link":"libnvvm"}, + {"path":"core::intrinsics::sinf32","variant":"SinF32","placeholder":"sinf32","libdevice":"__nv_sinf","llvm":null,"link":"always"}, + {"path":"core::intrinsics::sinf64","variant":"SinF64","placeholder":"sinf64","libdevice":"__nv_sin","llvm":null,"link":"always"}, + {"path":"core::intrinsics::sqrtf32","variant":"SqrtF32","placeholder":"sqrtf32","libdevice":"__nv_sqrtf","llvm":null,"link":"always"}, + {"path":"core::intrinsics::sqrtf64","variant":"SqrtF64","placeholder":"sqrtf64","libdevice":"__nv_sqrt","llvm":null,"link":"always"}, + {"path":"core::intrinsics::tanf32","variant":"TanF32","placeholder":"tanf32","libdevice":"__nv_tanf","llvm":null,"link":"always"}, + {"path":"core::intrinsics::tanf64","variant":"TanF64","placeholder":"tanf64","libdevice":"__nv_tan","llvm":null,"link":"always"}, + {"path":"core::intrinsics::truncf32","variant":"TruncF32","placeholder":"truncf32","libdevice":"__nv_truncf","llvm":"llvm_trunc_f32","link":"libnvvm"}, + {"path":"core::intrinsics::truncf64","variant":"TruncF64","placeholder":"truncf64","libdevice":"__nv_trunc","llvm":"llvm_trunc_f64","link":"libnvvm"}, + {"path":"core::num::imp::libm::cbrt","variant":"CbrtF64","placeholder":"cbrtf64","libdevice":"__nv_cbrt","llvm":null,"link":"always"}, + {"path":"core::num::imp::libm::cbrtf","variant":"CbrtF32","placeholder":"cbrtf32","libdevice":"__nv_cbrtf","llvm":null,"link":"always"}, + {"path":"libm::acos","variant":"AcosF64","placeholder":"acosf64","libdevice":"__nv_acos","llvm":null,"link":"always"}, + {"path":"libm::acosf","variant":"AcosF32","placeholder":"acosf32","libdevice":"__nv_acosf","llvm":null,"link":"always"}, + {"path":"libm::acosh","variant":"AcoshF64","placeholder":"acoshf64","libdevice":"__nv_acosh","llvm":null,"link":"always"}, + {"path":"libm::acoshf","variant":"AcoshF32","placeholder":"acoshf32","libdevice":"__nv_acoshf","llvm":null,"link":"always"}, + {"path":"libm::asin","variant":"AsinF64","placeholder":"asinf64","libdevice":"__nv_asin","llvm":null,"link":"always"}, + {"path":"libm::asinf","variant":"AsinF32","placeholder":"asinf32","libdevice":"__nv_asinf","llvm":null,"link":"always"}, + {"path":"libm::asinh","variant":"AsinhF64","placeholder":"asinhf64","libdevice":"__nv_asinh","llvm":null,"link":"always"}, + {"path":"libm::asinhf","variant":"AsinhF32","placeholder":"asinhf32","libdevice":"__nv_asinhf","llvm":null,"link":"always"}, + {"path":"libm::atan","variant":"AtanF64","placeholder":"atanf64","libdevice":"__nv_atan","llvm":null,"link":"always"}, + {"path":"libm::atan2","variant":"Atan2F64","placeholder":"atan2f64","libdevice":"__nv_atan2","llvm":null,"link":"always"}, + {"path":"libm::atan2f","variant":"Atan2F32","placeholder":"atan2f32","libdevice":"__nv_atan2f","llvm":null,"link":"always"}, + {"path":"libm::atanf","variant":"AtanF32","placeholder":"atanf32","libdevice":"__nv_atanf","llvm":null,"link":"always"}, + {"path":"libm::atanh","variant":"AtanhF64","placeholder":"atanhf64","libdevice":"__nv_atanh","llvm":null,"link":"always"}, + {"path":"libm::atanhf","variant":"AtanhF32","placeholder":"atanhf32","libdevice":"__nv_atanhf","llvm":null,"link":"always"}, + {"path":"libm::cbrt","variant":"CbrtF64","placeholder":"cbrtf64","libdevice":"__nv_cbrt","llvm":null,"link":"always"}, + {"path":"libm::cbrtf","variant":"CbrtF32","placeholder":"cbrtf32","libdevice":"__nv_cbrtf","llvm":null,"link":"always"}, + {"path":"libm::ceil","variant":"CeilF64","placeholder":"ceilf64","libdevice":"__nv_ceil","llvm":"llvm_ceil_f64","link":"libnvvm"}, + {"path":"libm::ceilf","variant":"CeilF32","placeholder":"ceilf32","libdevice":"__nv_ceilf","llvm":"llvm_ceil_f32","link":"libnvvm"}, + {"path":"libm::copysign","variant":"CopysignF64","placeholder":"copysignf64","libdevice":"__nv_copysign","llvm":"llvm_copysign_f64","link":"libnvvm"}, + {"path":"libm::copysignf","variant":"CopysignF32","placeholder":"copysignf32","libdevice":"__nv_copysignf","llvm":"llvm_copysign_f32","link":"libnvvm"}, + {"path":"libm::cos","variant":"CosF64","placeholder":"cosf64","libdevice":"__nv_cos","llvm":null,"link":"always"}, + {"path":"libm::cosf","variant":"CosF32","placeholder":"cosf32","libdevice":"__nv_cosf","llvm":null,"link":"always"}, + {"path":"libm::cosh","variant":"CoshF64","placeholder":"coshf64","libdevice":"__nv_cosh","llvm":null,"link":"always"}, + {"path":"libm::coshf","variant":"CoshF32","placeholder":"coshf32","libdevice":"__nv_coshf","llvm":null,"link":"always"}, + {"path":"libm::exp","variant":"ExpF64","placeholder":"expf64","libdevice":"__nv_exp","llvm":null,"link":"always"}, + {"path":"libm::exp2","variant":"Exp2F64","placeholder":"exp2f64","libdevice":"__nv_exp2","llvm":null,"link":"always"}, + {"path":"libm::exp2f","variant":"Exp2F32","placeholder":"exp2f32","libdevice":"__nv_exp2f","llvm":null,"link":"always"}, + {"path":"libm::expf","variant":"ExpF32","placeholder":"expf32","libdevice":"__nv_expf","llvm":null,"link":"always"}, + {"path":"libm::expm1","variant":"Expm1F64","placeholder":"expm1f64","libdevice":"__nv_expm1","llvm":null,"link":"always"}, + {"path":"libm::expm1f","variant":"Expm1F32","placeholder":"expm1f32","libdevice":"__nv_expm1f","llvm":null,"link":"always"}, + {"path":"libm::fabs","variant":"Fabs","placeholder":"fabs","libdevice":"__nv_fabs","llvm":"llvm_fabs_f64","link":"libnvvm"}, + {"path":"libm::fabsf","variant":"Fabs","placeholder":"fabs","libdevice":"__nv_fabsf","llvm":"llvm_fabs_f32","link":"libnvvm"}, + {"path":"libm::floor","variant":"FloorF64","placeholder":"floorf64","libdevice":"__nv_floor","llvm":"llvm_floor_f64","link":"libnvvm"}, + {"path":"libm::floorf","variant":"FloorF32","placeholder":"floorf32","libdevice":"__nv_floorf","llvm":"llvm_floor_f32","link":"libnvvm"}, + {"path":"libm::fma","variant":"FmaF64","placeholder":"fmaf64","libdevice":"__nv_fma","llvm":null,"link":"always"}, + {"path":"libm::fmaf","variant":"FmaF32","placeholder":"fmaf32","libdevice":"__nv_fmaf","llvm":null,"link":"always"}, + {"path":"libm::fmax","variant":"MaxNumNszF64","placeholder":"maximum_number_nsz_f64","libdevice":null,"llvm":null,"link":"never"}, + {"path":"libm::fmaxf","variant":"MaxNumNszF32","placeholder":"maximum_number_nsz_f32","libdevice":null,"llvm":null,"link":"never"}, + {"path":"libm::fmin","variant":"MinNumNszF64","placeholder":"minimum_number_nsz_f64","libdevice":null,"llvm":null,"link":"never"}, + {"path":"libm::fminf","variant":"MinNumNszF32","placeholder":"minimum_number_nsz_f32","libdevice":null,"llvm":null,"link":"never"}, + {"path":"libm::frexp","variant":"FrexpF64","placeholder":"frexpf64","libdevice":"__nv_frexp","llvm":null,"link":"always"}, + {"path":"libm::frexpf","variant":"FrexpF32","placeholder":"frexpf32","libdevice":"__nv_frexpf","llvm":null,"link":"always"}, + {"path":"libm::hypot","variant":"HypotF64","placeholder":"hypotf64","libdevice":"__nv_hypot","llvm":null,"link":"always"}, + {"path":"libm::hypotf","variant":"HypotF32","placeholder":"hypotf32","libdevice":"__nv_hypotf","llvm":null,"link":"always"}, + {"path":"libm::ilogb","variant":"IlogbF64","placeholder":"ilogbf64","libdevice":"__nv_ilogb","llvm":null,"link":"always"}, + {"path":"libm::ilogbf","variant":"IlogbF32","placeholder":"ilogbf32","libdevice":"__nv_ilogbf","llvm":null,"link":"always"}, + {"path":"libm::ldexp","variant":"LdexpF64","placeholder":"ldexpf64","libdevice":"__nv_ldexp","llvm":null,"link":"always"}, + {"path":"libm::ldexpf","variant":"LdexpF32","placeholder":"ldexpf32","libdevice":"__nv_ldexpf","llvm":null,"link":"always"}, + {"path":"libm::log","variant":"LogF64","placeholder":"logf64","libdevice":"__nv_log","llvm":null,"link":"always"}, + {"path":"libm::log10","variant":"Log10F64","placeholder":"log10f64","libdevice":"__nv_log10","llvm":null,"link":"always"}, + {"path":"libm::log10f","variant":"Log10F32","placeholder":"log10f32","libdevice":"__nv_log10f","llvm":null,"link":"always"}, + {"path":"libm::log1p","variant":"Log1pF64","placeholder":"log1pf64","libdevice":"__nv_log1p","llvm":null,"link":"always"}, + {"path":"libm::log1pf","variant":"Log1pF32","placeholder":"log1pf32","libdevice":"__nv_log1pf","llvm":null,"link":"always"}, + {"path":"libm::log2","variant":"Log2F64","placeholder":"log2f64","libdevice":"__nv_log2","llvm":null,"link":"always"}, + {"path":"libm::log2f","variant":"Log2F32","placeholder":"log2f32","libdevice":"__nv_log2f","llvm":null,"link":"always"}, + {"path":"libm::logf","variant":"LogF32","placeholder":"logf32","libdevice":"__nv_logf","llvm":null,"link":"always"}, + {"path":"libm::pow","variant":"PowfF64","placeholder":"powf64","libdevice":"__nv_pow","llvm":null,"link":"always"}, + {"path":"libm::powf","variant":"PowfF32","placeholder":"powf32","libdevice":"__nv_powf","llvm":null,"link":"always"}, + {"path":"libm::rint","variant":"RoundevenF64","placeholder":"round_ties_even_f64","libdevice":"__nv_rint","llvm":"llvm_roundeven_f64","link":"libnvvm"}, + {"path":"libm::rintf","variant":"RoundevenF32","placeholder":"round_ties_even_f32","libdevice":"__nv_rintf","llvm":"llvm_roundeven_f32","link":"libnvvm"}, + {"path":"libm::round","variant":"RoundF64","placeholder":"roundf64","libdevice":"__nv_round","llvm":"llvm_round_f64","link":"libnvvm"}, + {"path":"libm::roundeven","variant":"RoundevenF64","placeholder":"round_ties_even_f64","libdevice":"__nv_rint","llvm":"llvm_roundeven_f64","link":"libnvvm"}, + {"path":"libm::roundevenf","variant":"RoundevenF32","placeholder":"round_ties_even_f32","libdevice":"__nv_rintf","llvm":"llvm_roundeven_f32","link":"libnvvm"}, + {"path":"libm::roundf","variant":"RoundF32","placeholder":"roundf32","libdevice":"__nv_roundf","llvm":"llvm_round_f32","link":"libnvvm"}, + {"path":"libm::scalbn","variant":"LdexpF64","placeholder":"ldexpf64","libdevice":"__nv_ldexp","llvm":null,"link":"always"}, + {"path":"libm::scalbnf","variant":"LdexpF32","placeholder":"ldexpf32","libdevice":"__nv_ldexpf","llvm":null,"link":"always"}, + {"path":"libm::sin","variant":"SinF64","placeholder":"sinf64","libdevice":"__nv_sin","llvm":null,"link":"always"}, + {"path":"libm::sinf","variant":"SinF32","placeholder":"sinf32","libdevice":"__nv_sinf","llvm":null,"link":"always"}, + {"path":"libm::sinh","variant":"SinhF64","placeholder":"sinhf64","libdevice":"__nv_sinh","llvm":null,"link":"always"}, + {"path":"libm::sinhf","variant":"SinhF32","placeholder":"sinhf32","libdevice":"__nv_sinhf","llvm":null,"link":"always"}, + {"path":"libm::sqrt","variant":"SqrtF64","placeholder":"sqrtf64","libdevice":"__nv_sqrt","llvm":null,"link":"always"}, + {"path":"libm::sqrtf","variant":"SqrtF32","placeholder":"sqrtf32","libdevice":"__nv_sqrtf","llvm":null,"link":"always"}, + {"path":"libm::tan","variant":"TanF64","placeholder":"tanf64","libdevice":"__nv_tan","llvm":null,"link":"always"}, + {"path":"libm::tanf","variant":"TanF32","placeholder":"tanf32","libdevice":"__nv_tanf","llvm":null,"link":"always"}, + {"path":"libm::tanh","variant":"TanhF64","placeholder":"tanhf64","libdevice":"__nv_tanh","llvm":null,"link":"always"}, + {"path":"libm::tanhf","variant":"TanhF32","placeholder":"tanhf32","libdevice":"__nv_tanhf","llvm":null,"link":"always"}, + {"path":"libm::tgamma","variant":"TgammaF64","placeholder":"tgammaf64","libdevice":"__nv_tgamma","llvm":null,"link":"always"}, + {"path":"libm::tgammaf","variant":"TgammaF32","placeholder":"tgammaf32","libdevice":"__nv_tgammaf","llvm":null,"link":"always"}, + {"path":"libm::trunc","variant":"TruncF64","placeholder":"truncf64","libdevice":"__nv_trunc","llvm":"llvm_trunc_f64","link":"libnvvm"}, + {"path":"libm::truncf","variant":"TruncF32","placeholder":"truncf32","libdevice":"__nv_truncf","llvm":"llvm_trunc_f32","link":"libnvvm"}, + {"path":"std::intrinsics::ceilf32","variant":"CeilF32","placeholder":"ceilf32","libdevice":"__nv_ceilf","llvm":"llvm_ceil_f32","link":"libnvvm"}, + {"path":"std::intrinsics::ceilf64","variant":"CeilF64","placeholder":"ceilf64","libdevice":"__nv_ceil","llvm":"llvm_ceil_f64","link":"libnvvm"}, + {"path":"std::intrinsics::copysignf32","variant":"CopysignF32","placeholder":"copysignf32","libdevice":"__nv_copysignf","llvm":"llvm_copysign_f32","link":"libnvvm"}, + {"path":"std::intrinsics::copysignf64","variant":"CopysignF64","placeholder":"copysignf64","libdevice":"__nv_copysign","llvm":"llvm_copysign_f64","link":"libnvvm"}, + {"path":"std::intrinsics::cosf32","variant":"CosF32","placeholder":"cosf32","libdevice":"__nv_cosf","llvm":null,"link":"always"}, + {"path":"std::intrinsics::cosf64","variant":"CosF64","placeholder":"cosf64","libdevice":"__nv_cos","llvm":null,"link":"always"}, + {"path":"std::intrinsics::exp2f32","variant":"Exp2F32","placeholder":"exp2f32","libdevice":"__nv_exp2f","llvm":null,"link":"always"}, + {"path":"std::intrinsics::exp2f64","variant":"Exp2F64","placeholder":"exp2f64","libdevice":"__nv_exp2","llvm":null,"link":"always"}, + {"path":"std::intrinsics::expf32","variant":"ExpF32","placeholder":"expf32","libdevice":"__nv_expf","llvm":null,"link":"always"}, + {"path":"std::intrinsics::expf64","variant":"ExpF64","placeholder":"expf64","libdevice":"__nv_exp","llvm":null,"link":"always"}, + {"path":"std::intrinsics::fabs","variant":"Fabs","placeholder":"fabs","libdevice":"__nv_fabs","llvm":"llvm_fabs_f64","link":"libnvvm"}, + {"path":"std::intrinsics::fadd_fast","variant":"FaddFast","placeholder":"fadd_fast","libdevice":null,"llvm":null,"link":"never"}, + {"path":"std::intrinsics::fdiv_fast","variant":"FdivFast","placeholder":"fdiv_fast","libdevice":null,"llvm":null,"link":"never"}, + {"path":"std::intrinsics::floorf32","variant":"FloorF32","placeholder":"floorf32","libdevice":"__nv_floorf","llvm":"llvm_floor_f32","link":"libnvvm"}, + {"path":"std::intrinsics::floorf64","variant":"FloorF64","placeholder":"floorf64","libdevice":"__nv_floor","llvm":"llvm_floor_f64","link":"libnvvm"}, + {"path":"std::intrinsics::fmaf32","variant":"FmaF32","placeholder":"fmaf32","libdevice":"__nv_fmaf","llvm":null,"link":"always"}, + {"path":"std::intrinsics::fmaf64","variant":"FmaF64","placeholder":"fmaf64","libdevice":"__nv_fma","llvm":null,"link":"always"}, + {"path":"std::intrinsics::fmul_fast","variant":"FmulFast","placeholder":"fmul_fast","libdevice":null,"llvm":null,"link":"never"}, + {"path":"std::intrinsics::fmuladdf32","variant":"FmuladdF32","placeholder":"fmuladdf32","libdevice":"__nv_fmaf","llvm":null,"link":"always"}, + {"path":"std::intrinsics::fmuladdf64","variant":"FmuladdF64","placeholder":"fmuladdf64","libdevice":"__nv_fma","llvm":null,"link":"always"}, + {"path":"std::intrinsics::frem_fast","variant":"FremFast","placeholder":"frem_fast","libdevice":null,"llvm":null,"link":"never"}, + {"path":"std::intrinsics::fsub_fast","variant":"FsubFast","placeholder":"fsub_fast","libdevice":null,"llvm":null,"link":"never"}, + {"path":"std::intrinsics::log10f32","variant":"Log10F32","placeholder":"log10f32","libdevice":"__nv_log10f","llvm":null,"link":"always"}, + {"path":"std::intrinsics::log10f64","variant":"Log10F64","placeholder":"log10f64","libdevice":"__nv_log10","llvm":null,"link":"always"}, + {"path":"std::intrinsics::log2f32","variant":"Log2F32","placeholder":"log2f32","libdevice":"__nv_log2f","llvm":null,"link":"always"}, + {"path":"std::intrinsics::log2f64","variant":"Log2F64","placeholder":"log2f64","libdevice":"__nv_log2","llvm":null,"link":"always"}, + {"path":"std::intrinsics::logf32","variant":"LogF32","placeholder":"logf32","libdevice":"__nv_logf","llvm":null,"link":"always"}, + {"path":"std::intrinsics::logf64","variant":"LogF64","placeholder":"logf64","libdevice":"__nv_log","llvm":null,"link":"always"}, + {"path":"std::intrinsics::maximum_number_nsz_f32","variant":"MaxNumNszF32","placeholder":"maximum_number_nsz_f32","libdevice":null,"llvm":null,"link":"never"}, + {"path":"std::intrinsics::maximum_number_nsz_f64","variant":"MaxNumNszF64","placeholder":"maximum_number_nsz_f64","libdevice":null,"llvm":null,"link":"never"}, + {"path":"std::intrinsics::minimum_number_nsz_f32","variant":"MinNumNszF32","placeholder":"minimum_number_nsz_f32","libdevice":null,"llvm":null,"link":"never"}, + {"path":"std::intrinsics::minimum_number_nsz_f64","variant":"MinNumNszF64","placeholder":"minimum_number_nsz_f64","libdevice":null,"llvm":null,"link":"never"}, + {"path":"std::intrinsics::powf32","variant":"PowfF32","placeholder":"powf32","libdevice":"__nv_powf","llvm":null,"link":"always"}, + {"path":"std::intrinsics::powf64","variant":"PowfF64","placeholder":"powf64","libdevice":"__nv_pow","llvm":null,"link":"always"}, + {"path":"std::intrinsics::powif32","variant":"PowiF32","placeholder":"powif32","libdevice":"__nv_powif","llvm":null,"link":"always"}, + {"path":"std::intrinsics::powif64","variant":"PowiF64","placeholder":"powif64","libdevice":"__nv_powi","llvm":null,"link":"always"}, + {"path":"std::intrinsics::round_ties_even_f32","variant":"RoundevenF32","placeholder":"round_ties_even_f32","libdevice":"__nv_rintf","llvm":"llvm_roundeven_f32","link":"libnvvm"}, + {"path":"std::intrinsics::round_ties_even_f64","variant":"RoundevenF64","placeholder":"round_ties_even_f64","libdevice":"__nv_rint","llvm":"llvm_roundeven_f64","link":"libnvvm"}, + {"path":"std::intrinsics::roundf32","variant":"RoundF32","placeholder":"roundf32","libdevice":"__nv_roundf","llvm":"llvm_round_f32","link":"libnvvm"}, + {"path":"std::intrinsics::roundf64","variant":"RoundF64","placeholder":"roundf64","libdevice":"__nv_round","llvm":"llvm_round_f64","link":"libnvvm"}, + {"path":"std::intrinsics::sinf32","variant":"SinF32","placeholder":"sinf32","libdevice":"__nv_sinf","llvm":null,"link":"always"}, + {"path":"std::intrinsics::sinf64","variant":"SinF64","placeholder":"sinf64","libdevice":"__nv_sin","llvm":null,"link":"always"}, + {"path":"std::intrinsics::sqrtf32","variant":"SqrtF32","placeholder":"sqrtf32","libdevice":"__nv_sqrtf","llvm":null,"link":"always"}, + {"path":"std::intrinsics::sqrtf64","variant":"SqrtF64","placeholder":"sqrtf64","libdevice":"__nv_sqrt","llvm":null,"link":"always"}, + {"path":"std::intrinsics::tanf32","variant":"TanF32","placeholder":"tanf32","libdevice":"__nv_tanf","llvm":null,"link":"always"}, + {"path":"std::intrinsics::tanf64","variant":"TanF64","placeholder":"tanf64","libdevice":"__nv_tan","llvm":null,"link":"always"}, + {"path":"std::intrinsics::truncf32","variant":"TruncF32","placeholder":"truncf32","libdevice":"__nv_truncf","llvm":"llvm_trunc_f32","link":"libnvvm"}, + {"path":"std::intrinsics::truncf64","variant":"TruncF64","placeholder":"truncf64","libdevice":"__nv_trunc","llvm":"llvm_trunc_f64","link":"libnvvm"}, + {"path":"std::sys::cmath::acos","variant":"AcosF64","placeholder":"acosf64","libdevice":"__nv_acos","llvm":null,"link":"always"}, + {"path":"std::sys::cmath::acosf","variant":"AcosF32","placeholder":"acosf32","libdevice":"__nv_acosf","llvm":null,"link":"always"}, + {"path":"std::sys::cmath::acosh","variant":"AcoshF64","placeholder":"acoshf64","libdevice":"__nv_acosh","llvm":null,"link":"always"}, + {"path":"std::sys::cmath::acoshf","variant":"AcoshF32","placeholder":"acoshf32","libdevice":"__nv_acoshf","llvm":null,"link":"always"}, + {"path":"std::sys::cmath::asin","variant":"AsinF64","placeholder":"asinf64","libdevice":"__nv_asin","llvm":null,"link":"always"}, + {"path":"std::sys::cmath::asinf","variant":"AsinF32","placeholder":"asinf32","libdevice":"__nv_asinf","llvm":null,"link":"always"}, + {"path":"std::sys::cmath::asinh","variant":"AsinhF64","placeholder":"asinhf64","libdevice":"__nv_asinh","llvm":null,"link":"always"}, + {"path":"std::sys::cmath::asinhf","variant":"AsinhF32","placeholder":"asinhf32","libdevice":"__nv_asinhf","llvm":null,"link":"always"}, + {"path":"std::sys::cmath::atan","variant":"AtanF64","placeholder":"atanf64","libdevice":"__nv_atan","llvm":null,"link":"always"}, + {"path":"std::sys::cmath::atan2","variant":"Atan2F64","placeholder":"atan2f64","libdevice":"__nv_atan2","llvm":null,"link":"always"}, + {"path":"std::sys::cmath::atan2f","variant":"Atan2F32","placeholder":"atan2f32","libdevice":"__nv_atan2f","llvm":null,"link":"always"}, + {"path":"std::sys::cmath::atanf","variant":"AtanF32","placeholder":"atanf32","libdevice":"__nv_atanf","llvm":null,"link":"always"}, + {"path":"std::sys::cmath::atanh","variant":"AtanhF64","placeholder":"atanhf64","libdevice":"__nv_atanh","llvm":null,"link":"always"}, + {"path":"std::sys::cmath::atanhf","variant":"AtanhF32","placeholder":"atanhf32","libdevice":"__nv_atanhf","llvm":null,"link":"always"}, + {"path":"std::sys::cmath::cbrt","variant":"CbrtF64","placeholder":"cbrtf64","libdevice":"__nv_cbrt","llvm":null,"link":"always"}, + {"path":"std::sys::cmath::cbrtf","variant":"CbrtF32","placeholder":"cbrtf32","libdevice":"__nv_cbrtf","llvm":null,"link":"always"}, + {"path":"std::sys::cmath::cos","variant":"CosF64","placeholder":"cosf64","libdevice":"__nv_cos","llvm":null,"link":"always"}, + {"path":"std::sys::cmath::cosf","variant":"CosF32","placeholder":"cosf32","libdevice":"__nv_cosf","llvm":null,"link":"always"}, + {"path":"std::sys::cmath::cosh","variant":"CoshF64","placeholder":"coshf64","libdevice":"__nv_cosh","llvm":null,"link":"always"}, + {"path":"std::sys::cmath::coshf","variant":"CoshF32","placeholder":"coshf32","libdevice":"__nv_coshf","llvm":null,"link":"always"}, + {"path":"std::sys::cmath::expm1","variant":"Expm1F64","placeholder":"expm1f64","libdevice":"__nv_expm1","llvm":null,"link":"always"}, + {"path":"std::sys::cmath::expm1f","variant":"Expm1F32","placeholder":"expm1f32","libdevice":"__nv_expm1f","llvm":null,"link":"always"}, + {"path":"std::sys::cmath::hypot","variant":"HypotF64","placeholder":"hypotf64","libdevice":"__nv_hypot","llvm":null,"link":"always"}, + {"path":"std::sys::cmath::hypotf","variant":"HypotF32","placeholder":"hypotf32","libdevice":"__nv_hypotf","llvm":null,"link":"always"}, + {"path":"std::sys::cmath::log1p","variant":"Log1pF64","placeholder":"log1pf64","libdevice":"__nv_log1p","llvm":null,"link":"always"}, + {"path":"std::sys::cmath::log1pf","variant":"Log1pF32","placeholder":"log1pf32","libdevice":"__nv_log1pf","llvm":null,"link":"always"}, + {"path":"std::sys::cmath::sin","variant":"SinF64","placeholder":"sinf64","libdevice":"__nv_sin","llvm":null,"link":"always"}, + {"path":"std::sys::cmath::sinf","variant":"SinF32","placeholder":"sinf32","libdevice":"__nv_sinf","llvm":null,"link":"always"}, + {"path":"std::sys::cmath::sinh","variant":"SinhF64","placeholder":"sinhf64","libdevice":"__nv_sinh","llvm":null,"link":"always"}, + {"path":"std::sys::cmath::sinhf","variant":"SinhF32","placeholder":"sinhf32","libdevice":"__nv_sinhf","llvm":null,"link":"always"}, + {"path":"std::sys::cmath::tan","variant":"TanF64","placeholder":"tanf64","libdevice":"__nv_tan","llvm":null,"link":"always"}, + {"path":"std::sys::cmath::tanf","variant":"TanF32","placeholder":"tanf32","libdevice":"__nv_tanf","llvm":null,"link":"always"}, + {"path":"std::sys::cmath::tanh","variant":"TanhF64","placeholder":"tanhf64","libdevice":"__nv_tanh","llvm":null,"link":"always"}, + {"path":"std::sys::cmath::tanhf","variant":"TanhF32","placeholder":"tanhf32","libdevice":"__nv_tanhf","llvm":null,"link":"always"}, + {"path":"std::sys::cmath::tgamma","variant":"TgammaF64","placeholder":"tgammaf64","libdevice":"__nv_tgamma","llvm":null,"link":"always"}, + {"path":"std::sys::cmath::tgammaf","variant":"TgammaF32","placeholder":"tgammaf32","libdevice":"__nv_tgammaf","llvm":null,"link":"always"} +] diff --git a/cuda-oxide/crates/cuda-intrinsics-gen/src/libdevice_math/mod.rs b/cuda-oxide/crates/cuda-intrinsics-gen/src/libdevice_math/mod.rs index c21399936b..9b2bb01b11 100644 --- a/cuda-oxide/crates/cuda-intrinsics-gen/src/libdevice_math/mod.rs +++ b/cuda-oxide/crates/cuda-intrinsics-gen/src/libdevice_math/mod.rs @@ -3,8 +3,9 @@ * SPDX-License-Identifier: Apache-2.0 */ -//! Staged libm mappings. This family has bitcode evidence, not hardware ABI IDs. +//! Compiler math mappings. This family has bitcode evidence, not hardware ABI IDs. +mod compiler; mod evidence; mod render; #[cfg(test)] @@ -45,7 +46,15 @@ struct Function { name: String, signature: Signature, #[serde(default)] - libm: Vec, + libm: Option>, + #[serde(default)] + core: Vec, + #[serde(default)] + paths: Vec, + #[serde(default)] + variant: Option, + #[serde(default)] + constant: Option, #[serde(default)] cmath: Option, #[serde(default)] @@ -130,7 +139,11 @@ impl Link { #[derive(Debug, Clone, PartialEq, Eq)] struct Mapping { - libm: String, + libm: Option, + width: String, + variant: Option, + constant: Option, + paths: Vec, signature: String, placeholders: Vec, cmath: Option, @@ -161,6 +174,8 @@ fn identifier(value: &str) -> bool { fn expand(template: &str, width: &str) -> Result { let value = template .replace("{width}", width) + .replace("{Width}", if width == "f32" { "F32" } else { "F64" }) + .replace("{WIDTH}", if width == "f32" { "F32" } else { "F64" }) .replace("{suffix}", if width == "f32" { "f" } else { "" }); ensure!( identifier(&value), @@ -171,7 +186,7 @@ fn expand(template: &str, width: &str) -> Result { fn parse(bytes: &[u8]) -> Result { let shard: Shard = toml::from_slice(bytes).context("parse libdevice_math shard")?; - ensure!(shard.schema == 1, "unsupported libdevice_math schema"); + ensure!(shard.schema == 2, "unsupported libdevice_math schema"); ensure!( shard.family == "libdevice_math", "wrong libdevice_math family" @@ -215,6 +230,10 @@ fn mappings(shard: &Shard) -> Result> { let mut exports = BTreeSet::new(); let mut cmath_paths = BTreeSet::new(); let mut scalar_placeholders = BTreeSet::new(); + let mut compiler_paths = BTreeMap::new(); + let mut variants = BTreeMap::new(); + let mut constants = BTreeMap::new(); + let mut registered_placeholders = BTreeMap::new(); let mut result = Vec::new(); for function in &shard.libdevice_math.functions { ensure!(identifier(&function.name), "invalid math function name"); @@ -230,6 +249,15 @@ fn mappings(shard: &Shard) -> Result> { .is_some_and(identifier), "invalid handwritten math lowering {lowering:?}" ); + ensure!( + matches!( + lowering.as_str(), + "handwritten:emit_sincos" + | "handwritten:emit_frexp" + | "handwritten:lower_float_minmax_nsz" + ), + "unsupported handwritten math lowering {lowering:?}" + ); } else { ensure!( !matches!( @@ -239,11 +267,42 @@ fn mappings(shard: &Shard) -> Result> { "tuple-returning math requires an explicit handwritten adapter" ); } - let aliases = if function.libm.is_empty() { - vec![function.name.clone()] - } else { - function.libm.clone() - }; + let aliases = function + .libm + .clone() + .unwrap_or_else(|| vec![function.name.clone()]); + let sincos = function.lowering.as_deref() == Some("handwritten:emit_sincos"); + if sincos { + ensure!( + function.signature == Signature::FloatPair + && function.libdevice.is_none() + && function.link == Link::Always, + "sincos must expand a float pair without a direct libdevice call" + ); + ensure!( + !aliases.is_empty() + && function.variant.is_none() + && function.constant.is_none() + && function.core.is_empty() + && function.paths.is_empty() + && function.cmath.is_none(), + "sincos must use its handwritten libm tuple route" + ); + } + if function.lowering.as_deref() == Some("handwritten:emit_frexp") { + ensure!( + function.signature == Signature::FloatIntegerPair + && function.link == Link::Always + && function.libdevice.is_some(), + "frexp requires its float/i32 tuple ABI and libdevice symbol" + ); + } + if function.lowering.as_deref() == Some("handwritten:lower_float_minmax_nsz") { + ensure!( + function.signature == Signature::Binary && function.link == Link::Never, + "min/max must use the handwritten binary never-linked route" + ); + } for width in &shard.libdevice_math.widths { let symbol = if let Some(symbol) = &function.libdevice { Some(expand(symbol, width)?) @@ -288,6 +347,18 @@ fn mappings(shard: &Shard) -> Result> { .iter() .map(|name| expand(name, width)) .collect::>>()?; + ensure!(!placeholders.is_empty(), "math requires a placeholder"); + if sincos { + ensure!( + placeholders.len() == 2, + "sincos requires two scalar placeholders" + ); + } else { + ensure!( + placeholders.len() == 1, + "plain math requires one placeholder" + ); + } if function.lowering.is_none() { ensure!( placeholders.len() == 1, @@ -310,15 +381,159 @@ fn mappings(shard: &Shard) -> Result> { "duplicate cmath spelling {path}" ); } - for alias in &aliases { - ensure!(identifier(alias), "invalid libm alias {alias:?}"); - let libm = format!("{alias}{}", if width == "f32" { "f" } else { "" }); + let mut paths = Vec::new(); + for core in &function.core { + let name = expand(core, width)?; + paths.extend([ + format!("core::intrinsics::{name}"), + format!("std::intrinsics::{name}"), + ]); + } + for path in &function.paths { + paths.push( + path.split("::") + .map(|segment| expand(segment, width)) + .collect::>>()? + .join("::"), + ); + } + paths.extend(cmath.clone()); + ensure!( + !aliases.is_empty() || !paths.is_empty(), + "math row has no compiler entry point" + ); + let (variant, constant) = if sincos { + (None, None) + } else { + let title = format!( + "{}{}", + function.name[..1].to_ascii_uppercase(), + &function.name[1..] + ); + let variant = expand( + function + .variant + .as_deref() + .unwrap_or(&format!("{title}{{Width}}")), + width, + )?; + let constant = expand( + function.constant.as_deref().unwrap_or(&format!( + "CALLEE_{}_{{WIDTH}}", + function.name.to_ascii_uppercase() + )), + width, + )?; + ensure!( + variant.as_bytes()[0].is_ascii_uppercase() && !variant.contains('_'), + "math variant must be CamelCase" + ); ensure!( - exports.insert(libm.clone()), - "duplicate libm spelling {libm}" + !compiler::FAST + .iter() + .any(|(name, callee, suffix)| *name == variant + || *callee == constant + || *suffix == placeholders[0]), + "math row collides with fast arithmetic" ); + ensure!( + constant.starts_with("CALLEE_") + && constant.bytes().all(|byte| byte.is_ascii_uppercase() + || byte.is_ascii_digit() + || byte == b'_'), + "math constant must start with CALLEE_ and use uppercase identifiers" + ); + let role = if width == "f32" { "F32" } else { "F64" }; + ensure!( + (variant == format!("Frexp{role}")) + == (function.lowering.as_deref() == Some("handwritten:emit_frexp")), + "frexp identity must use the matching handwritten adapter" + ); + ensure!( + (variant == format!("MaxNumNsz{role}") + || variant == format!("MinNumNsz{role}")) + == (function.lowering.as_deref() + == Some("handwritten:lower_float_minmax_nsz")), + "min/max identity must use the matching handwritten expansion" + ); + if variant == "Fabs" { + ensure!( + function.signature == Signature::Unary && function.link == Link::Libnvvm, + "generic fabs requires its unary backend-dependent route" + ); + } + let identity = ( + function.name.clone(), + constant.clone(), + placeholders[0].clone(), + ); + if let Some(previous) = variants.insert(variant.clone(), identity.clone()) { + ensure!(previous == identity, "duplicate math variant {variant}"); + ensure!( + variant == "Fabs", + "only fabs uses a generic-width math identity" + ); + } + let identity = ( + function.name.clone(), + variant.clone(), + placeholders[0].clone(), + ); + if let Some(previous) = constants.insert(constant.clone(), identity.clone()) { + ensure!(previous == identity, "duplicate math constant {constant}"); + } + if let Some(previous) = + registered_placeholders.insert(placeholders[0].clone(), variant.clone()) + { + ensure!( + previous == variant, + "duplicate compiler math placeholder {}", + placeholders[0] + ); + } + for path in &paths { + ensure!( + !compiler::FAST.iter().any(|(_, _, suffix)| path + == &format!("core::intrinsics::{suffix}") + || path == &format!("std::intrinsics::{suffix}")), + "compiler math path collides with fast arithmetic" + ); + } + (Some(variant), Some(constant)) + }; + for path in &paths { + if let Some((previous_variant, previous_width)) = + compiler_paths.insert(path.clone(), (variant.clone(), width.clone())) + { + ensure!( + previous_variant == variant && previous_width != *width, + "duplicate compiler math path {path}" + ); + } + } + let libm_names = if aliases.is_empty() { + vec![None] + } else { + aliases + .iter() + .map(|alias| { + ensure!(identifier(alias), "invalid libm alias {alias:?}"); + let libm = format!("{alias}{}", if width == "f32" { "f" } else { "" }); + ensure!( + exports.insert(libm.clone()), + "duplicate libm spelling {libm}" + ); + Ok(Some(libm)) + }) + .collect::>>()? + }; + for libm in libm_names { result.push(Mapping { libm, + width: width.clone(), + variant: variant.clone(), + constant: constant.clone(), + paths: paths.clone(), signature: function.signature.rust(width), placeholders: placeholders.clone(), cmath: cmath.clone(), @@ -331,7 +546,9 @@ fn mappings(shard: &Shard) -> Result> { } } } - result.sort_by(|left, right| left.libm.cmp(&right.libm)); + result.sort_by(|left, right| { + (&left.libm, &left.variant, &left.width).cmp(&(&right.libm, &right.variant, &right.width)) + }); Ok(result) } diff --git a/cuda-oxide/crates/cuda-intrinsics-gen/src/libdevice_math/render.rs b/cuda-oxide/crates/cuda-intrinsics-gen/src/libdevice_math/render.rs index 612dc00999..e0c1cadf83 100644 --- a/cuda-oxide/crates/cuda-intrinsics-gen/src/libdevice_math/render.rs +++ b/cuda-oxide/crates/cuda-intrinsics-gen/src/libdevice_math/render.rs @@ -3,25 +3,25 @@ * SPDX-License-Identifier: Apache-2.0 */ -use super::Mapping; +use super::{Mapping, compiler}; use std::collections::BTreeMap; use std::fmt::Write; use std::path::PathBuf; pub(super) fn outputs(rows: &[Mapping], bitcode_hash: &str) -> BTreeMap { let mut rust = String::from( - "/*\n * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n * SPDX-License-Identifier: Apache-2.0\n */\n\n// @generated by cuda-intrinsics-gen from overlay/libdevice_math.toml. DO NOT EDIT.\n// Staged metadata; compiler dispatch is still handwritten.\n\n#[derive(Debug, Clone, Copy, PartialEq, Eq)]\npub struct MathMapping {\n pub libm: &'static str,\n pub signature: &'static str,\n pub placeholders: &'static [&'static str],\n pub cmath: Option<&'static str>,\n pub libdevice: Option<&'static str>,\n pub llvm: Option<&'static str>,\n pub lowering: Option<&'static str>,\n pub link: &'static str,\n}\n\npub const MAPPINGS: &[MathMapping] = &[\n", + "/*\n * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n * SPDX-License-Identifier: Apache-2.0\n */\n\n// @generated by cuda-intrinsics-gen from overlay/libdevice_math.toml. DO NOT EDIT.\n\n#[derive(Debug, Clone, Copy, PartialEq, Eq)]\npub struct MathMapping {\n pub libm: Option<&'static str>,\n pub width: &'static str,\n pub variant: Option<&'static str>,\n pub constant: Option<&'static str>,\n pub paths: &'static [&'static str],\n pub signature: &'static str,\n pub placeholders: &'static [&'static str],\n pub cmath: Option<&'static str>,\n pub libdevice: Option<&'static str>,\n pub llvm: Option<&'static str>,\n pub lowering: Option<&'static str>,\n pub link: &'static str,\n}\n\npub const MAPPINGS: &[MathMapping] = &[\n", ); + let exports = rows.iter().filter(|row| row.libm.is_some()).count(); let mut reference = format!( - "\n\n\n\n# Staged libm mappings\n\n{} mapped f32/f64 libm exports. Compiler dispatch remains handwritten.\n\nLibdevice bitcode SHA-256: `{bitcode_hash}`.\n\n`libnvvm` links only on that backend; `always` links on both; `never` uses\na handwritten expansion. Aliases are separate exports sharing a mapping.\n\n| libm export | Rust signature | libdevice symbol | LLVM NVPTX alternative | Handwritten lowering | Linking |\n| --- | --- | --- | --- | --- | --- |\n", - rows.len() + "\n\n\n\n# Compiler math mappings\n\n{exports} mapped f32/f64 libm exports, plus core-only compiler routes.\n\nThe same shard generates compiler dispatch, placeholder constants, symbol lookup,\noperand counts, cmath interception, and libdevice link requirements. Handwritten\nsemantic expansions remain responsible for sincos, frexp, min/max, and fast binops.\n\nLibdevice bitcode SHA-256: `{bitcode_hash}`.\n\n`libnvvm` links only on that backend; `always` links on both; `never` uses\na handwritten expansion. Aliases are separate exports sharing a mapping.\n\n| libm export | Rust signature | libdevice symbol | LLVM NVPTX alternative | Handwritten lowering | Linking |\n| --- | --- | --- | --- | --- | --- |\n" ); for row in rows { - writeln!(rust, "MathMapping {{ libm: {:?}, signature: {:?}, placeholders: &{:?}, cmath: {:?}, libdevice: {:?}, llvm: {:?}, lowering: {:?}, link: {:?} }},", row.libm, row.signature, row.placeholders, row.cmath, row.libdevice, row.llvm, row.lowering, row.link.as_str()).unwrap(); + writeln!(rust, "MathMapping {{ libm: {:?}, width: {:?}, variant: {:?}, constant: {:?}, paths: &{:?}, signature: {:?}, placeholders: &{:?}, cmath: {:?}, libdevice: {:?}, llvm: {:?}, lowering: {:?}, link: {:?} }},", row.libm, row.width, row.variant, row.constant, row.paths, row.signature, row.placeholders, row.cmath, row.libdevice, row.llvm, row.lowering, row.link.as_str()).unwrap(); writeln!( reference, - "| `{}` | `{}` | {} | {} | {} | `{}` |", - row.libm, + "| {} | `{}` | {} | {} | {} | `{}` |", + code(row.libm.as_deref()), row.signature, code(row.libdevice.as_deref()), code(row.llvm.as_deref()), @@ -37,6 +37,10 @@ pub(super) fn outputs(rows: &[Mapping], bitcode_hash: &str) -> BTreeMap PathBuf { } fn fixture(functions: &str) -> Shard { - parse(format!("schema = 1\nfamily = \"libdevice_math\"\n[libdevice_math]\nevidence = \"evidence/test.json\"\nlibdevice_sha256 = \"{}\"\nwidths = [\"f32\", \"f64\"]\n{functions}", "a".repeat(64)).as_bytes()).unwrap() + parse(format!("schema = 2\nfamily = \"libdevice_math\"\n[libdevice_math]\nevidence = \"evidence/test.json\"\nlibdevice_sha256 = \"{}\"\nwidths = [\"f32\", \"f64\"]\n{functions}", "a".repeat(64)).as_bytes()).unwrap() } #[test] -fn staged_snapshot_has_exactly_the_original_72_exports() { +fn generated_metadata_covers_the_existing_82_exports_and_core_only_routes() { let shard = read(&root()).unwrap(); let rows = mappings(&shard).unwrap(); - assert_eq!(rows.len(), 72); - assert_eq!(expected_symbols(&rows).unwrap().len(), 64); + assert_eq!(rows.len(), 86); + assert_eq!(rows.iter().filter(|row| row.libm.is_some()).count(), 82); + assert_eq!(expected_symbols(&rows).unwrap().len(), 74); assert_eq!(generated::MAPPINGS.len(), rows.len()); for (actual, row) in generated::MAPPINGS.iter().zip(&rows) { - assert_eq!(actual.libm, row.libm); + assert_eq!(actual.libm, row.libm.as_deref()); + assert_eq!(actual.width, row.width); + assert_eq!(actual.variant, row.variant.as_deref()); + assert_eq!(actual.constant, row.constant.as_deref()); + assert_eq!( + actual.paths, + row.paths.iter().map(String::as_str).collect::>() + ); assert_eq!(actual.signature, row.signature); assert_eq!( actual.placeholders, @@ -42,136 +50,181 @@ fn staged_snapshot_has_exactly_the_original_72_exports() { assert_eq!(actual.llvm, row.llvm.as_deref()); assert_eq!(actual.lowering, row.lowering.as_deref()); assert_eq!(actual.link, row.link.as_str()); - } -} - -fn quoted(text: &str) -> impl Iterator { - text.split('"').skip(1).step_by(2) -} - -fn function_body<'a>(source: &'a str, name: &str) -> &'a str { - let start = source.find(&format!("fn {name}(")).unwrap(); - let rest = &source[start..]; - let end = [rest.find("\n fn "), rest.find("\n}\n")] - .into_iter() - .flatten() - .min() - .unwrap(); - &rest[..end] -} - -fn variants(source: &str, name: &str, target: &str) -> BTreeMap { - let mut result = BTreeMap::new(); - for line in function_body(source, name).lines() { - let Some((left, right)) = line.split_once(target) else { - continue; - }; - for variant in left.split('|') { - if let Some(variant) = variant.trim().strip_prefix("Self::") { - result.insert( - variant.into(), - right.split([')', ',']).next().unwrap().trim().into(), + if let Some(variant) = &row.variant { + let callee = format!("{}{}", compiled::PLACEHOLDER_PREFIX, row.placeholders[0]); + let intrinsic = + compiled::RustFloatMathIntrinsic::from_placeholder_callee(&callee).unwrap(); + assert_eq!(format!("{intrinsic:?}"), *variant); + assert_eq!(intrinsic.placeholder_callee(), callee); + assert_eq!( + intrinsic.libdevice_symbol(row.width == "f64"), + row.libdevice.as_deref() + ); + assert_eq!( + intrinsic.llvm_symbol(row.width == "f64"), + row.llvm.as_deref() + ); + for path in &row.paths { + assert_eq!( + compiled::RustFloatMathIntrinsic::from_core_path(path), + Some(intrinsic) + ); + } + if let Some(alias) = &row.libm { + assert_eq!( + compiled::RustFloatMathIntrinsic::from_libm_path(&format!("libm::{alias}")), + Some(intrinsic) ); } } } - result } -/// Compare staged rows against the live handwritten routes, not another copy -/// of an expected mapping list. Additional hand-landed functions may coexist -/// until the compiler-table migration, without making this PR depend on them. +// Compile the actual compiler consumer, including its generated dispatch. The +// baseline records the handwritten routes from PRs #1638 and #1645, integrated +// with #1646 before migration. It is independent of the new generator/shard and +// remains unchanged when later PRs add aliases. +#[path = "../../../dialect-mir/src/rust_intrinsics.rs"] +mod compiled; + +#[derive(Deserialize)] +struct LegacyRoute { + path: String, + variant: String, + placeholder: String, + libdevice: Option, + llvm: Option, + link: String, +} + #[test] -fn staged_rows_preserve_existing_import_placeholder_symbol_and_link_routes() { - let importer = - include_str!("../../../mir-importer/src/translator/terminator/intrinsics/float_math.rs"); - let dialect = include_str!("../../../dialect-mir/src/rust_intrinsics.rs"); - let lower = include_str!("../../../mir-lower/src/convert/ops/call.rs"); - let start = importer.find("fn from_libm_path(").unwrap(); - let end = start - + importer[start..] - .find("fn from_fast_intrinsic_path(") - .unwrap(); - let mut imports = BTreeMap::new(); - for line in importer[start..end].lines() { - if let Some((left, right)) = line.split_once("=> Some(Self::") { - for name in quoted(left) { - imports.insert(name, right.split(')').next().unwrap()); - } - } - } - let placeholders = variants(importer, "placeholder_callee", "=> rust_intrinsics::"); - let constants: BTreeMap<_, _> = dialect - .lines() - .filter_map(|line| { - let line = line.strip_prefix("pub const ")?; - let (name, _) = line.split_once(':')?; - Some((name, quoted(line).next()?)) - }) +fn compiler_dispatch_preserves_every_pre_migration_path_symbol_and_link_route() { + let expected: Vec = + serde_json::from_str(include_str!("migration-baseline.json")).unwrap(); + assert_eq!(expected.len(), 214); + let shard = read(&root()).unwrap(); + let rows = mappings(&shard).unwrap(); + let mut actual_paths: BTreeSet<_> = rows + .iter() + .flat_map(|row| row.paths.iter().cloned()) .collect(); - let symbols = variants(lower, "libdevice_name", "=> Ok("); - let native = variants(lower, "llvm_intrinsic_name", "=> Ok(Some("); - let always = function_body(dialect, "is_libdevice_backed_placeholder"); - let conditional = function_body(dialect, "is_backend_dependent_libdevice_placeholder"); - let contains_constant = |body: &str, name: &str| { - body.split(|ch: char| !ch.is_ascii_alphanumeric() && ch != '_') - .any(|token| token == name) - }; - for row in mappings(&read(&root()).unwrap()).unwrap() { - if row.libm == "sincos" || row.libm == "sincosf" { - assert!(importer.contains(&format!("\"{}\" => Some(", row.libm))); - assert_eq!(row.placeholders.len(), 2); - assert_eq!(row.lowering.as_deref(), Some("handwritten:emit_sincos")); - continue; - } - let variant = imports[row.libm.as_str()]; - let constant = &placeholders[variant]; + actual_paths.extend( + rows.iter() + .filter(|row| row.variant.is_some()) + .filter_map(|row| row.libm.as_ref().map(|name| format!("libm::{name}"))), + ); + for (_, _, name) in compiler::FAST { + actual_paths.extend([ + format!("core::intrinsics::{name}"), + format!("std::intrinsics::{name}"), + ]); + } + let legacy_paths: BTreeSet<_> = expected.iter().map(|row| row.path.clone()).collect(); + assert!(legacy_paths.is_subset(&actual_paths)); + for row in expected { + let intrinsic = compiled::RustFloatMathIntrinsic::from_core_path(&row.path).unwrap(); + assert_eq!(format!("{intrinsic:?}"), row.variant, "{}", row.path); + let placeholder = format!("{}{}", compiled::PLACEHOLDER_PREFIX, row.placeholder); + assert_eq!(intrinsic.placeholder_callee(), placeholder, "{}", row.path); assert_eq!( - row.placeholders, - [constants[constant.as_str()]], + compiled::RustFloatMathIntrinsic::from_placeholder_callee(&placeholder), + Some(intrinsic) + ); + let is_f64 = row.variant == "Fabs" && row.path.ends_with("fabs"); + assert_eq!( + intrinsic.libdevice_symbol(is_f64), + row.libdevice.as_deref(), "{}", - row.libm + row.path ); assert_eq!( - contains_constant(always, constant), - row.link == Link::Always, + intrinsic.llvm_symbol(is_f64), + row.llvm.as_deref(), "{}", - row.libm + row.path ); assert_eq!( - contains_constant(conditional, constant), - row.link == Link::Libnvvm, + compiled::is_libdevice_backed_placeholder(&placeholder), + row.link == "always", "{}", - row.libm + row.path ); - if variant == "Fabs" { - for symbol in [&row.libdevice, &row.llvm].into_iter().flatten() { - assert!(lower.contains(&format!("\"{symbol}\""))); - } - } else { + assert_eq!( + compiled::is_backend_dependent_libdevice_placeholder(&placeholder), + row.link == "libnvvm", + "{}", + row.path + ); + assert!(compiled::is_known_placeholder(&placeholder)); + assert_eq!( + compiled::is_intrinsic_lowered_cmath_shim(&row.path), + row.path.starts_with("std::sys::cmath::") + ); + if let Some(alias) = row.path.strip_prefix("libm::") { assert_eq!( - symbols.get(variant).map(|name| name.trim_matches('"')), - row.libdevice.as_deref(), - "{}", - row.libm + compiled::RustFloatMathIntrinsic::from_core_path(&format!( + "libm::math::module::{alias}" + )), + Some(intrinsic) ); assert_eq!( - native.get(variant).map(|name| name.trim_matches('"')), - row.llvm.as_deref(), - "{}", - row.libm + compiled::RustFloatMathIntrinsic::from_core_path(&format!("user::libm::{alias}")), + None ); } - if let Some(path) = &row.cmath { - assert!(importer.contains(&format!("\"{path}\" => Some(Self::{variant})"))); - } } } +#[test] +fn unregistered_aliases_and_unknown_placeholders_remain_rejected() { + for path in [ + "libm::erf", + "libm::modf", + "libm::remquo", + "libm::sqrtf128", + "user::std::sys::cmath::tgamma", + "libmath::sqrtf", + ] { + assert_eq!( + compiled::RustFloatMathIntrinsic::from_core_path(path), + None, + "{path}" + ); + assert!(!compiled::is_intrinsic_lowered_cmath_shim(path)); + } + let unknown = format!("{}future_math_f32", compiled::PLACEHOLDER_PREFIX); + assert_eq!( + compiled::RustFloatMathIntrinsic::from_placeholder_callee(&unknown), + None + ); + assert!(!compiled::is_known_placeholder(&unknown)); + assert!(!compiled::is_libdevice_backed_placeholder(&unknown)); + assert!(!compiled::is_backend_dependent_libdevice_placeholder( + &unknown + )); + assert_eq!( + compiled::RustFloatMathIntrinsic::from_libm_path("libm::sincosf"), + None + ); + assert_eq!(compiled::libm_sincos_is_f64("libm::sincosf"), Some(false)); + assert_eq!( + compiled::libm_sincos_is_f64("libm::math::sincos::sincos"), + Some(true) + ); + assert_eq!(compiled::libm_sincos_is_f64("user::libm::sincos"), None); + assert_eq!(compiled::RustFloatMathIntrinsic::LdexpF64.arg_count(), 2); + assert_eq!(compiled::RustFloatMathIntrinsic::FmaF32.arg_count(), 3); + assert_eq!(compiled::RustFloatMathIntrinsic::FrexpF64.arg_count(), 1); +} + #[test] fn aliases_and_handwritten_controls_keep_their_semantics() { let rows = mappings(&read(&root()).unwrap()).unwrap(); - let find = |name: &str| rows.iter().find(|row| row.libm == name).unwrap(); + let find = |name: &str| { + rows.iter() + .find(|row| row.libm.as_deref() == Some(name)) + .unwrap() + }; for (alias, original) in [("roundeven", "rint"), ("roundevenf", "rintf")] { assert_eq!(find(alias).placeholders, find(original).placeholders); assert_eq!(find(alias).libdevice, find(original).libdevice); @@ -187,6 +240,54 @@ fn aliases_and_handwritten_controls_keep_their_semantics() { assert_eq!(find("sqrt").link, Link::Always); } +#[test] +fn compiler_identities_and_handwritten_handlers_fail_closed() { + let base = "[[libdevice_math.function]]\nname = \"demo\"\nsignature = \"(F) -> F\"\n"; + for (suffix, expected) in [ + ("libm = []\n", "no compiler entry point"), + ("variant = \"demo{width}\"\n", "CamelCase"), + ("constant = \"callee_demo\"\n", "CALLEE_"), + ("variant = \"FaddFast\"\n", "collides with fast arithmetic"), + ("core = [\"fadd_fast\"]\n", "collides with fast arithmetic"), + ("variant = \"Demo\"\n", "duplicate math variant"), + ( + "variant = \"Demo\"\nconstant = \"CALLEE_DEMO\"\nplaceholders = [\"demo\"]\n", + "generic-width math identity", + ), + ("variant = \"Frexp{Width}\"\n", "frexp identity"), + ("variant = \"MaxNumNsz{Width}\"\n", "min/max identity"), + ( + "variant = \"Fabs\"\nconstant = \"CALLEE_FABS\"\nplaceholders = [\"fabs\"]\n", + "generic fabs", + ), + ( + "core = [\"demo{width}\", \"demo{width}\"]\n", + "duplicate compiler math path", + ), + ( + "lowering = \"handwritten:unknown_adapter\"\n", + "unsupported handwritten", + ), + ( + "lowering = \"handwritten:emit_frexp\"\nlibdevice = \"__nv_demo{suffix}\"\n", + "float/i32 tuple ABI", + ), + ( + "[[libdevice_math.function]]\nname = \"other\"\nsignature = \"(F) -> F\"\nvariant = \"Demo{Width}\"\n", + "duplicate math variant", + ), + ( + "[[libdevice_math.function]]\nname = \"other\"\nsignature = \"(F) -> F\"\nconstant = \"CALLEE_DEMO_{WIDTH}\"\n", + "duplicate math constant", + ), + ] { + let error = mappings(&fixture(&format!("{base}{suffix}"))) + .unwrap_err() + .to_string(); + assert!(error.contains(expected), "{suffix}: {error}"); + } +} + #[test] fn render_is_independent_of_row_order_and_check_detects_drift() { let mut shard = read(&root()).unwrap(); @@ -285,7 +386,7 @@ fn malformed_rows_fail_closed() { ), ( "lowering = \"handwritten:emit_demo\"\nllvm = \"llvm_demo_{width}\"\nlink = \"libnvvm\"\n", - "libdevice ABI", + "unsupported handwritten", ), ( "[[libdevice_math.function]]\nname = \"other\"\nsignature = \"(F) -> F\"\nplaceholders = [\"demo{width}\"]\n", @@ -335,7 +436,7 @@ fn malformed_rows_fail_closed() { fn schema_and_evidence_pins_are_strict() { let source = fs::read_to_string(root().join("intrinsics").join(SHARD)).unwrap(); for changed in [ - source.replacen("schema = 1", "schema = 2", 1), + source.replacen("schema = 2", "schema = 3", 1), source.replacen("family = \"libdevice_math\"", "family = \"scalar_math\"", 1), source.replacen("[\"f32\", \"f64\"]", "[\"f16\"]", 1), source.replacen( @@ -343,7 +444,7 @@ fn schema_and_evidence_pins_are_strict() { "evidence/../outside.json", 1, ), - source.replacen("schema = 1", "schema = 1\nunknown = true", 1), + source.replacen("schema = 2", "schema = 2\nunknown = true", 1), source.replacen("(F) -> F", "(F) -> bool", 1), ] { assert!(parse(changed.as_bytes()).is_err()); diff --git a/cuda-oxide/crates/dialect-mir/src/generated/libdevice_math.rs b/cuda-oxide/crates/dialect-mir/src/generated/libdevice_math.rs new file mode 100644 index 0000000000..3f20ea03b0 --- /dev/null +++ b/cuda-oxide/crates/dialect-mir/src/generated/libdevice_math.rs @@ -0,0 +1,1103 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +// @generated by cuda-intrinsics-gen from overlay/libdevice_math.toml. DO NOT EDIT. + +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_ACOSH_F32: &str = placeholder!("acoshf32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_ACOSH_F64: &str = placeholder!("acoshf64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_ACOS_F32: &str = placeholder!("acosf32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_ACOS_F64: &str = placeholder!("acosf64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_ASINH_F32: &str = placeholder!("asinhf32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_ASINH_F64: &str = placeholder!("asinhf64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_ASIN_F32: &str = placeholder!("asinf32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_ASIN_F64: &str = placeholder!("asinf64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_ATAN2_F32: &str = placeholder!("atan2f32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_ATAN2_F64: &str = placeholder!("atan2f64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_ATANH_F32: &str = placeholder!("atanhf32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_ATANH_F64: &str = placeholder!("atanhf64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_ATAN_F32: &str = placeholder!("atanf32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_ATAN_F64: &str = placeholder!("atanf64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_CBRT_F32: &str = placeholder!("cbrtf32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_CBRT_F64: &str = placeholder!("cbrtf64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_CEIL_F32: &str = placeholder!("ceilf32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_CEIL_F64: &str = placeholder!("ceilf64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_COPYSIGN_F32: &str = placeholder!("copysignf32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_COPYSIGN_F64: &str = placeholder!("copysignf64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_COSH_F32: &str = placeholder!("coshf32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_COSH_F64: &str = placeholder!("coshf64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_COS_F32: &str = placeholder!("cosf32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_COS_F64: &str = placeholder!("cosf64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_EXP2_F32: &str = placeholder!("exp2f32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_EXP2_F64: &str = placeholder!("exp2f64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_EXPM1_F32: &str = placeholder!("expm1f32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_EXPM1_F64: &str = placeholder!("expm1f64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_EXP_F32: &str = placeholder!("expf32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_EXP_F64: &str = placeholder!("expf64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_FABS: &str = placeholder!("fabs"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_FLOOR_F32: &str = placeholder!("floorf32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_FLOOR_F64: &str = placeholder!("floorf64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_FMA_F32: &str = placeholder!("fmaf32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_FMA_F64: &str = placeholder!("fmaf64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_FMULADD_F32: &str = placeholder!("fmuladdf32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_FMULADD_F64: &str = placeholder!("fmuladdf64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_FREXP_F32: &str = placeholder!("frexpf32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_FREXP_F64: &str = placeholder!("frexpf64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_HYPOT_F32: &str = placeholder!("hypotf32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_HYPOT_F64: &str = placeholder!("hypotf64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_ILOGB_F32: &str = placeholder!("ilogbf32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_ILOGB_F64: &str = placeholder!("ilogbf64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_LDEXP_F32: &str = placeholder!("ldexpf32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_LDEXP_F64: &str = placeholder!("ldexpf64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_LOG10_F32: &str = placeholder!("log10f32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_LOG10_F64: &str = placeholder!("log10f64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_LOG1P_F32: &str = placeholder!("log1pf32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_LOG1P_F64: &str = placeholder!("log1pf64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_LOG2_F32: &str = placeholder!("log2f32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_LOG2_F64: &str = placeholder!("log2f64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_LOG_F32: &str = placeholder!("logf32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_LOG_F64: &str = placeholder!("logf64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_MAXNUM_NSZ_F32: &str = placeholder!("maximum_number_nsz_f32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_MAXNUM_NSZ_F64: &str = placeholder!("maximum_number_nsz_f64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_MINNUM_NSZ_F32: &str = placeholder!("minimum_number_nsz_f32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_MINNUM_NSZ_F64: &str = placeholder!("minimum_number_nsz_f64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_POWF_F32: &str = placeholder!("powf32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_POWF_F64: &str = placeholder!("powf64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_POWI_F32: &str = placeholder!("powif32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_POWI_F64: &str = placeholder!("powif64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_ROUNDEVEN_F32: &str = placeholder!("round_ties_even_f32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_ROUNDEVEN_F64: &str = placeholder!("round_ties_even_f64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_ROUND_F32: &str = placeholder!("roundf32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_ROUND_F64: &str = placeholder!("roundf64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_SINH_F32: &str = placeholder!("sinhf32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_SINH_F64: &str = placeholder!("sinhf64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_SIN_F32: &str = placeholder!("sinf32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_SIN_F64: &str = placeholder!("sinf64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_SQRT_F32: &str = placeholder!("sqrtf32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_SQRT_F64: &str = placeholder!("sqrtf64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_TANH_F32: &str = placeholder!("tanhf32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_TANH_F64: &str = placeholder!("tanhf64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_TAN_F32: &str = placeholder!("tanf32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_TAN_F64: &str = placeholder!("tanf64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_TGAMMA_F32: &str = placeholder!("tgammaf32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_TGAMMA_F64: &str = placeholder!("tgammaf64"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_TRUNC_F32: &str = placeholder!("truncf32"); +/// Internal math placeholder generated from the libdevice shard. +pub const CALLEE_TRUNC_F64: &str = placeholder!("truncf64"); + +/// Shared math identity. Semantic expansions remain in the importer and lowerer. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RustFloatMathIntrinsic { + AcosF32, + AcosF64, + AcoshF32, + AcoshF64, + AsinF32, + AsinF64, + AsinhF32, + AsinhF64, + Atan2F32, + Atan2F64, + AtanF32, + AtanF64, + AtanhF32, + AtanhF64, + CbrtF32, + CbrtF64, + CeilF32, + CeilF64, + CopysignF32, + CopysignF64, + CosF32, + CosF64, + CoshF32, + CoshF64, + Exp2F32, + Exp2F64, + ExpF32, + ExpF64, + Expm1F32, + Expm1F64, + Fabs, + FloorF32, + FloorF64, + FmaF32, + FmaF64, + FmuladdF32, + FmuladdF64, + FrexpF32, + FrexpF64, + HypotF32, + HypotF64, + IlogbF32, + IlogbF64, + LdexpF32, + LdexpF64, + Log10F32, + Log10F64, + Log1pF32, + Log1pF64, + Log2F32, + Log2F64, + LogF32, + LogF64, + MaxNumNszF32, + MaxNumNszF64, + MinNumNszF32, + MinNumNszF64, + PowfF32, + PowfF64, + PowiF32, + PowiF64, + RoundF32, + RoundF64, + RoundevenF32, + RoundevenF64, + SinF32, + SinF64, + SinhF32, + SinhF64, + SqrtF32, + SqrtF64, + TanF32, + TanF64, + TanhF32, + TanhF64, + TgammaF32, + TgammaF64, + TruncF32, + TruncF64, + FaddFast, + FsubFast, + FmulFast, + FdivFast, + FremFast, +} + +/// Whether a math placeholder needs libdevice under the selected backend. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MathLink { + Always, + Libnvvm, + Never, +} + +impl RustFloatMathIntrinsic { + /// Recognize exact core, cmath, or libm compiler paths. + pub fn from_core_path(name: &str) -> Option { + match name { + "core::intrinsics::ceilf32" => Some(Self::CeilF32), + "core::intrinsics::ceilf64" => Some(Self::CeilF64), + "core::intrinsics::copysignf32" => Some(Self::CopysignF32), + "core::intrinsics::copysignf64" => Some(Self::CopysignF64), + "core::intrinsics::cosf32" => Some(Self::CosF32), + "core::intrinsics::cosf64" => Some(Self::CosF64), + "core::intrinsics::exp2f32" => Some(Self::Exp2F32), + "core::intrinsics::exp2f64" => Some(Self::Exp2F64), + "core::intrinsics::expf32" => Some(Self::ExpF32), + "core::intrinsics::expf64" => Some(Self::ExpF64), + "core::intrinsics::fabs" => Some(Self::Fabs), + "core::intrinsics::floorf32" => Some(Self::FloorF32), + "core::intrinsics::floorf64" => Some(Self::FloorF64), + "core::intrinsics::fmaf32" => Some(Self::FmaF32), + "core::intrinsics::fmaf64" => Some(Self::FmaF64), + "core::intrinsics::fmuladdf32" => Some(Self::FmuladdF32), + "core::intrinsics::fmuladdf64" => Some(Self::FmuladdF64), + "core::intrinsics::log10f32" => Some(Self::Log10F32), + "core::intrinsics::log10f64" => Some(Self::Log10F64), + "core::intrinsics::log2f32" => Some(Self::Log2F32), + "core::intrinsics::log2f64" => Some(Self::Log2F64), + "core::intrinsics::logf32" => Some(Self::LogF32), + "core::intrinsics::logf64" => Some(Self::LogF64), + "core::intrinsics::maximum_number_nsz_f32" => Some(Self::MaxNumNszF32), + "core::intrinsics::maximum_number_nsz_f64" => Some(Self::MaxNumNszF64), + "core::intrinsics::minimum_number_nsz_f32" => Some(Self::MinNumNszF32), + "core::intrinsics::minimum_number_nsz_f64" => Some(Self::MinNumNszF64), + "core::intrinsics::powf32" => Some(Self::PowfF32), + "core::intrinsics::powf64" => Some(Self::PowfF64), + "core::intrinsics::powif32" => Some(Self::PowiF32), + "core::intrinsics::powif64" => Some(Self::PowiF64), + "core::intrinsics::round_ties_even_f32" => Some(Self::RoundevenF32), + "core::intrinsics::round_ties_even_f64" => Some(Self::RoundevenF64), + "core::intrinsics::roundf32" => Some(Self::RoundF32), + "core::intrinsics::roundf64" => Some(Self::RoundF64), + "core::intrinsics::sinf32" => Some(Self::SinF32), + "core::intrinsics::sinf64" => Some(Self::SinF64), + "core::intrinsics::sqrtf32" => Some(Self::SqrtF32), + "core::intrinsics::sqrtf64" => Some(Self::SqrtF64), + "core::intrinsics::tanf32" => Some(Self::TanF32), + "core::intrinsics::tanf64" => Some(Self::TanF64), + "core::intrinsics::truncf32" => Some(Self::TruncF32), + "core::intrinsics::truncf64" => Some(Self::TruncF64), + "core::num::imp::libm::cbrt" => Some(Self::CbrtF64), + "core::num::imp::libm::cbrtf" => Some(Self::CbrtF32), + "std::intrinsics::ceilf32" => Some(Self::CeilF32), + "std::intrinsics::ceilf64" => Some(Self::CeilF64), + "std::intrinsics::copysignf32" => Some(Self::CopysignF32), + "std::intrinsics::copysignf64" => Some(Self::CopysignF64), + "std::intrinsics::cosf32" => Some(Self::CosF32), + "std::intrinsics::cosf64" => Some(Self::CosF64), + "std::intrinsics::exp2f32" => Some(Self::Exp2F32), + "std::intrinsics::exp2f64" => Some(Self::Exp2F64), + "std::intrinsics::expf32" => Some(Self::ExpF32), + "std::intrinsics::expf64" => Some(Self::ExpF64), + "std::intrinsics::fabs" => Some(Self::Fabs), + "std::intrinsics::floorf32" => Some(Self::FloorF32), + "std::intrinsics::floorf64" => Some(Self::FloorF64), + "std::intrinsics::fmaf32" => Some(Self::FmaF32), + "std::intrinsics::fmaf64" => Some(Self::FmaF64), + "std::intrinsics::fmuladdf32" => Some(Self::FmuladdF32), + "std::intrinsics::fmuladdf64" => Some(Self::FmuladdF64), + "std::intrinsics::log10f32" => Some(Self::Log10F32), + "std::intrinsics::log10f64" => Some(Self::Log10F64), + "std::intrinsics::log2f32" => Some(Self::Log2F32), + "std::intrinsics::log2f64" => Some(Self::Log2F64), + "std::intrinsics::logf32" => Some(Self::LogF32), + "std::intrinsics::logf64" => Some(Self::LogF64), + "std::intrinsics::maximum_number_nsz_f32" => Some(Self::MaxNumNszF32), + "std::intrinsics::maximum_number_nsz_f64" => Some(Self::MaxNumNszF64), + "std::intrinsics::minimum_number_nsz_f32" => Some(Self::MinNumNszF32), + "std::intrinsics::minimum_number_nsz_f64" => Some(Self::MinNumNszF64), + "std::intrinsics::powf32" => Some(Self::PowfF32), + "std::intrinsics::powf64" => Some(Self::PowfF64), + "std::intrinsics::powif32" => Some(Self::PowiF32), + "std::intrinsics::powif64" => Some(Self::PowiF64), + "std::intrinsics::round_ties_even_f32" => Some(Self::RoundevenF32), + "std::intrinsics::round_ties_even_f64" => Some(Self::RoundevenF64), + "std::intrinsics::roundf32" => Some(Self::RoundF32), + "std::intrinsics::roundf64" => Some(Self::RoundF64), + "std::intrinsics::sinf32" => Some(Self::SinF32), + "std::intrinsics::sinf64" => Some(Self::SinF64), + "std::intrinsics::sqrtf32" => Some(Self::SqrtF32), + "std::intrinsics::sqrtf64" => Some(Self::SqrtF64), + "std::intrinsics::tanf32" => Some(Self::TanF32), + "std::intrinsics::tanf64" => Some(Self::TanF64), + "std::intrinsics::truncf32" => Some(Self::TruncF32), + "std::intrinsics::truncf64" => Some(Self::TruncF64), + "std::sys::cmath::acos" => Some(Self::AcosF64), + "std::sys::cmath::acosf" => Some(Self::AcosF32), + "std::sys::cmath::acosh" => Some(Self::AcoshF64), + "std::sys::cmath::acoshf" => Some(Self::AcoshF32), + "std::sys::cmath::asin" => Some(Self::AsinF64), + "std::sys::cmath::asinf" => Some(Self::AsinF32), + "std::sys::cmath::asinh" => Some(Self::AsinhF64), + "std::sys::cmath::asinhf" => Some(Self::AsinhF32), + "std::sys::cmath::atan" => Some(Self::AtanF64), + "std::sys::cmath::atan2" => Some(Self::Atan2F64), + "std::sys::cmath::atan2f" => Some(Self::Atan2F32), + "std::sys::cmath::atanf" => Some(Self::AtanF32), + "std::sys::cmath::atanh" => Some(Self::AtanhF64), + "std::sys::cmath::atanhf" => Some(Self::AtanhF32), + "std::sys::cmath::cbrt" => Some(Self::CbrtF64), + "std::sys::cmath::cbrtf" => Some(Self::CbrtF32), + "std::sys::cmath::cos" => Some(Self::CosF64), + "std::sys::cmath::cosf" => Some(Self::CosF32), + "std::sys::cmath::cosh" => Some(Self::CoshF64), + "std::sys::cmath::coshf" => Some(Self::CoshF32), + "std::sys::cmath::expm1" => Some(Self::Expm1F64), + "std::sys::cmath::expm1f" => Some(Self::Expm1F32), + "std::sys::cmath::hypot" => Some(Self::HypotF64), + "std::sys::cmath::hypotf" => Some(Self::HypotF32), + "std::sys::cmath::log1p" => Some(Self::Log1pF64), + "std::sys::cmath::log1pf" => Some(Self::Log1pF32), + "std::sys::cmath::sin" => Some(Self::SinF64), + "std::sys::cmath::sinf" => Some(Self::SinF32), + "std::sys::cmath::sinh" => Some(Self::SinhF64), + "std::sys::cmath::sinhf" => Some(Self::SinhF32), + "std::sys::cmath::tan" => Some(Self::TanF64), + "std::sys::cmath::tanf" => Some(Self::TanF32), + "std::sys::cmath::tanh" => Some(Self::TanhF64), + "std::sys::cmath::tanhf" => Some(Self::TanhF32), + "std::sys::cmath::tgamma" => Some(Self::TgammaF64), + "std::sys::cmath::tgammaf" => Some(Self::TgammaF32), + "core::intrinsics::fadd_fast" | "std::intrinsics::fadd_fast" => Some(Self::FaddFast), + "core::intrinsics::fsub_fast" | "std::intrinsics::fsub_fast" => Some(Self::FsubFast), + "core::intrinsics::fmul_fast" | "std::intrinsics::fmul_fast" => Some(Self::FmulFast), + "core::intrinsics::fdiv_fast" | "std::intrinsics::fdiv_fast" => Some(Self::FdivFast), + "core::intrinsics::frem_fast" | "std::intrinsics::frem_fast" => Some(Self::FremFast), + _ => Self::from_libm_path(name), + } + } + + /// Resolve aliases only within the libm crate namespace. + pub fn from_libm_path(name: &str) -> Option { + if !is_libm_path(name) { + return None; + } + match name.rsplit("::").next().unwrap_or(name) { + "acos" => Some(Self::AcosF64), + "acosf" => Some(Self::AcosF32), + "acosh" => Some(Self::AcoshF64), + "acoshf" => Some(Self::AcoshF32), + "asin" => Some(Self::AsinF64), + "asinf" => Some(Self::AsinF32), + "asinh" => Some(Self::AsinhF64), + "asinhf" => Some(Self::AsinhF32), + "atan" => Some(Self::AtanF64), + "atan2" => Some(Self::Atan2F64), + "atan2f" => Some(Self::Atan2F32), + "atanf" => Some(Self::AtanF32), + "atanh" => Some(Self::AtanhF64), + "atanhf" => Some(Self::AtanhF32), + "cbrt" => Some(Self::CbrtF64), + "cbrtf" => Some(Self::CbrtF32), + "ceil" => Some(Self::CeilF64), + "ceilf" => Some(Self::CeilF32), + "copysign" => Some(Self::CopysignF64), + "copysignf" => Some(Self::CopysignF32), + "cos" => Some(Self::CosF64), + "cosf" => Some(Self::CosF32), + "cosh" => Some(Self::CoshF64), + "coshf" => Some(Self::CoshF32), + "exp" => Some(Self::ExpF64), + "exp2" => Some(Self::Exp2F64), + "exp2f" => Some(Self::Exp2F32), + "expf" => Some(Self::ExpF32), + "expm1" => Some(Self::Expm1F64), + "expm1f" => Some(Self::Expm1F32), + "fabs" => Some(Self::Fabs), + "fabsf" => Some(Self::Fabs), + "floor" => Some(Self::FloorF64), + "floorf" => Some(Self::FloorF32), + "fma" => Some(Self::FmaF64), + "fmaf" => Some(Self::FmaF32), + "fmax" => Some(Self::MaxNumNszF64), + "fmaxf" => Some(Self::MaxNumNszF32), + "fmin" => Some(Self::MinNumNszF64), + "fminf" => Some(Self::MinNumNszF32), + "frexp" => Some(Self::FrexpF64), + "frexpf" => Some(Self::FrexpF32), + "hypot" => Some(Self::HypotF64), + "hypotf" => Some(Self::HypotF32), + "ilogb" => Some(Self::IlogbF64), + "ilogbf" => Some(Self::IlogbF32), + "ldexp" => Some(Self::LdexpF64), + "ldexpf" => Some(Self::LdexpF32), + "log" => Some(Self::LogF64), + "log10" => Some(Self::Log10F64), + "log10f" => Some(Self::Log10F32), + "log1p" => Some(Self::Log1pF64), + "log1pf" => Some(Self::Log1pF32), + "log2" => Some(Self::Log2F64), + "log2f" => Some(Self::Log2F32), + "logf" => Some(Self::LogF32), + "pow" => Some(Self::PowfF64), + "powf" => Some(Self::PowfF32), + "rint" => Some(Self::RoundevenF64), + "rintf" => Some(Self::RoundevenF32), + "round" => Some(Self::RoundF64), + "roundeven" => Some(Self::RoundevenF64), + "roundevenf" => Some(Self::RoundevenF32), + "roundf" => Some(Self::RoundF32), + "scalbn" => Some(Self::LdexpF64), + "scalbnf" => Some(Self::LdexpF32), + "sin" => Some(Self::SinF64), + "sinf" => Some(Self::SinF32), + "sinh" => Some(Self::SinhF64), + "sinhf" => Some(Self::SinhF32), + "sqrt" => Some(Self::SqrtF64), + "sqrtf" => Some(Self::SqrtF32), + "tan" => Some(Self::TanF64), + "tanf" => Some(Self::TanF32), + "tanh" => Some(Self::TanhF64), + "tanhf" => Some(Self::TanhF32), + "tgamma" => Some(Self::TgammaF64), + "tgammaf" => Some(Self::TgammaF32), + "trunc" => Some(Self::TruncF64), + "truncf" => Some(Self::TruncF32), + _ => None, + } + } + + /// Return the placeholder emitted for this math identity. + pub fn placeholder_callee(self) -> &'static str { + match self { + Self::AcosF32 => CALLEE_ACOS_F32, + Self::AcosF64 => CALLEE_ACOS_F64, + Self::AcoshF32 => CALLEE_ACOSH_F32, + Self::AcoshF64 => CALLEE_ACOSH_F64, + Self::AsinF32 => CALLEE_ASIN_F32, + Self::AsinF64 => CALLEE_ASIN_F64, + Self::AsinhF32 => CALLEE_ASINH_F32, + Self::AsinhF64 => CALLEE_ASINH_F64, + Self::Atan2F32 => CALLEE_ATAN2_F32, + Self::Atan2F64 => CALLEE_ATAN2_F64, + Self::AtanF32 => CALLEE_ATAN_F32, + Self::AtanF64 => CALLEE_ATAN_F64, + Self::AtanhF32 => CALLEE_ATANH_F32, + Self::AtanhF64 => CALLEE_ATANH_F64, + Self::CbrtF32 => CALLEE_CBRT_F32, + Self::CbrtF64 => CALLEE_CBRT_F64, + Self::CeilF32 => CALLEE_CEIL_F32, + Self::CeilF64 => CALLEE_CEIL_F64, + Self::CopysignF32 => CALLEE_COPYSIGN_F32, + Self::CopysignF64 => CALLEE_COPYSIGN_F64, + Self::CosF32 => CALLEE_COS_F32, + Self::CosF64 => CALLEE_COS_F64, + Self::CoshF32 => CALLEE_COSH_F32, + Self::CoshF64 => CALLEE_COSH_F64, + Self::Exp2F32 => CALLEE_EXP2_F32, + Self::Exp2F64 => CALLEE_EXP2_F64, + Self::ExpF32 => CALLEE_EXP_F32, + Self::ExpF64 => CALLEE_EXP_F64, + Self::Expm1F32 => CALLEE_EXPM1_F32, + Self::Expm1F64 => CALLEE_EXPM1_F64, + Self::Fabs => CALLEE_FABS, + Self::FloorF32 => CALLEE_FLOOR_F32, + Self::FloorF64 => CALLEE_FLOOR_F64, + Self::FmaF32 => CALLEE_FMA_F32, + Self::FmaF64 => CALLEE_FMA_F64, + Self::FmuladdF32 => CALLEE_FMULADD_F32, + Self::FmuladdF64 => CALLEE_FMULADD_F64, + Self::FrexpF32 => CALLEE_FREXP_F32, + Self::FrexpF64 => CALLEE_FREXP_F64, + Self::HypotF32 => CALLEE_HYPOT_F32, + Self::HypotF64 => CALLEE_HYPOT_F64, + Self::IlogbF32 => CALLEE_ILOGB_F32, + Self::IlogbF64 => CALLEE_ILOGB_F64, + Self::LdexpF32 => CALLEE_LDEXP_F32, + Self::LdexpF64 => CALLEE_LDEXP_F64, + Self::Log10F32 => CALLEE_LOG10_F32, + Self::Log10F64 => CALLEE_LOG10_F64, + Self::Log1pF32 => CALLEE_LOG1P_F32, + Self::Log1pF64 => CALLEE_LOG1P_F64, + Self::Log2F32 => CALLEE_LOG2_F32, + Self::Log2F64 => CALLEE_LOG2_F64, + Self::LogF32 => CALLEE_LOG_F32, + Self::LogF64 => CALLEE_LOG_F64, + Self::MaxNumNszF32 => CALLEE_MAXNUM_NSZ_F32, + Self::MaxNumNszF64 => CALLEE_MAXNUM_NSZ_F64, + Self::MinNumNszF32 => CALLEE_MINNUM_NSZ_F32, + Self::MinNumNszF64 => CALLEE_MINNUM_NSZ_F64, + Self::PowfF32 => CALLEE_POWF_F32, + Self::PowfF64 => CALLEE_POWF_F64, + Self::PowiF32 => CALLEE_POWI_F32, + Self::PowiF64 => CALLEE_POWI_F64, + Self::RoundF32 => CALLEE_ROUND_F32, + Self::RoundF64 => CALLEE_ROUND_F64, + Self::RoundevenF32 => CALLEE_ROUNDEVEN_F32, + Self::RoundevenF64 => CALLEE_ROUNDEVEN_F64, + Self::SinF32 => CALLEE_SIN_F32, + Self::SinF64 => CALLEE_SIN_F64, + Self::SinhF32 => CALLEE_SINH_F32, + Self::SinhF64 => CALLEE_SINH_F64, + Self::SqrtF32 => CALLEE_SQRT_F32, + Self::SqrtF64 => CALLEE_SQRT_F64, + Self::TanF32 => CALLEE_TAN_F32, + Self::TanF64 => CALLEE_TAN_F64, + Self::TanhF32 => CALLEE_TANH_F32, + Self::TanhF64 => CALLEE_TANH_F64, + Self::TgammaF32 => CALLEE_TGAMMA_F32, + Self::TgammaF64 => CALLEE_TGAMMA_F64, + Self::TruncF32 => CALLEE_TRUNC_F32, + Self::TruncF64 => CALLEE_TRUNC_F64, + Self::FaddFast => CALLEE_FADD_FAST, + Self::FsubFast => CALLEE_FSUB_FAST, + Self::FmulFast => CALLEE_FMUL_FAST, + Self::FdivFast => CALLEE_FDIV_FAST, + Self::FremFast => CALLEE_FREM_FAST, + } + } + + /// Recover an identity from an exact known placeholder. + pub fn from_placeholder_callee(callee: &str) -> Option { + match callee { + CALLEE_ACOS_F32 => Some(Self::AcosF32), + CALLEE_ACOS_F64 => Some(Self::AcosF64), + CALLEE_ACOSH_F32 => Some(Self::AcoshF32), + CALLEE_ACOSH_F64 => Some(Self::AcoshF64), + CALLEE_ASIN_F32 => Some(Self::AsinF32), + CALLEE_ASIN_F64 => Some(Self::AsinF64), + CALLEE_ASINH_F32 => Some(Self::AsinhF32), + CALLEE_ASINH_F64 => Some(Self::AsinhF64), + CALLEE_ATAN2_F32 => Some(Self::Atan2F32), + CALLEE_ATAN2_F64 => Some(Self::Atan2F64), + CALLEE_ATAN_F32 => Some(Self::AtanF32), + CALLEE_ATAN_F64 => Some(Self::AtanF64), + CALLEE_ATANH_F32 => Some(Self::AtanhF32), + CALLEE_ATANH_F64 => Some(Self::AtanhF64), + CALLEE_CBRT_F32 => Some(Self::CbrtF32), + CALLEE_CBRT_F64 => Some(Self::CbrtF64), + CALLEE_CEIL_F32 => Some(Self::CeilF32), + CALLEE_CEIL_F64 => Some(Self::CeilF64), + CALLEE_COPYSIGN_F32 => Some(Self::CopysignF32), + CALLEE_COPYSIGN_F64 => Some(Self::CopysignF64), + CALLEE_COS_F32 => Some(Self::CosF32), + CALLEE_COS_F64 => Some(Self::CosF64), + CALLEE_COSH_F32 => Some(Self::CoshF32), + CALLEE_COSH_F64 => Some(Self::CoshF64), + CALLEE_EXP2_F32 => Some(Self::Exp2F32), + CALLEE_EXP2_F64 => Some(Self::Exp2F64), + CALLEE_EXP_F32 => Some(Self::ExpF32), + CALLEE_EXP_F64 => Some(Self::ExpF64), + CALLEE_EXPM1_F32 => Some(Self::Expm1F32), + CALLEE_EXPM1_F64 => Some(Self::Expm1F64), + CALLEE_FABS => Some(Self::Fabs), + CALLEE_FLOOR_F32 => Some(Self::FloorF32), + CALLEE_FLOOR_F64 => Some(Self::FloorF64), + CALLEE_FMA_F32 => Some(Self::FmaF32), + CALLEE_FMA_F64 => Some(Self::FmaF64), + CALLEE_FMULADD_F32 => Some(Self::FmuladdF32), + CALLEE_FMULADD_F64 => Some(Self::FmuladdF64), + CALLEE_FREXP_F32 => Some(Self::FrexpF32), + CALLEE_FREXP_F64 => Some(Self::FrexpF64), + CALLEE_HYPOT_F32 => Some(Self::HypotF32), + CALLEE_HYPOT_F64 => Some(Self::HypotF64), + CALLEE_ILOGB_F32 => Some(Self::IlogbF32), + CALLEE_ILOGB_F64 => Some(Self::IlogbF64), + CALLEE_LDEXP_F32 => Some(Self::LdexpF32), + CALLEE_LDEXP_F64 => Some(Self::LdexpF64), + CALLEE_LOG10_F32 => Some(Self::Log10F32), + CALLEE_LOG10_F64 => Some(Self::Log10F64), + CALLEE_LOG1P_F32 => Some(Self::Log1pF32), + CALLEE_LOG1P_F64 => Some(Self::Log1pF64), + CALLEE_LOG2_F32 => Some(Self::Log2F32), + CALLEE_LOG2_F64 => Some(Self::Log2F64), + CALLEE_LOG_F32 => Some(Self::LogF32), + CALLEE_LOG_F64 => Some(Self::LogF64), + CALLEE_MAXNUM_NSZ_F32 => Some(Self::MaxNumNszF32), + CALLEE_MAXNUM_NSZ_F64 => Some(Self::MaxNumNszF64), + CALLEE_MINNUM_NSZ_F32 => Some(Self::MinNumNszF32), + CALLEE_MINNUM_NSZ_F64 => Some(Self::MinNumNszF64), + CALLEE_POWF_F32 => Some(Self::PowfF32), + CALLEE_POWF_F64 => Some(Self::PowfF64), + CALLEE_POWI_F32 => Some(Self::PowiF32), + CALLEE_POWI_F64 => Some(Self::PowiF64), + CALLEE_ROUND_F32 => Some(Self::RoundF32), + CALLEE_ROUND_F64 => Some(Self::RoundF64), + CALLEE_ROUNDEVEN_F32 => Some(Self::RoundevenF32), + CALLEE_ROUNDEVEN_F64 => Some(Self::RoundevenF64), + CALLEE_SIN_F32 => Some(Self::SinF32), + CALLEE_SIN_F64 => Some(Self::SinF64), + CALLEE_SINH_F32 => Some(Self::SinhF32), + CALLEE_SINH_F64 => Some(Self::SinhF64), + CALLEE_SQRT_F32 => Some(Self::SqrtF32), + CALLEE_SQRT_F64 => Some(Self::SqrtF64), + CALLEE_TAN_F32 => Some(Self::TanF32), + CALLEE_TAN_F64 => Some(Self::TanF64), + CALLEE_TANH_F32 => Some(Self::TanhF32), + CALLEE_TANH_F64 => Some(Self::TanhF64), + CALLEE_TGAMMA_F32 => Some(Self::TgammaF32), + CALLEE_TGAMMA_F64 => Some(Self::TgammaF64), + CALLEE_TRUNC_F32 => Some(Self::TruncF32), + CALLEE_TRUNC_F64 => Some(Self::TruncF64), + CALLEE_FADD_FAST => Some(Self::FaddFast), + CALLEE_FSUB_FAST => Some(Self::FsubFast), + CALLEE_FMUL_FAST => Some(Self::FmulFast), + CALLEE_FDIV_FAST => Some(Self::FdivFast), + CALLEE_FREM_FAST => Some(Self::FremFast), + _ => None, + } + } + + /// Operand count of the Rust placeholder, before C ABI adaptation. + pub fn arg_count(self) -> usize { + match self { + Self::AcosF32 => 1, + Self::AcosF64 => 1, + Self::AcoshF32 => 1, + Self::AcoshF64 => 1, + Self::AsinF32 => 1, + Self::AsinF64 => 1, + Self::AsinhF32 => 1, + Self::AsinhF64 => 1, + Self::Atan2F32 => 2, + Self::Atan2F64 => 2, + Self::AtanF32 => 1, + Self::AtanF64 => 1, + Self::AtanhF32 => 1, + Self::AtanhF64 => 1, + Self::CbrtF32 => 1, + Self::CbrtF64 => 1, + Self::CeilF32 => 1, + Self::CeilF64 => 1, + Self::CopysignF32 => 2, + Self::CopysignF64 => 2, + Self::CosF32 => 1, + Self::CosF64 => 1, + Self::CoshF32 => 1, + Self::CoshF64 => 1, + Self::Exp2F32 => 1, + Self::Exp2F64 => 1, + Self::ExpF32 => 1, + Self::ExpF64 => 1, + Self::Expm1F32 => 1, + Self::Expm1F64 => 1, + Self::Fabs => 1, + Self::FloorF32 => 1, + Self::FloorF64 => 1, + Self::FmaF32 => 3, + Self::FmaF64 => 3, + Self::FmuladdF32 => 3, + Self::FmuladdF64 => 3, + Self::FrexpF32 => 1, + Self::FrexpF64 => 1, + Self::HypotF32 => 2, + Self::HypotF64 => 2, + Self::IlogbF32 => 1, + Self::IlogbF64 => 1, + Self::LdexpF32 => 2, + Self::LdexpF64 => 2, + Self::Log10F32 => 1, + Self::Log10F64 => 1, + Self::Log1pF32 => 1, + Self::Log1pF64 => 1, + Self::Log2F32 => 1, + Self::Log2F64 => 1, + Self::LogF32 => 1, + Self::LogF64 => 1, + Self::MaxNumNszF32 => 2, + Self::MaxNumNszF64 => 2, + Self::MinNumNszF32 => 2, + Self::MinNumNszF64 => 2, + Self::PowfF32 => 2, + Self::PowfF64 => 2, + Self::PowiF32 => 2, + Self::PowiF64 => 2, + Self::RoundF32 => 1, + Self::RoundF64 => 1, + Self::RoundevenF32 => 1, + Self::RoundevenF64 => 1, + Self::SinF32 => 1, + Self::SinF64 => 1, + Self::SinhF32 => 1, + Self::SinhF64 => 1, + Self::SqrtF32 => 1, + Self::SqrtF64 => 1, + Self::TanF32 => 1, + Self::TanF64 => 1, + Self::TanhF32 => 1, + Self::TanhF64 => 1, + Self::TgammaF32 => 1, + Self::TgammaF64 => 1, + Self::TruncF32 => 1, + Self::TruncF64 => 1, + Self::FaddFast => 2, + Self::FsubFast => 2, + Self::FmulFast => 2, + Self::FdivFast => 2, + Self::FremFast => 2, + } + } + + /// Classify libdevice linking without accepting unknown placeholder prefixes. + pub fn link_requirement(self) -> MathLink { + match self { + Self::AcosF32 => MathLink::Always, + Self::AcosF64 => MathLink::Always, + Self::AcoshF32 => MathLink::Always, + Self::AcoshF64 => MathLink::Always, + Self::AsinF32 => MathLink::Always, + Self::AsinF64 => MathLink::Always, + Self::AsinhF32 => MathLink::Always, + Self::AsinhF64 => MathLink::Always, + Self::Atan2F32 => MathLink::Always, + Self::Atan2F64 => MathLink::Always, + Self::AtanF32 => MathLink::Always, + Self::AtanF64 => MathLink::Always, + Self::AtanhF32 => MathLink::Always, + Self::AtanhF64 => MathLink::Always, + Self::CbrtF32 => MathLink::Always, + Self::CbrtF64 => MathLink::Always, + Self::CeilF32 => MathLink::Libnvvm, + Self::CeilF64 => MathLink::Libnvvm, + Self::CopysignF32 => MathLink::Libnvvm, + Self::CopysignF64 => MathLink::Libnvvm, + Self::CosF32 => MathLink::Always, + Self::CosF64 => MathLink::Always, + Self::CoshF32 => MathLink::Always, + Self::CoshF64 => MathLink::Always, + Self::Exp2F32 => MathLink::Always, + Self::Exp2F64 => MathLink::Always, + Self::ExpF32 => MathLink::Always, + Self::ExpF64 => MathLink::Always, + Self::Expm1F32 => MathLink::Always, + Self::Expm1F64 => MathLink::Always, + Self::Fabs => MathLink::Libnvvm, + Self::FloorF32 => MathLink::Libnvvm, + Self::FloorF64 => MathLink::Libnvvm, + Self::FmaF32 => MathLink::Always, + Self::FmaF64 => MathLink::Always, + Self::FmuladdF32 => MathLink::Always, + Self::FmuladdF64 => MathLink::Always, + Self::FrexpF32 => MathLink::Always, + Self::FrexpF64 => MathLink::Always, + Self::HypotF32 => MathLink::Always, + Self::HypotF64 => MathLink::Always, + Self::IlogbF32 => MathLink::Always, + Self::IlogbF64 => MathLink::Always, + Self::LdexpF32 => MathLink::Always, + Self::LdexpF64 => MathLink::Always, + Self::Log10F32 => MathLink::Always, + Self::Log10F64 => MathLink::Always, + Self::Log1pF32 => MathLink::Always, + Self::Log1pF64 => MathLink::Always, + Self::Log2F32 => MathLink::Always, + Self::Log2F64 => MathLink::Always, + Self::LogF32 => MathLink::Always, + Self::LogF64 => MathLink::Always, + Self::MaxNumNszF32 => MathLink::Never, + Self::MaxNumNszF64 => MathLink::Never, + Self::MinNumNszF32 => MathLink::Never, + Self::MinNumNszF64 => MathLink::Never, + Self::PowfF32 => MathLink::Always, + Self::PowfF64 => MathLink::Always, + Self::PowiF32 => MathLink::Always, + Self::PowiF64 => MathLink::Always, + Self::RoundF32 => MathLink::Libnvvm, + Self::RoundF64 => MathLink::Libnvvm, + Self::RoundevenF32 => MathLink::Libnvvm, + Self::RoundevenF64 => MathLink::Libnvvm, + Self::SinF32 => MathLink::Always, + Self::SinF64 => MathLink::Always, + Self::SinhF32 => MathLink::Always, + Self::SinhF64 => MathLink::Always, + Self::SqrtF32 => MathLink::Always, + Self::SqrtF64 => MathLink::Always, + Self::TanF32 => MathLink::Always, + Self::TanF64 => MathLink::Always, + Self::TanhF32 => MathLink::Always, + Self::TanhF64 => MathLink::Always, + Self::TgammaF32 => MathLink::Always, + Self::TgammaF64 => MathLink::Always, + Self::TruncF32 => MathLink::Libnvvm, + Self::TruncF64 => MathLink::Libnvvm, + Self::FaddFast => MathLink::Never, + Self::FsubFast => MathLink::Never, + Self::FmulFast => MathLink::Never, + Self::FdivFast => MathLink::Never, + Self::FremFast => MathLink::Never, + } + } + + /// Resolve the symbol; only generic placeholders consult the concrete width. + pub fn libdevice_symbol(self, is_f64: bool) -> Option<&'static str> { + match self { + Self::AcosF32 => Some("__nv_acosf"), + Self::AcosF64 => Some("__nv_acos"), + Self::AcoshF32 => Some("__nv_acoshf"), + Self::AcoshF64 => Some("__nv_acosh"), + Self::AsinF32 => Some("__nv_asinf"), + Self::AsinF64 => Some("__nv_asin"), + Self::AsinhF32 => Some("__nv_asinhf"), + Self::AsinhF64 => Some("__nv_asinh"), + Self::Atan2F32 => Some("__nv_atan2f"), + Self::Atan2F64 => Some("__nv_atan2"), + Self::AtanF32 => Some("__nv_atanf"), + Self::AtanF64 => Some("__nv_atan"), + Self::AtanhF32 => Some("__nv_atanhf"), + Self::AtanhF64 => Some("__nv_atanh"), + Self::CbrtF32 => Some("__nv_cbrtf"), + Self::CbrtF64 => Some("__nv_cbrt"), + Self::CeilF32 => Some("__nv_ceilf"), + Self::CeilF64 => Some("__nv_ceil"), + Self::CopysignF32 => Some("__nv_copysignf"), + Self::CopysignF64 => Some("__nv_copysign"), + Self::CosF32 => Some("__nv_cosf"), + Self::CosF64 => Some("__nv_cos"), + Self::CoshF32 => Some("__nv_coshf"), + Self::CoshF64 => Some("__nv_cosh"), + Self::Exp2F32 => Some("__nv_exp2f"), + Self::Exp2F64 => Some("__nv_exp2"), + Self::ExpF32 => Some("__nv_expf"), + Self::ExpF64 => Some("__nv_exp"), + Self::Expm1F32 => Some("__nv_expm1f"), + Self::Expm1F64 => Some("__nv_expm1"), + Self::Fabs => { + if is_f64 { + Some("__nv_fabs") + } else { + Some("__nv_fabsf") + } + } + Self::FloorF32 => Some("__nv_floorf"), + Self::FloorF64 => Some("__nv_floor"), + Self::FmaF32 => Some("__nv_fmaf"), + Self::FmaF64 => Some("__nv_fma"), + Self::FmuladdF32 => Some("__nv_fmaf"), + Self::FmuladdF64 => Some("__nv_fma"), + Self::FrexpF32 => Some("__nv_frexpf"), + Self::FrexpF64 => Some("__nv_frexp"), + Self::HypotF32 => Some("__nv_hypotf"), + Self::HypotF64 => Some("__nv_hypot"), + Self::IlogbF32 => Some("__nv_ilogbf"), + Self::IlogbF64 => Some("__nv_ilogb"), + Self::LdexpF32 => Some("__nv_ldexpf"), + Self::LdexpF64 => Some("__nv_ldexp"), + Self::Log10F32 => Some("__nv_log10f"), + Self::Log10F64 => Some("__nv_log10"), + Self::Log1pF32 => Some("__nv_log1pf"), + Self::Log1pF64 => Some("__nv_log1p"), + Self::Log2F32 => Some("__nv_log2f"), + Self::Log2F64 => Some("__nv_log2"), + Self::LogF32 => Some("__nv_logf"), + Self::LogF64 => Some("__nv_log"), + Self::MaxNumNszF32 => None, + Self::MaxNumNszF64 => None, + Self::MinNumNszF32 => None, + Self::MinNumNszF64 => None, + Self::PowfF32 => Some("__nv_powf"), + Self::PowfF64 => Some("__nv_pow"), + Self::PowiF32 => Some("__nv_powif"), + Self::PowiF64 => Some("__nv_powi"), + Self::RoundF32 => Some("__nv_roundf"), + Self::RoundF64 => Some("__nv_round"), + Self::RoundevenF32 => Some("__nv_rintf"), + Self::RoundevenF64 => Some("__nv_rint"), + Self::SinF32 => Some("__nv_sinf"), + Self::SinF64 => Some("__nv_sin"), + Self::SinhF32 => Some("__nv_sinhf"), + Self::SinhF64 => Some("__nv_sinh"), + Self::SqrtF32 => Some("__nv_sqrtf"), + Self::SqrtF64 => Some("__nv_sqrt"), + Self::TanF32 => Some("__nv_tanf"), + Self::TanF64 => Some("__nv_tan"), + Self::TanhF32 => Some("__nv_tanhf"), + Self::TanhF64 => Some("__nv_tanh"), + Self::TgammaF32 => Some("__nv_tgammaf"), + Self::TgammaF64 => Some("__nv_tgamma"), + Self::TruncF32 => Some("__nv_truncf"), + Self::TruncF64 => Some("__nv_trunc"), + Self::FaddFast => None, + Self::FsubFast => None, + Self::FmulFast => None, + Self::FdivFast => None, + Self::FremFast => None, + } + } + + /// Resolve the symbol; only generic placeholders consult the concrete width. + pub fn llvm_symbol(self, is_f64: bool) -> Option<&'static str> { + match self { + Self::AcosF32 => None, + Self::AcosF64 => None, + Self::AcoshF32 => None, + Self::AcoshF64 => None, + Self::AsinF32 => None, + Self::AsinF64 => None, + Self::AsinhF32 => None, + Self::AsinhF64 => None, + Self::Atan2F32 => None, + Self::Atan2F64 => None, + Self::AtanF32 => None, + Self::AtanF64 => None, + Self::AtanhF32 => None, + Self::AtanhF64 => None, + Self::CbrtF32 => None, + Self::CbrtF64 => None, + Self::CeilF32 => Some("llvm_ceil_f32"), + Self::CeilF64 => Some("llvm_ceil_f64"), + Self::CopysignF32 => Some("llvm_copysign_f32"), + Self::CopysignF64 => Some("llvm_copysign_f64"), + Self::CosF32 => None, + Self::CosF64 => None, + Self::CoshF32 => None, + Self::CoshF64 => None, + Self::Exp2F32 => None, + Self::Exp2F64 => None, + Self::ExpF32 => None, + Self::ExpF64 => None, + Self::Expm1F32 => None, + Self::Expm1F64 => None, + Self::Fabs => { + if is_f64 { + Some("llvm_fabs_f64") + } else { + Some("llvm_fabs_f32") + } + } + Self::FloorF32 => Some("llvm_floor_f32"), + Self::FloorF64 => Some("llvm_floor_f64"), + Self::FmaF32 => None, + Self::FmaF64 => None, + Self::FmuladdF32 => None, + Self::FmuladdF64 => None, + Self::FrexpF32 => None, + Self::FrexpF64 => None, + Self::HypotF32 => None, + Self::HypotF64 => None, + Self::IlogbF32 => None, + Self::IlogbF64 => None, + Self::LdexpF32 => None, + Self::LdexpF64 => None, + Self::Log10F32 => None, + Self::Log10F64 => None, + Self::Log1pF32 => None, + Self::Log1pF64 => None, + Self::Log2F32 => None, + Self::Log2F64 => None, + Self::LogF32 => None, + Self::LogF64 => None, + Self::MaxNumNszF32 => None, + Self::MaxNumNszF64 => None, + Self::MinNumNszF32 => None, + Self::MinNumNszF64 => None, + Self::PowfF32 => None, + Self::PowfF64 => None, + Self::PowiF32 => None, + Self::PowiF64 => None, + Self::RoundF32 => Some("llvm_round_f32"), + Self::RoundF64 => Some("llvm_round_f64"), + Self::RoundevenF32 => Some("llvm_roundeven_f32"), + Self::RoundevenF64 => Some("llvm_roundeven_f64"), + Self::SinF32 => None, + Self::SinF64 => None, + Self::SinhF32 => None, + Self::SinhF64 => None, + Self::SqrtF32 => None, + Self::SqrtF64 => None, + Self::TanF32 => None, + Self::TanF64 => None, + Self::TanhF32 => None, + Self::TanhF64 => None, + Self::TgammaF32 => None, + Self::TgammaF64 => None, + Self::TruncF32 => Some("llvm_trunc_f32"), + Self::TruncF64 => Some("llvm_trunc_f64"), + Self::FaddFast => None, + Self::FsubFast => None, + Self::FmulFast => None, + Self::FdivFast => None, + Self::FremFast => None, + } + } +} + +/// Whether the path belongs to the libm crate, rather than a user namespace. +pub fn is_libm_path(name: &str) -> bool { + name.split("::").next() == Some("libm") +} + +/// Exact std shims intercepted before the collector's forbidden-std guard. +pub fn is_intrinsic_lowered_cmath_shim(name: &str) -> bool { + matches!( + name, + "std::sys::cmath::acos" + | "std::sys::cmath::acosf" + | "std::sys::cmath::acosh" + | "std::sys::cmath::acoshf" + | "std::sys::cmath::asin" + | "std::sys::cmath::asinf" + | "std::sys::cmath::asinh" + | "std::sys::cmath::asinhf" + | "std::sys::cmath::atan" + | "std::sys::cmath::atan2" + | "std::sys::cmath::atan2f" + | "std::sys::cmath::atanf" + | "std::sys::cmath::atanh" + | "std::sys::cmath::atanhf" + | "std::sys::cmath::cbrt" + | "std::sys::cmath::cbrtf" + | "std::sys::cmath::cos" + | "std::sys::cmath::cosf" + | "std::sys::cmath::cosh" + | "std::sys::cmath::coshf" + | "std::sys::cmath::expm1" + | "std::sys::cmath::expm1f" + | "std::sys::cmath::hypot" + | "std::sys::cmath::hypotf" + | "std::sys::cmath::log1p" + | "std::sys::cmath::log1pf" + | "std::sys::cmath::sin" + | "std::sys::cmath::sinf" + | "std::sys::cmath::sinh" + | "std::sys::cmath::sinhf" + | "std::sys::cmath::tan" + | "std::sys::cmath::tanf" + | "std::sys::cmath::tanh" + | "std::sys::cmath::tanhf" + | "std::sys::cmath::tgamma" + | "std::sys::cmath::tgammaf" + ) +} + +/// Sincos retains its importer tuple expansion into two scalar placeholders. +pub fn libm_sincos_is_f64(name: &str) -> Option { + if !is_libm_path(name) { + return None; + } + match name.rsplit("::").next().unwrap_or(name) { + "sincos" => Some(true), + "sincosf" => Some(false), + _ => None, + } +} diff --git a/cuda-oxide/crates/dialect-mir/src/rust_intrinsics.rs b/cuda-oxide/crates/dialect-mir/src/rust_intrinsics.rs index 9b640c64ea..27cc8b2764 100644 --- a/cuda-oxide/crates/dialect-mir/src/rust_intrinsics.rs +++ b/cuda-oxide/crates/dialect-mir/src/rust_intrinsics.rs @@ -57,177 +57,8 @@ pub const CALLEE_EXACT_DIV: &str = placeholder!("exact_div"); /// and `widening_mul` on integer types. pub const CALLEE_CARRYING_MUL_ADD: &str = placeholder!("carrying_mul_add"); -/// Placeholder call used for `core::intrinsics::sqrtf32`. -pub const CALLEE_SQRT_F32: &str = placeholder!("sqrtf32"); -/// Placeholder call used for `core::intrinsics::sqrtf64`. -pub const CALLEE_SQRT_F64: &str = placeholder!("sqrtf64"); -/// Placeholder call used for `core::intrinsics::powif32`. -pub const CALLEE_POWI_F32: &str = placeholder!("powif32"); -/// Placeholder call used for `core::intrinsics::powif64`. -pub const CALLEE_POWI_F64: &str = placeholder!("powif64"); -/// Placeholder call used for `core::intrinsics::sinf32`. -pub const CALLEE_SIN_F32: &str = placeholder!("sinf32"); -/// Placeholder call used for `core::intrinsics::sinf64`. -pub const CALLEE_SIN_F64: &str = placeholder!("sinf64"); -/// Placeholder call used for `core::intrinsics::cosf32`. -pub const CALLEE_COS_F32: &str = placeholder!("cosf32"); -/// Placeholder call used for `core::intrinsics::cosf64`. -pub const CALLEE_COS_F64: &str = placeholder!("cosf64"); -/// Placeholder call used for `core::intrinsics::tanf32`. -pub const CALLEE_TAN_F32: &str = placeholder!("tanf32"); -/// Placeholder call used for `core::intrinsics::tanf64`. -pub const CALLEE_TAN_F64: &str = placeholder!("tanf64"); -/// Placeholder call used for `core::intrinsics::powf32`. -pub const CALLEE_POWF_F32: &str = placeholder!("powf32"); -/// Placeholder call used for `core::intrinsics::powf64`. -pub const CALLEE_POWF_F64: &str = placeholder!("powf64"); -/// Placeholder call used for `core::intrinsics::expf32`. -pub const CALLEE_EXP_F32: &str = placeholder!("expf32"); -/// Placeholder call used for `core::intrinsics::expf64`. -pub const CALLEE_EXP_F64: &str = placeholder!("expf64"); -/// Placeholder call used for `core::intrinsics::exp2f32`. -pub const CALLEE_EXP2_F32: &str = placeholder!("exp2f32"); -/// Placeholder call used for `core::intrinsics::exp2f64`. -pub const CALLEE_EXP2_F64: &str = placeholder!("exp2f64"); -/// Placeholder call used for `core::intrinsics::logf32`. -pub const CALLEE_LOG_F32: &str = placeholder!("logf32"); -/// Placeholder call used for `core::intrinsics::logf64`. -pub const CALLEE_LOG_F64: &str = placeholder!("logf64"); -/// Placeholder call used for `core::intrinsics::log2f32`. -pub const CALLEE_LOG2_F32: &str = placeholder!("log2f32"); -/// Placeholder call used for `core::intrinsics::log2f64`. -pub const CALLEE_LOG2_F64: &str = placeholder!("log2f64"); -/// Placeholder call used for `core::intrinsics::log10f32`. -pub const CALLEE_LOG10_F32: &str = placeholder!("log10f32"); -/// Placeholder call used for `core::intrinsics::log10f64`. -pub const CALLEE_LOG10_F64: &str = placeholder!("log10f64"); -/// Placeholder call used for `core::intrinsics::fmaf32`. -pub const CALLEE_FMA_F32: &str = placeholder!("fmaf32"); -/// Placeholder call used for `core::intrinsics::fmaf64`. -pub const CALLEE_FMA_F64: &str = placeholder!("fmaf64"); -/// Placeholder call used for `core::intrinsics::fmuladdf32`. -pub const CALLEE_FMULADD_F32: &str = placeholder!("fmuladdf32"); -/// Placeholder call used for `core::intrinsics::fmuladdf64`. -pub const CALLEE_FMULADD_F64: &str = placeholder!("fmuladdf64"); -/// Placeholder call used for `core::intrinsics::floorf32`. -pub const CALLEE_FLOOR_F32: &str = placeholder!("floorf32"); -/// Placeholder call used for `core::intrinsics::floorf64`. -pub const CALLEE_FLOOR_F64: &str = placeholder!("floorf64"); -/// Placeholder call used for `core::intrinsics::ceilf32`. -pub const CALLEE_CEIL_F32: &str = placeholder!("ceilf32"); -/// Placeholder call used for `core::intrinsics::ceilf64`. -pub const CALLEE_CEIL_F64: &str = placeholder!("ceilf64"); -/// Placeholder call used for `core::intrinsics::truncf32`. -pub const CALLEE_TRUNC_F32: &str = placeholder!("truncf32"); -/// Placeholder call used for `core::intrinsics::truncf64`. -pub const CALLEE_TRUNC_F64: &str = placeholder!("truncf64"); -/// Placeholder call used for `core::intrinsics::roundf32`. -pub const CALLEE_ROUND_F32: &str = placeholder!("roundf32"); -/// Placeholder call used for `core::intrinsics::roundf64`. -pub const CALLEE_ROUND_F64: &str = placeholder!("roundf64"); -/// Placeholder call used for `core::intrinsics::round_ties_even_f32`. -pub const CALLEE_ROUNDEVEN_F32: &str = placeholder!("round_ties_even_f32"); -/// Placeholder call used for `core::intrinsics::round_ties_even_f64`. -pub const CALLEE_ROUNDEVEN_F64: &str = placeholder!("round_ties_even_f64"); -/// Placeholder call used for generic `core::intrinsics::fabs`. -pub const CALLEE_FABS: &str = placeholder!("fabs"); -/// Placeholder call used for `core::intrinsics::copysignf32`. -pub const CALLEE_COPYSIGN_F32: &str = placeholder!("copysignf32"); -/// Placeholder call used for `core::intrinsics::copysignf64`. -pub const CALLEE_COPYSIGN_F64: &str = placeholder!("copysignf64"); -/// Placeholder call used for `core::intrinsics::maximum_number_nsz_f32` -/// (the intrinsic backing `f32::max`). -pub const CALLEE_MAXNUM_NSZ_F32: &str = placeholder!("maximum_number_nsz_f32"); -/// Placeholder call used for `core::intrinsics::maximum_number_nsz_f64` -/// (the intrinsic backing `f64::max`). -pub const CALLEE_MAXNUM_NSZ_F64: &str = placeholder!("maximum_number_nsz_f64"); -/// Placeholder call used for `core::intrinsics::minimum_number_nsz_f32` -/// (the intrinsic backing `f32::min`). -pub const CALLEE_MINNUM_NSZ_F32: &str = placeholder!("minimum_number_nsz_f32"); -/// Placeholder call used for `core::intrinsics::minimum_number_nsz_f64` -/// (the intrinsic backing `f64::min`). -pub const CALLEE_MINNUM_NSZ_F64: &str = placeholder!("minimum_number_nsz_f64"); -/// Placeholder call used for `f32::asin` / `std::sys::cmath::asinf`. -pub const CALLEE_ASIN_F32: &str = placeholder!("asinf32"); -/// Placeholder call used for `f64::asin` / `std::sys::cmath::asin`. -pub const CALLEE_ASIN_F64: &str = placeholder!("asinf64"); -/// Placeholder call used for `f32::acos` / `std::sys::cmath::acosf`. -pub const CALLEE_ACOS_F32: &str = placeholder!("acosf32"); -/// Placeholder call used for `f64::acos` / `std::sys::cmath::acos`. -pub const CALLEE_ACOS_F64: &str = placeholder!("acosf64"); -/// Placeholder call used for `f32::atan2` / `std::sys::cmath::atan2f`. -pub const CALLEE_ATAN2_F32: &str = placeholder!("atan2f32"); -/// Placeholder call used for `f64::atan2` / `std::sys::cmath::atan2`. -pub const CALLEE_ATAN2_F64: &str = placeholder!("atan2f64"); -/// Placeholder call used for `f32::atan` / `std::sys::cmath::atanf`. -pub const CALLEE_ATAN_F32: &str = placeholder!("atanf32"); -/// Placeholder call used for `f64::atan` / `std::sys::cmath::atan`. -pub const CALLEE_ATAN_F64: &str = placeholder!("atanf64"); -/// Placeholder call used for `f32::cbrt` / `std::sys::cmath::cbrtf`. -pub const CALLEE_CBRT_F32: &str = placeholder!("cbrtf32"); -/// Placeholder call used for `f64::cbrt` / `std::sys::cmath::cbrt`. -pub const CALLEE_CBRT_F64: &str = placeholder!("cbrtf64"); - -/// Placeholder call used for `f32::sinh` / `std::sys::cmath::sinhf`. -pub const CALLEE_SINH_F32: &str = placeholder!("sinhf32"); -/// Placeholder call used for `f64::sinh` / `std::sys::cmath::sinh`. -pub const CALLEE_SINH_F64: &str = placeholder!("sinhf64"); -/// Placeholder call used for `f32::cosh` / `std::sys::cmath::coshf`. -pub const CALLEE_COSH_F32: &str = placeholder!("coshf32"); -/// Placeholder call used for `f64::cosh` / `std::sys::cmath::cosh`. -pub const CALLEE_COSH_F64: &str = placeholder!("coshf64"); -/// Placeholder call used for `f32::tanh` / `std::sys::cmath::tanhf`. -pub const CALLEE_TANH_F32: &str = placeholder!("tanhf32"); -/// Placeholder call used for `f64::tanh` / `std::sys::cmath::tanh`. -pub const CALLEE_TANH_F64: &str = placeholder!("tanhf64"); -// The inverse hyperbolics were pure-Rust formulas in `std` (compositions of -// `ln`/`sqrt`/`ln_1p`). nightly-2026-08-28 (rustc e457a7b0d) reworked -// asinh/acosh into `std::sys::cmath::{asinh,acosh}{,f}` shims like the rest -// of this family, so they need placeholders of their own; atanh is still the -// pure-Rust `ln_1p` formula there and cmath declares no atanh, so its -// placeholders are defensive, for when std makes the same move. -/// Placeholder call used for `f32::asinh` / `std::sys::cmath::asinhf`. -pub const CALLEE_ASINH_F32: &str = placeholder!("asinhf32"); -/// Placeholder call used for `f64::asinh` / `std::sys::cmath::asinh`. -pub const CALLEE_ASINH_F64: &str = placeholder!("asinhf64"); -/// Placeholder call used for `f32::acosh` / `std::sys::cmath::acoshf`. -pub const CALLEE_ACOSH_F32: &str = placeholder!("acoshf32"); -/// Placeholder call used for `f64::acosh` / `std::sys::cmath::acosh`. -pub const CALLEE_ACOSH_F64: &str = placeholder!("acoshf64"); -/// Placeholder call used for `f32::atanh` / `std::sys::cmath::atanhf`. -pub const CALLEE_ATANH_F32: &str = placeholder!("atanhf32"); -/// Placeholder call used for `f64::atanh` / `std::sys::cmath::atanh`. -pub const CALLEE_ATANH_F64: &str = placeholder!("atanhf64"); -/// Placeholder call used for `f32::exp_m1` / `std::sys::cmath::expm1f`. -pub const CALLEE_EXPM1_F32: &str = placeholder!("expm1f32"); -/// Placeholder call used for `f64::exp_m1` / `std::sys::cmath::expm1`. -pub const CALLEE_EXPM1_F64: &str = placeholder!("expm1f64"); -/// Placeholder call used for `f32::ln_1p` / `std::sys::cmath::log1pf`. -pub const CALLEE_LOG1P_F32: &str = placeholder!("log1pf32"); -/// Placeholder call used for `f64::ln_1p` / `std::sys::cmath::log1p`. -pub const CALLEE_LOG1P_F64: &str = placeholder!("log1pf64"); -/// Placeholder call used for `f32::hypot` / `std::sys::cmath::hypotf` (binary). -pub const CALLEE_HYPOT_F32: &str = placeholder!("hypotf32"); -/// Placeholder call used for `f64::hypot` / `std::sys::cmath::hypot` (binary). -pub const CALLEE_HYPOT_F64: &str = placeholder!("hypotf64"); - -/// Placeholder call used for `libm::tgammaf`. -pub const CALLEE_TGAMMA_F32: &str = placeholder!("tgammaf32"); -/// Placeholder call used for `libm::tgamma`. -pub const CALLEE_TGAMMA_F64: &str = placeholder!("tgammaf64"); -/// Placeholder call used for `libm::ilogbf`. -pub const CALLEE_ILOGB_F32: &str = placeholder!("ilogbf32"); -/// Placeholder call used for `libm::ilogb`. -pub const CALLEE_ILOGB_F64: &str = placeholder!("ilogbf64"); -/// Placeholder call used for `libm::ldexpf`. -pub const CALLEE_LDEXP_F32: &str = placeholder!("ldexpf32"); -/// Placeholder call used for `libm::ldexp`. -pub const CALLEE_LDEXP_F64: &str = placeholder!("ldexpf64"); - -/// Placeholder call used for `libm::frexpf`, returning `(f32, i32)`. -pub const CALLEE_FREXP_F32: &str = placeholder!("frexpf32"); -/// Placeholder call used for `libm::frexp`, returning `(f64, i32)`. -pub const CALLEE_FREXP_F64: &str = placeholder!("frexpf64"); +// The shard owns math identities, placeholders, paths, symbols, and operand counts. +include!("generated/libdevice_math.rs"); /// Placeholder call used for `core::intrinsics::fadd_fast` (generic over float type). /// @@ -259,8 +90,7 @@ pub const CALLEE_SELECT_UNPREDICTABLE: &str = placeholder!("select_unpredictable /// malformed or future placeholders too; accepting those as an external call /// would let an unknown pseudo-call bypass its source-level signature rules. pub fn is_known_placeholder(callee: &str) -> bool { - is_libdevice_backed_placeholder(callee) - || is_backend_dependent_libdevice_placeholder(callee) + RustFloatMathIntrinsic::from_placeholder_callee(callee).is_some() || matches!( callee, CALLEE_ROTATE_LEFT @@ -276,15 +106,6 @@ pub fn is_known_placeholder(callee: &str) -> bool { | CALLEE_SATURATING_SUB | CALLEE_EXACT_DIV | CALLEE_CARRYING_MUL_ADD - | CALLEE_MAXNUM_NSZ_F32 - | CALLEE_MAXNUM_NSZ_F64 - | CALLEE_MINNUM_NSZ_F32 - | CALLEE_MINNUM_NSZ_F64 - | CALLEE_FADD_FAST - | CALLEE_FSUB_FAST - | CALLEE_FMUL_FAST - | CALLEE_FDIV_FAST - | CALLEE_FREM_FAST | CALLEE_SELECT_UNPREDICTABLE ) } @@ -312,71 +133,8 @@ pub fn is_known_placeholder(callee: &str) -> bool { /// because under LLVM 21 they propagate signaling NaNs, contradicting /// Rust's ignore-any-NaN contract; see #390.) pub fn is_libdevice_backed_placeholder(callee: &str) -> bool { - matches!( - callee, - CALLEE_SQRT_F32 - | CALLEE_SQRT_F64 - | CALLEE_POWI_F32 - | CALLEE_POWI_F64 - | CALLEE_SIN_F32 - | CALLEE_SIN_F64 - | CALLEE_COS_F32 - | CALLEE_COS_F64 - | CALLEE_TAN_F32 - | CALLEE_TAN_F64 - | CALLEE_POWF_F32 - | CALLEE_POWF_F64 - | CALLEE_EXP_F32 - | CALLEE_EXP_F64 - | CALLEE_EXP2_F32 - | CALLEE_EXP2_F64 - | CALLEE_LOG_F32 - | CALLEE_LOG_F64 - | CALLEE_LOG2_F32 - | CALLEE_LOG2_F64 - | CALLEE_LOG10_F32 - | CALLEE_LOG10_F64 - | CALLEE_FMA_F32 - | CALLEE_FMA_F64 - | CALLEE_FMULADD_F32 - | CALLEE_FMULADD_F64 - | CALLEE_ASIN_F32 - | CALLEE_ASIN_F64 - | CALLEE_ACOS_F32 - | CALLEE_ACOS_F64 - | CALLEE_ATAN2_F32 - | CALLEE_ATAN2_F64 - | CALLEE_ATAN_F32 - | CALLEE_ATAN_F64 - | CALLEE_CBRT_F32 - | CALLEE_CBRT_F64 - | CALLEE_SINH_F32 - | CALLEE_SINH_F64 - | CALLEE_COSH_F32 - | CALLEE_COSH_F64 - | CALLEE_TANH_F32 - | CALLEE_TANH_F64 - | CALLEE_ASINH_F32 - | CALLEE_ASINH_F64 - | CALLEE_ACOSH_F32 - | CALLEE_ACOSH_F64 - | CALLEE_ATANH_F32 - | CALLEE_ATANH_F64 - | CALLEE_EXPM1_F32 - | CALLEE_EXPM1_F64 - | CALLEE_LOG1P_F32 - | CALLEE_LOG1P_F64 - | CALLEE_HYPOT_F32 - | CALLEE_HYPOT_F64 - | CALLEE_TGAMMA_F32 - | CALLEE_TGAMMA_F64 - | CALLEE_ILOGB_F32 - | CALLEE_ILOGB_F64 - | CALLEE_LDEXP_F32 - | CALLEE_LDEXP_F64 - | CALLEE_FREXP_F32 - | CALLEE_FREXP_F64 - ) + RustFloatMathIntrinsic::from_placeholder_callee(callee) + .is_some_and(|intrinsic| intrinsic.link_requirement() == MathLink::Always) } /// Return whether an internal placeholder lowers to a CUDA libdevice call @@ -399,22 +157,8 @@ pub fn is_libdevice_backed_placeholder(callee: &str) -> bool { /// libdevice-backed, libdevice-backed only under libNVVM, or never /// libdevice-backed. pub fn is_backend_dependent_libdevice_placeholder(callee: &str) -> bool { - matches!( - callee, - CALLEE_FLOOR_F32 - | CALLEE_FLOOR_F64 - | CALLEE_CEIL_F32 - | CALLEE_CEIL_F64 - | CALLEE_TRUNC_F32 - | CALLEE_TRUNC_F64 - | CALLEE_ROUND_F32 - | CALLEE_ROUND_F64 - | CALLEE_ROUNDEVEN_F32 - | CALLEE_ROUNDEVEN_F64 - | CALLEE_FABS - | CALLEE_COPYSIGN_F32 - | CALLEE_COPYSIGN_F64 - ) + RustFloatMathIntrinsic::from_placeholder_callee(callee) + .is_some_and(|intrinsic| intrinsic.link_requirement() == MathLink::Libnvvm) } #[cfg(test)] diff --git a/cuda-oxide/crates/mir-importer/src/translator/terminator/intrinsics/float_math.rs b/cuda-oxide/crates/mir-importer/src/translator/terminator/intrinsics/float_math.rs index 92bbb5cef0..e6a8227167 100644 --- a/cuda-oxide/crates/mir-importer/src/translator/terminator/intrinsics/float_math.rs +++ b/cuda-oxide/crates/mir-importer/src/translator/terminator/intrinsics/float_math.rs @@ -16,577 +16,74 @@ use pliron::location::Location; use pliron::operation::Operation; use rustc_public::mir; -/// Floating-point math intrinsic from core, std, or libm. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum RustFloatMathIntrinsic { - /// `core::intrinsics::sqrtf32`. - SqrtF32, - /// `core::intrinsics::sqrtf64`. - SqrtF64, - /// `core::intrinsics::powif32`. - PowiF32, - /// `core::intrinsics::powif64`. - PowiF64, - /// `core::intrinsics::sinf32`. - SinF32, - /// `core::intrinsics::sinf64`. - SinF64, - /// `core::intrinsics::cosf32`. - CosF32, - /// `core::intrinsics::cosf64`. - CosF64, - /// `core::intrinsics::tanf32`. - TanF32, - /// `core::intrinsics::tanf64`. - TanF64, - /// `core::intrinsics::powf32`. - PowfF32, - /// `core::intrinsics::powf64`. - PowfF64, - /// `core::intrinsics::expf32`. - ExpF32, - /// `core::intrinsics::expf64`. - ExpF64, - /// `core::intrinsics::exp2f32`. - Exp2F32, - /// `core::intrinsics::exp2f64`. - Exp2F64, - /// `core::intrinsics::logf32`. - LogF32, - /// `core::intrinsics::logf64`. - LogF64, - /// `core::intrinsics::log2f32`. - Log2F32, - /// `core::intrinsics::log2f64`. - Log2F64, - /// `core::intrinsics::log10f32`. - Log10F32, - /// `core::intrinsics::log10f64`. - Log10F64, - /// `core::intrinsics::fmaf32`. - FmaF32, - /// `core::intrinsics::fmaf64`. - FmaF64, - /// `core::intrinsics::fmuladdf32`. - FmuladdF32, - /// `core::intrinsics::fmuladdf64`. - FmuladdF64, - /// `core::intrinsics::floorf32`. - FloorF32, - /// `core::intrinsics::floorf64`. - FloorF64, - /// `core::intrinsics::ceilf32`. - CeilF32, - /// `core::intrinsics::ceilf64`. - CeilF64, - /// `core::intrinsics::truncf32`. - TruncF32, - /// `core::intrinsics::truncf64`. - TruncF64, - /// `core::intrinsics::roundf32`. - RoundF32, - /// `core::intrinsics::roundf64`. - RoundF64, - /// `core::intrinsics::round_ties_even_f32`. - RoundevenF32, - /// `core::intrinsics::round_ties_even_f64`. - RoundevenF64, - /// Generic `core::intrinsics::fabs`. - Fabs, - /// `core::intrinsics::copysignf32`. - CopysignF32, - /// `core::intrinsics::copysignf64`. - CopysignF64, - /// `core::intrinsics::maximum_number_nsz_f32` (backs `f32::max`). - MaxNumNszF32, - /// `core::intrinsics::maximum_number_nsz_f64` (backs `f64::max`). - MaxNumNszF64, - /// `core::intrinsics::minimum_number_nsz_f32` (backs `f32::min`). - MinNumNszF32, - /// `core::intrinsics::minimum_number_nsz_f64` (backs `f64::min`). - MinNumNszF64, - /// `f32::asin` / `std::sys::cmath::asinf`. - AsinF32, - /// `f64::asin` / `std::sys::cmath::asin`. - AsinF64, - /// `f32::acos` / `std::sys::cmath::acosf`. - AcosF32, - /// `f64::acos` / `std::sys::cmath::acos`. - AcosF64, - /// `f32::atan2` / `std::sys::cmath::atan2f`. - Atan2F32, - /// `f64::atan2` / `std::sys::cmath::atan2`. - Atan2F64, - /// `f32::atan` / `std::sys::cmath::atanf`. - AtanF32, - /// `f64::atan` / `std::sys::cmath::atan`. - AtanF64, - /// `f32::cbrt` / `std::sys::cmath::cbrtf`. - CbrtF32, - /// `f64::cbrt` / `std::sys::cmath::cbrt`. - CbrtF64, - /// `f32::sinh` / `std::sys::cmath::sinhf`. - SinhF32, - /// `f64::sinh` / `std::sys::cmath::sinh`. - SinhF64, - /// `f32::cosh` / `std::sys::cmath::coshf`. - CoshF32, - /// `f64::cosh` / `std::sys::cmath::cosh`. - CoshF64, - /// `f32::tanh` / `std::sys::cmath::tanhf`. - TanhF32, - /// `f64::tanh` / `std::sys::cmath::tanh`. - TanhF64, - /// `f32::asinh` / `std::sys::cmath::asinhf`. - AsinhF32, - /// `f64::asinh` / `std::sys::cmath::asinh`. - AsinhF64, - /// `f32::acosh` / `std::sys::cmath::acoshf`. - AcoshF32, - /// `f64::acosh` / `std::sys::cmath::acosh`. - AcoshF64, - /// `f32::atanh` / `std::sys::cmath::atanhf`. - AtanhF32, - /// `f64::atanh` / `std::sys::cmath::atanh`. - AtanhF64, - /// `f32::exp_m1` / `std::sys::cmath::expm1f`. - Expm1F32, - /// `f64::exp_m1` / `std::sys::cmath::expm1`. - Expm1F64, - /// `f32::ln_1p` / `std::sys::cmath::log1pf`. - Log1pF32, - /// `f64::ln_1p` / `std::sys::cmath::log1p`. - Log1pF64, - /// `f32::hypot` / `std::sys::cmath::hypotf` (binary). - HypotF32, - /// `f64::hypot` / `std::sys::cmath::hypot` (binary). - HypotF64, - /// `libm::tgammaf`. - TgammaF32, - /// `libm::tgamma`. - TgammaF64, - /// `libm::ilogbf`. - IlogbF32, - /// `libm::ilogb`. - IlogbF64, - /// `libm::ldexpf`. - LdexpF32, - /// `libm::ldexp`. - LdexpF64, - /// `libm::frexpf`, returning `(f32, i32)`. - FrexpF32, - /// `libm::frexp`, returning `(f64, i32)`. - FrexpF64, - /// Generic `core::intrinsics::fadd_fast` (lowered to `llvm.fadd` + fast-math). - FaddFast, - /// Generic `core::intrinsics::fsub_fast` (lowered to `llvm.fsub` + fast-math). - FsubFast, - /// Generic `core::intrinsics::fmul_fast` (lowered to `llvm.fmul` + fast-math). - FmulFast, - /// Generic `core::intrinsics::fdiv_fast` (lowered to `llvm.fdiv` + fast-math). - FdivFast, - /// Generic `core::intrinsics::frem_fast` (lowered to `llvm.frem` + fast-math). - FremFast, -} - -impl RustFloatMathIntrinsic { - /// Resolve both the historical width-specific paths and the generic unary - /// intrinsics introduced by the pinned nightly. Generic calls carry their - /// width in rustc's monomorphized type arguments, never in a display name. - pub fn from_core_path_with_substs( - name: &str, - type_substs: &[rustc_public::ty::Ty], - ) -> Result, String> { - use rustc_public::ty::{RigidTy, TyKind}; - - if let Some(intrinsic) = Self::from_core_path(name) { - return Ok(Some(intrinsic)); - } - let float_ty = match type_substs { - [ty] => match ty.kind() { - TyKind::RigidTy(RigidTy::Float(float_ty)) => Some(float_ty), - _ => None, - }, - _ => None, - }; - Self::from_generic_core_path(name, float_ty) - } - - fn from_generic_core_path( - name: &str, - float_ty: Option, - ) -> Result, String> { - use rustc_public::ty::FloatTy; - - // These exact declarations are `fn(T) -> T`. - // Reuse the existing f32/f64 placeholders and libdevice lowerings; - // unsupported widths must not be silently narrowed. - let (f32_intrinsic, f64_intrinsic) = match name { - "core::intrinsics::sin" | "std::intrinsics::sin" => (Self::SinF32, Self::SinF64), - "core::intrinsics::cos" | "std::intrinsics::cos" => (Self::CosF32, Self::CosF64), - "core::intrinsics::exp" | "std::intrinsics::exp" => (Self::ExpF32, Self::ExpF64), - "core::intrinsics::exp2" | "std::intrinsics::exp2" => (Self::Exp2F32, Self::Exp2F64), - "core::intrinsics::log" | "std::intrinsics::log" => (Self::LogF32, Self::LogF64), - "core::intrinsics::log2" | "std::intrinsics::log2" => (Self::Log2F32, Self::Log2F64), - "core::intrinsics::log10" | "std::intrinsics::log10" => { - (Self::Log10F32, Self::Log10F64) - } - _ => return Ok(None), - }; - match float_ty { - Some(FloatTy::F32) => Ok(Some(f32_intrinsic)), - Some(FloatTy::F64) => Ok(Some(f64_intrinsic)), - _ => Err(format!( - "generic float intrinsic `{name}` requires one f32 or f64 type argument; found {float_ty:?}" - )), - } - } +pub use dialect_mir::rust_intrinsics::{RustFloatMathIntrinsic, is_libm_path, libm_sincos_is_f64}; - /// Recognize the libcore intrinsic path that survived into MIR. - pub fn from_core_path(name: &str) -> Option { - match name { - "core::intrinsics::sqrtf32" | "std::intrinsics::sqrtf32" => Some(Self::SqrtF32), - "core::intrinsics::sqrtf64" | "std::intrinsics::sqrtf64" => Some(Self::SqrtF64), - "core::intrinsics::powif32" | "std::intrinsics::powif32" => Some(Self::PowiF32), - "core::intrinsics::powif64" | "std::intrinsics::powif64" => Some(Self::PowiF64), - "core::intrinsics::sinf32" | "std::intrinsics::sinf32" => Some(Self::SinF32), - "core::intrinsics::sinf64" | "std::intrinsics::sinf64" => Some(Self::SinF64), - "core::intrinsics::cosf32" | "std::intrinsics::cosf32" => Some(Self::CosF32), - "core::intrinsics::cosf64" | "std::intrinsics::cosf64" => Some(Self::CosF64), - "core::intrinsics::tanf32" | "std::intrinsics::tanf32" => Some(Self::TanF32), - "core::intrinsics::tanf64" | "std::intrinsics::tanf64" => Some(Self::TanF64), - "core::intrinsics::powf32" | "std::intrinsics::powf32" => Some(Self::PowfF32), - "core::intrinsics::powf64" | "std::intrinsics::powf64" => Some(Self::PowfF64), - "core::intrinsics::expf32" | "std::intrinsics::expf32" => Some(Self::ExpF32), - "core::intrinsics::expf64" | "std::intrinsics::expf64" => Some(Self::ExpF64), - "core::intrinsics::exp2f32" | "std::intrinsics::exp2f32" => Some(Self::Exp2F32), - "core::intrinsics::exp2f64" | "std::intrinsics::exp2f64" => Some(Self::Exp2F64), - "core::intrinsics::logf32" | "std::intrinsics::logf32" => Some(Self::LogF32), - "core::intrinsics::logf64" | "std::intrinsics::logf64" => Some(Self::LogF64), - "core::intrinsics::log2f32" | "std::intrinsics::log2f32" => Some(Self::Log2F32), - "core::intrinsics::log2f64" | "std::intrinsics::log2f64" => Some(Self::Log2F64), - "core::intrinsics::log10f32" | "std::intrinsics::log10f32" => Some(Self::Log10F32), - "core::intrinsics::log10f64" | "std::intrinsics::log10f64" => Some(Self::Log10F64), - "core::intrinsics::fmaf32" | "std::intrinsics::fmaf32" => Some(Self::FmaF32), - "core::intrinsics::fmaf64" | "std::intrinsics::fmaf64" => Some(Self::FmaF64), - "core::intrinsics::fmuladdf32" | "std::intrinsics::fmuladdf32" => { - Some(Self::FmuladdF32) - } - "core::intrinsics::fmuladdf64" | "std::intrinsics::fmuladdf64" => { - Some(Self::FmuladdF64) - } - "core::intrinsics::floorf32" | "std::intrinsics::floorf32" => Some(Self::FloorF32), - "core::intrinsics::floorf64" | "std::intrinsics::floorf64" => Some(Self::FloorF64), - "core::intrinsics::ceilf32" | "std::intrinsics::ceilf32" => Some(Self::CeilF32), - "core::intrinsics::ceilf64" | "std::intrinsics::ceilf64" => Some(Self::CeilF64), - "core::intrinsics::truncf32" | "std::intrinsics::truncf32" => Some(Self::TruncF32), - "core::intrinsics::truncf64" | "std::intrinsics::truncf64" => Some(Self::TruncF64), - "core::intrinsics::roundf32" | "std::intrinsics::roundf32" => Some(Self::RoundF32), - "core::intrinsics::roundf64" | "std::intrinsics::roundf64" => Some(Self::RoundF64), - "core::intrinsics::round_ties_even_f32" | "std::intrinsics::round_ties_even_f32" => { - Some(Self::RoundevenF32) - } - "core::intrinsics::round_ties_even_f64" | "std::intrinsics::round_ties_even_f64" => { - Some(Self::RoundevenF64) - } - "core::intrinsics::fabs" | "std::intrinsics::fabs" => Some(Self::Fabs), - "core::intrinsics::copysignf32" | "std::intrinsics::copysignf32" => { - Some(Self::CopysignF32) - } - "core::intrinsics::copysignf64" | "std::intrinsics::copysignf64" => { - Some(Self::CopysignF64) - } - "core::intrinsics::maximum_number_nsz_f32" - | "std::intrinsics::maximum_number_nsz_f32" => Some(Self::MaxNumNszF32), - "core::intrinsics::maximum_number_nsz_f64" - | "std::intrinsics::maximum_number_nsz_f64" => Some(Self::MaxNumNszF64), - "core::intrinsics::minimum_number_nsz_f32" - | "std::intrinsics::minimum_number_nsz_f32" => Some(Self::MinNumNszF32), - "core::intrinsics::minimum_number_nsz_f64" - | "std::intrinsics::minimum_number_nsz_f64" => Some(Self::MinNumNszF64), - // Trig that still routes through `std`'s libm shim. On current - // nightlies `core_float_math` moves `sin`/`cos` into core - // intrinsics (caught above), but `tan` is *not* in that feature, - // so `f{32,64}::tan()` lowers to `std::sys::cmath::tan{,f}` and - // would otherwise trip the forbidden-`std`-crate guard. Intercept - // it (and sin/cos defensively, for toolchains/builds that take the - // std path) and lower to `__nv_tan{,f}` like every other libdevice - // math call. - "std::sys::cmath::sinf" => Some(Self::SinF32), - "std::sys::cmath::sin" => Some(Self::SinF64), - "std::sys::cmath::cosf" => Some(Self::CosF32), - "std::sys::cmath::cos" => Some(Self::CosF64), - "std::sys::cmath::tanf" => Some(Self::TanF32), - "std::sys::cmath::tan" => Some(Self::TanF64), - "std::sys::cmath::asinf" => Some(Self::AsinF32), - "std::sys::cmath::asin" => Some(Self::AsinF64), - "std::sys::cmath::acosf" => Some(Self::AcosF32), - "std::sys::cmath::acos" => Some(Self::AcosF64), - "std::sys::cmath::atan2f" => Some(Self::Atan2F32), - "std::sys::cmath::atan2" => Some(Self::Atan2F64), - "std::sys::cmath::atanf" => Some(Self::AtanF32), - "std::sys::cmath::atan" => Some(Self::AtanF64), - "std::sys::cmath::cbrtf" => Some(Self::CbrtF32), - "std::sys::cmath::cbrt" => Some(Self::CbrtF64), - "core::num::imp::libm::cbrtf" => Some(Self::CbrtF32), - "core::num::imp::libm::cbrt" => Some(Self::CbrtF64), - // Hyperbolic + extended transcendentals. Like `tan`, none of - // these are in `core_float_math`, so `f{32,64}::{sinh,..,hypot}()` - // lower to their `std::sys::cmath::*` shims and would hit the - // std-crate guard without interception. - "std::sys::cmath::sinhf" => Some(Self::SinhF32), - "std::sys::cmath::sinh" => Some(Self::SinhF64), - "std::sys::cmath::coshf" => Some(Self::CoshF32), - "std::sys::cmath::cosh" => Some(Self::CoshF64), - "std::sys::cmath::tanhf" => Some(Self::TanhF32), - "std::sys::cmath::tanh" => Some(Self::TanhF64), - // Inverse hyperbolics: pure-Rust formulas in `std` before - // nightly-2026-08-28. On this nightly (rustc e457a7b0d) only - // asinh/acosh became `std::sys::cmath` shims; atanh keeps its - // pure-Rust `ln_1p` formula and cmath declares no atanh, so the - // atanh arms are defensive, for when std makes the same move. - "std::sys::cmath::asinhf" => Some(Self::AsinhF32), - "std::sys::cmath::asinh" => Some(Self::AsinhF64), - "std::sys::cmath::acoshf" => Some(Self::AcoshF32), - "std::sys::cmath::acosh" => Some(Self::AcoshF64), - "std::sys::cmath::atanhf" => Some(Self::AtanhF32), - "std::sys::cmath::atanh" => Some(Self::AtanhF64), - "std::sys::cmath::expm1f" => Some(Self::Expm1F32), - "std::sys::cmath::expm1" => Some(Self::Expm1F64), - "std::sys::cmath::log1pf" => Some(Self::Log1pF32), - "std::sys::cmath::log1p" => Some(Self::Log1pF64), - "std::sys::cmath::hypotf" => Some(Self::HypotF32), - "std::sys::cmath::hypot" => Some(Self::HypotF64), - "std::sys::cmath::tgammaf" => Some(Self::TgammaF32), - "std::sys::cmath::tgamma" => Some(Self::TgammaF64), - other => Self::from_libm_path(other).or_else(|| Self::from_fast_intrinsic_path(other)), - } - } +/// Resolve generic math intrinsics using rustc's monomorphized float type. +pub fn from_core_path_with_substs( + name: &str, + type_substs: &[rustc_public::ty::Ty], +) -> Result, String> { + use rustc_public::ty::{RigidTy, TyKind}; - /// Recognize `libm` crate float functions (the `nostd-libm` lowering used - /// by glam on nvptx, e.g. `libm::math::sqrt::sqrtf`). These dependency-rlib - /// functions frequently lack exportable MIR downstream, so we intercept them - /// and lower to the same libdevice intrinsics as the core-intrinsic forms, - /// giving hardware math instead of software libm. Match on the final path - /// segment so both the canonical (`libm::math::sqrt::sqrtf`) and re-exported - /// (`libm::sqrtf`) spellings are caught. - fn from_libm_path(name: &str) -> Option { - if !is_libm_path(name) { - return None; - } - let seg = name.rsplit("::").next().unwrap_or(name); - match seg { - "sqrtf" => Some(Self::SqrtF32), - "sqrt" => Some(Self::SqrtF64), - "sinf" => Some(Self::SinF32), - "sin" => Some(Self::SinF64), - "cosf" => Some(Self::CosF32), - "cos" => Some(Self::CosF64), - "tanf" => Some(Self::TanF32), - "tan" => Some(Self::TanF64), - "expf" => Some(Self::ExpF32), - "exp" => Some(Self::ExpF64), - "exp2f" => Some(Self::Exp2F32), - "exp2" => Some(Self::Exp2F64), - "logf" => Some(Self::LogF32), - "log" => Some(Self::LogF64), - "log2f" => Some(Self::Log2F32), - "log2" => Some(Self::Log2F64), - "log10f" => Some(Self::Log10F32), - "log10" => Some(Self::Log10F64), - // libm: `powf` is the f32 power, `pow` is the f64 power. - "powf" => Some(Self::PowfF32), - "pow" => Some(Self::PowfF64), - "floorf" => Some(Self::FloorF32), - "floor" => Some(Self::FloorF64), - "ceilf" => Some(Self::CeilF32), - "ceil" => Some(Self::CeilF64), - "truncf" => Some(Self::TruncF32), - "trunc" => Some(Self::TruncF64), - "roundf" => Some(Self::RoundF32), - "round" => Some(Self::RoundF64), - // libm's `rint` documents round-half-to-even; the device has no - // dynamic rounding mode, so it shares the roundeven lowering. - "rintf" | "roundevenf" => Some(Self::RoundevenF32), - "rint" | "roundeven" => Some(Self::RoundevenF64), - "fmaf" => Some(Self::FmaF32), - "fma" => Some(Self::FmaF64), - "fabsf" | "fabs" => Some(Self::Fabs), - "copysignf" => Some(Self::CopysignF32), - "copysign" => Some(Self::CopysignF64), - "fmaxf" => Some(Self::MaxNumNszF32), - "fmax" => Some(Self::MaxNumNszF64), - "fminf" => Some(Self::MinNumNszF32), - "fmin" => Some(Self::MinNumNszF64), - "asinf" => Some(Self::AsinF32), - "asin" => Some(Self::AsinF64), - "acosf" => Some(Self::AcosF32), - "acos" => Some(Self::AcosF64), - "atan2f" => Some(Self::Atan2F32), - "atan2" => Some(Self::Atan2F64), - "atanf" => Some(Self::AtanF32), - "atan" => Some(Self::AtanF64), - "cbrtf" => Some(Self::CbrtF32), - "cbrt" => Some(Self::CbrtF64), - "sinhf" => Some(Self::SinhF32), - "sinh" => Some(Self::SinhF64), - "coshf" => Some(Self::CoshF32), - "cosh" => Some(Self::CoshF64), - "tanhf" => Some(Self::TanhF32), - "tanh" => Some(Self::TanhF64), - "asinhf" => Some(Self::AsinhF32), - "asinh" => Some(Self::AsinhF64), - "acoshf" => Some(Self::AcoshF32), - "acosh" => Some(Self::AcoshF64), - "atanhf" => Some(Self::AtanhF32), - "atanh" => Some(Self::AtanhF64), - "expm1f" => Some(Self::Expm1F32), - "expm1" => Some(Self::Expm1F64), - "log1pf" => Some(Self::Log1pF32), - "log1p" => Some(Self::Log1pF64), - "hypotf" => Some(Self::HypotF32), - "hypot" => Some(Self::HypotF64), - "tgammaf" => Some(Self::TgammaF32), - "tgamma" => Some(Self::TgammaF64), - "ilogbf" => Some(Self::IlogbF32), - "ilogb" => Some(Self::IlogbF64), - "ldexpf" | "scalbnf" => Some(Self::LdexpF32), - "ldexp" | "scalbn" => Some(Self::LdexpF64), - "frexpf" => Some(Self::FrexpF32), - "frexp" => Some(Self::FrexpF64), - _ => None, - } + if let Some(intrinsic) = RustFloatMathIntrinsic::from_core_path(name) { + return Ok(Some(intrinsic)); } - - /// Recognize `core::intrinsics` / `std::intrinsics` generic fast-float ops. - fn from_fast_intrinsic_path(name: &str) -> Option { - match name { - // Generic finite-input arithmetic intrinsics. The FQDN carries no - // float-type suffix because they're polymorphic over `T: - // FloatPrimitive`; the float type is in the call's substs and - // recovered from the destination's `body.locals()[…].ty` at - // lowering time. - "core::intrinsics::fadd_fast" | "std::intrinsics::fadd_fast" => Some(Self::FaddFast), - "core::intrinsics::fsub_fast" | "std::intrinsics::fsub_fast" => Some(Self::FsubFast), - "core::intrinsics::fmul_fast" | "std::intrinsics::fmul_fast" => Some(Self::FmulFast), - "core::intrinsics::fdiv_fast" | "std::intrinsics::fdiv_fast" => Some(Self::FdivFast), - "core::intrinsics::frem_fast" | "std::intrinsics::frem_fast" => Some(Self::FremFast), + let float_ty = match type_substs { + [ty] => match ty.kind() { + TyKind::RigidTy(RigidTy::Float(float_ty)) => Some(float_ty), _ => None, - } - } - - /// Return the internal placeholder name used until MIR-to-LLVM lowering. - pub fn placeholder_callee(self) -> &'static str { - match self { - Self::SqrtF32 => rust_intrinsics::CALLEE_SQRT_F32, - Self::SqrtF64 => rust_intrinsics::CALLEE_SQRT_F64, - Self::PowiF32 => rust_intrinsics::CALLEE_POWI_F32, - Self::PowiF64 => rust_intrinsics::CALLEE_POWI_F64, - Self::SinF32 => rust_intrinsics::CALLEE_SIN_F32, - Self::SinF64 => rust_intrinsics::CALLEE_SIN_F64, - Self::CosF32 => rust_intrinsics::CALLEE_COS_F32, - Self::CosF64 => rust_intrinsics::CALLEE_COS_F64, - Self::TanF32 => rust_intrinsics::CALLEE_TAN_F32, - Self::TanF64 => rust_intrinsics::CALLEE_TAN_F64, - Self::PowfF32 => rust_intrinsics::CALLEE_POWF_F32, - Self::PowfF64 => rust_intrinsics::CALLEE_POWF_F64, - Self::ExpF32 => rust_intrinsics::CALLEE_EXP_F32, - Self::ExpF64 => rust_intrinsics::CALLEE_EXP_F64, - Self::Exp2F32 => rust_intrinsics::CALLEE_EXP2_F32, - Self::Exp2F64 => rust_intrinsics::CALLEE_EXP2_F64, - Self::LogF32 => rust_intrinsics::CALLEE_LOG_F32, - Self::LogF64 => rust_intrinsics::CALLEE_LOG_F64, - Self::Log2F32 => rust_intrinsics::CALLEE_LOG2_F32, - Self::Log2F64 => rust_intrinsics::CALLEE_LOG2_F64, - Self::Log10F32 => rust_intrinsics::CALLEE_LOG10_F32, - Self::Log10F64 => rust_intrinsics::CALLEE_LOG10_F64, - Self::FmaF32 => rust_intrinsics::CALLEE_FMA_F32, - Self::FmaF64 => rust_intrinsics::CALLEE_FMA_F64, - Self::FmuladdF32 => rust_intrinsics::CALLEE_FMULADD_F32, - Self::FmuladdF64 => rust_intrinsics::CALLEE_FMULADD_F64, - Self::FloorF32 => rust_intrinsics::CALLEE_FLOOR_F32, - Self::FloorF64 => rust_intrinsics::CALLEE_FLOOR_F64, - Self::CeilF32 => rust_intrinsics::CALLEE_CEIL_F32, - Self::CeilF64 => rust_intrinsics::CALLEE_CEIL_F64, - Self::TruncF32 => rust_intrinsics::CALLEE_TRUNC_F32, - Self::TruncF64 => rust_intrinsics::CALLEE_TRUNC_F64, - Self::RoundF32 => rust_intrinsics::CALLEE_ROUND_F32, - Self::RoundF64 => rust_intrinsics::CALLEE_ROUND_F64, - Self::RoundevenF32 => rust_intrinsics::CALLEE_ROUNDEVEN_F32, - Self::RoundevenF64 => rust_intrinsics::CALLEE_ROUNDEVEN_F64, - Self::Fabs => rust_intrinsics::CALLEE_FABS, - Self::CopysignF32 => rust_intrinsics::CALLEE_COPYSIGN_F32, - Self::CopysignF64 => rust_intrinsics::CALLEE_COPYSIGN_F64, - Self::MaxNumNszF32 => rust_intrinsics::CALLEE_MAXNUM_NSZ_F32, - Self::MaxNumNszF64 => rust_intrinsics::CALLEE_MAXNUM_NSZ_F64, - Self::MinNumNszF32 => rust_intrinsics::CALLEE_MINNUM_NSZ_F32, - Self::MinNumNszF64 => rust_intrinsics::CALLEE_MINNUM_NSZ_F64, - Self::AsinF32 => rust_intrinsics::CALLEE_ASIN_F32, - Self::AsinF64 => rust_intrinsics::CALLEE_ASIN_F64, - Self::AcosF32 => rust_intrinsics::CALLEE_ACOS_F32, - Self::AcosF64 => rust_intrinsics::CALLEE_ACOS_F64, - Self::Atan2F32 => rust_intrinsics::CALLEE_ATAN2_F32, - Self::Atan2F64 => rust_intrinsics::CALLEE_ATAN2_F64, - Self::AtanF32 => rust_intrinsics::CALLEE_ATAN_F32, - Self::AtanF64 => rust_intrinsics::CALLEE_ATAN_F64, - Self::CbrtF32 => rust_intrinsics::CALLEE_CBRT_F32, - Self::CbrtF64 => rust_intrinsics::CALLEE_CBRT_F64, - Self::SinhF32 => rust_intrinsics::CALLEE_SINH_F32, - Self::SinhF64 => rust_intrinsics::CALLEE_SINH_F64, - Self::CoshF32 => rust_intrinsics::CALLEE_COSH_F32, - Self::CoshF64 => rust_intrinsics::CALLEE_COSH_F64, - Self::TanhF32 => rust_intrinsics::CALLEE_TANH_F32, - Self::TanhF64 => rust_intrinsics::CALLEE_TANH_F64, - Self::AsinhF32 => rust_intrinsics::CALLEE_ASINH_F32, - Self::AsinhF64 => rust_intrinsics::CALLEE_ASINH_F64, - Self::AcoshF32 => rust_intrinsics::CALLEE_ACOSH_F32, - Self::AcoshF64 => rust_intrinsics::CALLEE_ACOSH_F64, - Self::AtanhF32 => rust_intrinsics::CALLEE_ATANH_F32, - Self::AtanhF64 => rust_intrinsics::CALLEE_ATANH_F64, - Self::Expm1F32 => rust_intrinsics::CALLEE_EXPM1_F32, - Self::Expm1F64 => rust_intrinsics::CALLEE_EXPM1_F64, - Self::Log1pF32 => rust_intrinsics::CALLEE_LOG1P_F32, - Self::Log1pF64 => rust_intrinsics::CALLEE_LOG1P_F64, - Self::HypotF32 => rust_intrinsics::CALLEE_HYPOT_F32, - Self::HypotF64 => rust_intrinsics::CALLEE_HYPOT_F64, - Self::TgammaF32 => rust_intrinsics::CALLEE_TGAMMA_F32, - Self::TgammaF64 => rust_intrinsics::CALLEE_TGAMMA_F64, - Self::IlogbF32 => rust_intrinsics::CALLEE_ILOGB_F32, - Self::IlogbF64 => rust_intrinsics::CALLEE_ILOGB_F64, - Self::LdexpF32 => rust_intrinsics::CALLEE_LDEXP_F32, - Self::LdexpF64 => rust_intrinsics::CALLEE_LDEXP_F64, - Self::FrexpF32 => rust_intrinsics::CALLEE_FREXP_F32, - Self::FrexpF64 => rust_intrinsics::CALLEE_FREXP_F64, - Self::FaddFast => rust_intrinsics::CALLEE_FADD_FAST, - Self::FsubFast => rust_intrinsics::CALLEE_FSUB_FAST, - Self::FmulFast => rust_intrinsics::CALLEE_FMUL_FAST, - Self::FdivFast => rust_intrinsics::CALLEE_FDIV_FAST, - Self::FremFast => rust_intrinsics::CALLEE_FREM_FAST, - } - } -} - -/// Whether `name` is a path rooted in the `libm` crate: the first path -/// segment must be exactly `libm`. A bare substring test would also match -/// user functions whose path merely mentions libm (e.g. -/// `my_app::libm_compat::expf`), silently replacing the user's body with a -/// libdevice call. -pub fn is_libm_path(name: &str) -> bool { - name.split("::").next() == Some("libm") + }, + _ => None, + }; + from_generic_core_path(name, float_ty) } -/// Recognize `libm::sincosf` / `libm::sincos` (glam's `nostd-libm` lowering of -/// `f32::sin_cos`). [`emit_sincos`] handles these separately by emitting -/// two scalar placeholders and constructing their tuple in the importer. -/// Returns `Some(is_f64)` when `name` is a libm sincos function. -pub fn libm_sincos_is_f64(name: &str) -> Option { - if !is_libm_path(name) { - return None; - } - match name.rsplit("::").next().unwrap_or(name) { - "sincosf" => Some(false), - "sincos" => Some(true), - _ => None, +fn from_generic_core_path( + name: &str, + float_ty: Option, +) -> Result, String> { + use rustc_public::ty::FloatTy; + + // These exact declarations are `fn(T) -> T`. + // Reuse the existing f32/f64 placeholders and libdevice lowerings; + // unsupported widths must not be silently narrowed. + let (f32_intrinsic, f64_intrinsic) = match name { + "core::intrinsics::sin" | "std::intrinsics::sin" => ( + RustFloatMathIntrinsic::SinF32, + RustFloatMathIntrinsic::SinF64, + ), + "core::intrinsics::cos" | "std::intrinsics::cos" => ( + RustFloatMathIntrinsic::CosF32, + RustFloatMathIntrinsic::CosF64, + ), + "core::intrinsics::exp" | "std::intrinsics::exp" => ( + RustFloatMathIntrinsic::ExpF32, + RustFloatMathIntrinsic::ExpF64, + ), + "core::intrinsics::exp2" | "std::intrinsics::exp2" => ( + RustFloatMathIntrinsic::Exp2F32, + RustFloatMathIntrinsic::Exp2F64, + ), + "core::intrinsics::log" | "std::intrinsics::log" => ( + RustFloatMathIntrinsic::LogF32, + RustFloatMathIntrinsic::LogF64, + ), + "core::intrinsics::log2" | "std::intrinsics::log2" => ( + RustFloatMathIntrinsic::Log2F32, + RustFloatMathIntrinsic::Log2F64, + ), + "core::intrinsics::log10" | "std::intrinsics::log10" => ( + RustFloatMathIntrinsic::Log10F32, + RustFloatMathIntrinsic::Log10F64, + ), + _ => return Ok(None), + }; + match float_ty { + Some(FloatTy::F32) => Ok(Some(f32_intrinsic)), + Some(FloatTy::F64) => Ok(Some(f64_intrinsic)), + _ => Err(format!( + "generic float intrinsic `{name}` requires one f32 or f64 type argument; found {float_ty:?}" + )), } } @@ -867,15 +364,15 @@ mod tests { for root in ["core", "std"] { let path = format!("{root}::intrinsics::{name}"); assert_eq!( - I::from_generic_core_path(&path, Some(FloatTy::F32)).unwrap(), + from_generic_core_path(&path, Some(FloatTy::F32)).unwrap(), Some(f32_variant) ); assert_eq!( - I::from_generic_core_path(&path, Some(FloatTy::F64)).unwrap(), + from_generic_core_path(&path, Some(FloatTy::F64)).unwrap(), Some(f64_variant) ); for unsupported in [None, Some(FloatTy::F16), Some(FloatTy::F128)] { - let error = I::from_generic_core_path(&path, unsupported).unwrap_err(); + let error = from_generic_core_path(&path, unsupported).unwrap_err(); assert!( error.contains("requires one f32 or f64 type argument"), "{error}" @@ -891,7 +388,7 @@ mod tests { "core::intrinsics::expf32", ] { assert_eq!( - I::from_generic_core_path(path, Some(FloatTy::F32)).unwrap(), + from_generic_core_path(path, Some(FloatTy::F32)).unwrap(), None, "{path}" ); diff --git a/cuda-oxide/crates/mir-importer/src/translator/terminator/mod.rs b/cuda-oxide/crates/mir-importer/src/translator/terminator/mod.rs index 524771dea8..6359789a88 100644 --- a/cuda-oxide/crates/mir-importer/src/translator/terminator/mod.rs +++ b/cuda-oxide/crates/mir-importer/src/translator/terminator/mod.rs @@ -1349,7 +1349,7 @@ fn translate_call( loc, TranslationErr::unsupported(format!( "libm function `{name}` is not yet mapped to a libdevice intrinsic; \ - add it to `from_libm_path` in the mir-importer float-math dispatch" + add a row to `intrinsics/overlay/libdevice_math.toml` and regenerate" )) ); } @@ -2890,11 +2890,8 @@ fn try_dispatch_intrinsic( } if let Some(intrinsic) = - intrinsics::float_math::RustFloatMathIntrinsic::from_core_path_with_substs( - name, - type_substs, - ) - .map_err(|error| input_error!(loc.clone(), TranslationErr::unsupported(error)))? + intrinsics::float_math::from_core_path_with_substs(name, type_substs) + .map_err(|error| input_error!(loc.clone(), TranslationErr::unsupported(error)))? { return Ok(Some( intrinsics::float_math::emit_rust_float_math_intrinsic( diff --git a/cuda-oxide/crates/mir-lower/src/convert/ops/call.rs b/cuda-oxide/crates/mir-lower/src/convert/ops/call.rs index 8e98b2031b..a8dcf90150 100644 --- a/cuda-oxide/crates/mir-lower/src/convert/ops/call.rs +++ b/cuda-oxide/crates/mir-lower/src/convert/ops/call.rs @@ -188,188 +188,27 @@ impl RustBigIntIntrinsic { } } -/// Internal placeholder for rustc float math intrinsics. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum RustFloatMathIntrinsic { - SqrtF32, - SqrtF64, - PowiF32, - PowiF64, - SinF32, - SinF64, - CosF32, - CosF64, - TanF32, - TanF64, - PowfF32, - PowfF64, - ExpF32, - ExpF64, - Exp2F32, - Exp2F64, - LogF32, - LogF64, - Log2F32, - Log2F64, - Log10F32, - Log10F64, - FmaF32, - FmaF64, - FmuladdF32, - FmuladdF64, - FloorF32, - FloorF64, - CeilF32, - CeilF64, - TruncF32, - TruncF64, - RoundF32, - RoundF64, - RoundevenF32, - RoundevenF64, - Fabs, - CopysignF32, - CopysignF64, - MaxNumNszF32, - MaxNumNszF64, - MinNumNszF32, - MinNumNszF64, - AsinF32, - AsinF64, - AcosF32, - AcosF64, - Atan2F32, - Atan2F64, - AtanF32, - AtanF64, - CbrtF32, - CbrtF64, - SinhF32, - SinhF64, - CoshF32, - CoshF64, - TanhF32, - TanhF64, - AsinhF32, - AsinhF64, - AcoshF32, - AcoshF64, - AtanhF32, - AtanhF64, - Expm1F32, - Expm1F64, - Log1pF32, - Log1pF64, - HypotF32, - HypotF64, - TgammaF32, - TgammaF64, - IlogbF32, - IlogbF64, - LdexpF32, - LdexpF64, - FrexpF32, - FrexpF64, - FaddFast, - FsubFast, - FmulFast, - FdivFast, - FremFast, -} +use dialect_mir::rust_intrinsics::RustFloatMathIntrinsic; -impl RustFloatMathIntrinsic { - /// Convert an importer placeholder name back into the intrinsic it represents. - fn from_placeholder_callee(callee: &str) -> Option { - match callee { - rust_intrinsics::CALLEE_SQRT_F32 => Some(Self::SqrtF32), - rust_intrinsics::CALLEE_SQRT_F64 => Some(Self::SqrtF64), - rust_intrinsics::CALLEE_POWI_F32 => Some(Self::PowiF32), - rust_intrinsics::CALLEE_POWI_F64 => Some(Self::PowiF64), - rust_intrinsics::CALLEE_SIN_F32 => Some(Self::SinF32), - rust_intrinsics::CALLEE_SIN_F64 => Some(Self::SinF64), - rust_intrinsics::CALLEE_COS_F32 => Some(Self::CosF32), - rust_intrinsics::CALLEE_COS_F64 => Some(Self::CosF64), - rust_intrinsics::CALLEE_TAN_F32 => Some(Self::TanF32), - rust_intrinsics::CALLEE_TAN_F64 => Some(Self::TanF64), - rust_intrinsics::CALLEE_POWF_F32 => Some(Self::PowfF32), - rust_intrinsics::CALLEE_POWF_F64 => Some(Self::PowfF64), - rust_intrinsics::CALLEE_EXP_F32 => Some(Self::ExpF32), - rust_intrinsics::CALLEE_EXP_F64 => Some(Self::ExpF64), - rust_intrinsics::CALLEE_EXP2_F32 => Some(Self::Exp2F32), - rust_intrinsics::CALLEE_EXP2_F64 => Some(Self::Exp2F64), - rust_intrinsics::CALLEE_LOG_F32 => Some(Self::LogF32), - rust_intrinsics::CALLEE_LOG_F64 => Some(Self::LogF64), - rust_intrinsics::CALLEE_LOG2_F32 => Some(Self::Log2F32), - rust_intrinsics::CALLEE_LOG2_F64 => Some(Self::Log2F64), - rust_intrinsics::CALLEE_LOG10_F32 => Some(Self::Log10F32), - rust_intrinsics::CALLEE_LOG10_F64 => Some(Self::Log10F64), - rust_intrinsics::CALLEE_FMA_F32 => Some(Self::FmaF32), - rust_intrinsics::CALLEE_FMA_F64 => Some(Self::FmaF64), - rust_intrinsics::CALLEE_FMULADD_F32 => Some(Self::FmuladdF32), - rust_intrinsics::CALLEE_FMULADD_F64 => Some(Self::FmuladdF64), - rust_intrinsics::CALLEE_FLOOR_F32 => Some(Self::FloorF32), - rust_intrinsics::CALLEE_FLOOR_F64 => Some(Self::FloorF64), - rust_intrinsics::CALLEE_CEIL_F32 => Some(Self::CeilF32), - rust_intrinsics::CALLEE_CEIL_F64 => Some(Self::CeilF64), - rust_intrinsics::CALLEE_TRUNC_F32 => Some(Self::TruncF32), - rust_intrinsics::CALLEE_TRUNC_F64 => Some(Self::TruncF64), - rust_intrinsics::CALLEE_ROUND_F32 => Some(Self::RoundF32), - rust_intrinsics::CALLEE_ROUND_F64 => Some(Self::RoundF64), - rust_intrinsics::CALLEE_ROUNDEVEN_F32 => Some(Self::RoundevenF32), - rust_intrinsics::CALLEE_ROUNDEVEN_F64 => Some(Self::RoundevenF64), - rust_intrinsics::CALLEE_FABS => Some(Self::Fabs), - rust_intrinsics::CALLEE_COPYSIGN_F32 => Some(Self::CopysignF32), - rust_intrinsics::CALLEE_COPYSIGN_F64 => Some(Self::CopysignF64), - rust_intrinsics::CALLEE_MAXNUM_NSZ_F32 => Some(Self::MaxNumNszF32), - rust_intrinsics::CALLEE_MAXNUM_NSZ_F64 => Some(Self::MaxNumNszF64), - rust_intrinsics::CALLEE_MINNUM_NSZ_F32 => Some(Self::MinNumNszF32), - rust_intrinsics::CALLEE_MINNUM_NSZ_F64 => Some(Self::MinNumNszF64), - rust_intrinsics::CALLEE_ASIN_F32 => Some(Self::AsinF32), - rust_intrinsics::CALLEE_ASIN_F64 => Some(Self::AsinF64), - rust_intrinsics::CALLEE_ACOS_F32 => Some(Self::AcosF32), - rust_intrinsics::CALLEE_ACOS_F64 => Some(Self::AcosF64), - rust_intrinsics::CALLEE_ATAN2_F32 => Some(Self::Atan2F32), - rust_intrinsics::CALLEE_ATAN2_F64 => Some(Self::Atan2F64), - rust_intrinsics::CALLEE_ATAN_F32 => Some(Self::AtanF32), - rust_intrinsics::CALLEE_ATAN_F64 => Some(Self::AtanF64), - rust_intrinsics::CALLEE_CBRT_F32 => Some(Self::CbrtF32), - rust_intrinsics::CALLEE_CBRT_F64 => Some(Self::CbrtF64), - rust_intrinsics::CALLEE_SINH_F32 => Some(Self::SinhF32), - rust_intrinsics::CALLEE_SINH_F64 => Some(Self::SinhF64), - rust_intrinsics::CALLEE_COSH_F32 => Some(Self::CoshF32), - rust_intrinsics::CALLEE_COSH_F64 => Some(Self::CoshF64), - rust_intrinsics::CALLEE_TANH_F32 => Some(Self::TanhF32), - rust_intrinsics::CALLEE_TANH_F64 => Some(Self::TanhF64), - rust_intrinsics::CALLEE_ASINH_F32 => Some(Self::AsinhF32), - rust_intrinsics::CALLEE_ASINH_F64 => Some(Self::AsinhF64), - rust_intrinsics::CALLEE_ACOSH_F32 => Some(Self::AcoshF32), - rust_intrinsics::CALLEE_ACOSH_F64 => Some(Self::AcoshF64), - rust_intrinsics::CALLEE_ATANH_F32 => Some(Self::AtanhF32), - rust_intrinsics::CALLEE_ATANH_F64 => Some(Self::AtanhF64), - rust_intrinsics::CALLEE_EXPM1_F32 => Some(Self::Expm1F32), - rust_intrinsics::CALLEE_EXPM1_F64 => Some(Self::Expm1F64), - rust_intrinsics::CALLEE_LOG1P_F32 => Some(Self::Log1pF32), - rust_intrinsics::CALLEE_LOG1P_F64 => Some(Self::Log1pF64), - rust_intrinsics::CALLEE_HYPOT_F32 => Some(Self::HypotF32), - rust_intrinsics::CALLEE_HYPOT_F64 => Some(Self::HypotF64), - rust_intrinsics::CALLEE_TGAMMA_F32 => Some(Self::TgammaF32), - rust_intrinsics::CALLEE_TGAMMA_F64 => Some(Self::TgammaF64), - rust_intrinsics::CALLEE_ILOGB_F32 => Some(Self::IlogbF32), - rust_intrinsics::CALLEE_ILOGB_F64 => Some(Self::IlogbF64), - rust_intrinsics::CALLEE_LDEXP_F32 => Some(Self::LdexpF32), - rust_intrinsics::CALLEE_LDEXP_F64 => Some(Self::LdexpF64), - rust_intrinsics::CALLEE_FREXP_F32 => Some(Self::FrexpF32), - rust_intrinsics::CALLEE_FREXP_F64 => Some(Self::FrexpF64), - rust_intrinsics::CALLEE_FADD_FAST => Some(Self::FaddFast), - rust_intrinsics::CALLEE_FSUB_FAST => Some(Self::FsubFast), - rust_intrinsics::CALLEE_FMUL_FAST => Some(Self::FmulFast), - rust_intrinsics::CALLEE_FDIV_FAST => Some(Self::FdivFast), - rust_intrinsics::CALLEE_FREM_FAST => Some(Self::FremFast), - _ => None, - } - } +/// Type-aware semantic lowering around the generated shared math dispatch. +trait FloatMathLowering { + fn minmax_nsz_predicate(self) -> Option; + fn libdevice_name( + self, + ctx: &Context, + result_ty: TypeHandle, + loc: pliron::location::Location, + ) -> Result<&'static str>; + fn llvm_intrinsic_name( + self, + ctx: &Context, + result_ty: TypeHandle, + loc: pliron::location::Location, + ) -> Result>; + fn fast_binop(self) -> Option; +} - /// Whether this is Rust's NaN-ignoring scalar maximum/minimum operation. +impl FloatMathLowering for RustFloatMathIntrinsic { fn minmax_nsz_predicate(self) -> Option { match self { Self::MaxNumNszF32 | Self::MaxNumNszF64 => Some(FCmpPredicateAttr::OGE), @@ -378,210 +217,36 @@ impl RustFloatMathIntrinsic { } } - /// CUDA libdevice function name for this Rust math intrinsic. fn libdevice_name( self, ctx: &Context, result_ty: TypeHandle, loc: pliron::location::Location, ) -> Result<&'static str> { - match self { - Self::SqrtF32 => Ok("__nv_sqrtf"), - Self::SqrtF64 => Ok("__nv_sqrt"), - Self::PowiF32 => Ok("__nv_powif"), - Self::PowiF64 => Ok("__nv_powi"), - Self::SinF32 => Ok("__nv_sinf"), - Self::SinF64 => Ok("__nv_sin"), - Self::CosF32 => Ok("__nv_cosf"), - Self::CosF64 => Ok("__nv_cos"), - Self::TanF32 => Ok("__nv_tanf"), - Self::TanF64 => Ok("__nv_tan"), - Self::PowfF32 => Ok("__nv_powf"), - Self::PowfF64 => Ok("__nv_pow"), - Self::ExpF32 => Ok("__nv_expf"), - Self::ExpF64 => Ok("__nv_exp"), - Self::Exp2F32 => Ok("__nv_exp2f"), - Self::Exp2F64 => Ok("__nv_exp2"), - Self::LogF32 => Ok("__nv_logf"), - Self::LogF64 => Ok("__nv_log"), - Self::Log2F32 => Ok("__nv_log2f"), - Self::Log2F64 => Ok("__nv_log2"), - Self::Log10F32 => Ok("__nv_log10f"), - Self::Log10F64 => Ok("__nv_log10"), - Self::FmaF32 | Self::FmuladdF32 => Ok("__nv_fmaf"), - Self::FmaF64 | Self::FmuladdF64 => Ok("__nv_fma"), - Self::FloorF32 => Ok("__nv_floorf"), - Self::FloorF64 => Ok("__nv_floor"), - Self::CeilF32 => Ok("__nv_ceilf"), - Self::CeilF64 => Ok("__nv_ceil"), - Self::TruncF32 => Ok("__nv_truncf"), - Self::TruncF64 => Ok("__nv_trunc"), - Self::RoundF32 => Ok("__nv_roundf"), - Self::RoundF64 => Ok("__nv_round"), - Self::RoundevenF32 => Ok("__nv_rintf"), - Self::RoundevenF64 => Ok("__nv_rint"), - Self::Fabs => fabs_libdevice_name(ctx, result_ty, loc), - Self::CopysignF32 => Ok("__nv_copysignf"), - Self::CopysignF64 => Ok("__nv_copysign"), - // `f32::max` / `f32::min` (and the f64 forms) call the `_nsz` - // intrinsics, i.e. IEEE-754 maxNum/minNum with the "no signed - // zero" relaxation. They expand to an ordered compare/select - // sequence under every intrinsic backend (the caller routes them - // through `lower_float_minmax_nsz` before any symbol lookup), so - // they must never reach a libdevice name. - Self::MaxNumNszF32 | Self::MaxNumNszF64 | Self::MinNumNszF32 | Self::MinNumNszF64 => { - pliron::input_err!( - loc, - "float max/min intrinsics lower to a compare/select \ - expansion, never to libdevice" - ) - } - Self::AsinF32 => Ok("__nv_asinf"), - Self::AsinF64 => Ok("__nv_asin"), - Self::AcosF32 => Ok("__nv_acosf"), - Self::AcosF64 => Ok("__nv_acos"), - Self::Atan2F32 => Ok("__nv_atan2f"), - Self::Atan2F64 => Ok("__nv_atan2"), - Self::AtanF32 => Ok("__nv_atanf"), - Self::AtanF64 => Ok("__nv_atan"), - Self::CbrtF32 => Ok("__nv_cbrtf"), - Self::CbrtF64 => Ok("__nv_cbrt"), - Self::SinhF32 => Ok("__nv_sinhf"), - Self::SinhF64 => Ok("__nv_sinh"), - Self::CoshF32 => Ok("__nv_coshf"), - Self::CoshF64 => Ok("__nv_cosh"), - Self::TanhF32 => Ok("__nv_tanhf"), - Self::TanhF64 => Ok("__nv_tanh"), - Self::AsinhF32 => Ok("__nv_asinhf"), - Self::AsinhF64 => Ok("__nv_asinh"), - Self::AcoshF32 => Ok("__nv_acoshf"), - Self::AcoshF64 => Ok("__nv_acosh"), - Self::AtanhF32 => Ok("__nv_atanhf"), - Self::AtanhF64 => Ok("__nv_atanh"), - Self::Expm1F32 => Ok("__nv_expm1f"), - Self::Expm1F64 => Ok("__nv_expm1"), - Self::Log1pF32 => Ok("__nv_log1pf"), - Self::Log1pF64 => Ok("__nv_log1p"), - Self::HypotF32 => Ok("__nv_hypotf"), - Self::HypotF64 => Ok("__nv_hypot"), - Self::TgammaF32 => Ok("__nv_tgammaf"), - Self::TgammaF64 => Ok("__nv_tgamma"), - Self::IlogbF32 => Ok("__nv_ilogbf"), - Self::IlogbF64 => Ok("__nv_ilogb"), - Self::LdexpF32 => Ok("__nv_ldexpf"), - Self::LdexpF64 => Ok("__nv_ldexp"), - Self::FrexpF32 => Ok("__nv_frexpf"), - Self::FrexpF64 => Ok("__nv_frexp"), - Self::FaddFast | Self::FsubFast | Self::FmulFast | Self::FdivFast | Self::FremFast => { - // The `f*_fast` intrinsics lower directly to LLVM `fadd`/`fsub`/ - // `fmul`/`fdiv`/`frem` with fast-math flags, not to a libdevice - // call. The caller (`convert_rust_float_math_intrinsic`) routes - // them through `lower_fast_binop` and never reaches this branch. - pliron::input_err!( - loc, - "f*_fast intrinsics have no libdevice equivalent; \ - they lower to llvm float binops with fast-math flags" - ) - } + if self == Self::Fabs { + return fabs_libdevice_name(ctx, result_ty, loc); } - } - - /// Number of operands expected by the Rust placeholder. - fn arg_count(self) -> usize { - match self { - Self::PowiF32 - | Self::PowiF64 - | Self::PowfF32 - | Self::PowfF64 - | Self::CopysignF32 - | Self::CopysignF64 - | Self::MaxNumNszF32 - | Self::MaxNumNszF64 - | Self::MinNumNszF32 - | Self::MinNumNszF64 - | Self::Atan2F32 - | Self::Atan2F64 - | Self::HypotF32 - | Self::HypotF64 - | Self::LdexpF32 - | Self::LdexpF64 - | Self::FaddFast - | Self::FsubFast - | Self::FmulFast - | Self::FdivFast - | Self::FremFast => 2, - Self::FmaF32 | Self::FmaF64 | Self::FmuladdF32 | Self::FmuladdF64 => 3, - _ => 1, + self.libdevice_symbol(false).ok_or_else(|| { + if self.minmax_nsz_predicate().is_some() { + pliron::input_error!(loc, "float max/min intrinsics lower to a compare/select expansion, never to libdevice") + } else { + pliron::input_error!(loc, "f*_fast intrinsics have no libdevice equivalent; they lower to llvm float binops with fast-math flags") } + }) } - /// Return the LLVM intrinsic name for operations that can bypass - /// libdevice: the rounding family plus `fabs` and `copysign`. - /// - /// The NVPTX backend selects these without a libdevice call. Measured with - /// LLVM 21.1 `llc -march=nvptx64 -mcpu=sm_80`, counting only the body: - /// - /// | Intrinsic | PTX | Instructions | - /// |-----------|-----|--------------| - /// | `floor` | `cvt.rmi` | 1 | - /// | `ceil` | `cvt.rpi` | 1 | - /// | `trunc` | `cvt.rzi` | 1 | - /// | `roundeven` | `cvt.rni` | 1 | - /// | `round` | expanded sequence | 11 (f32), 8 (f64) | - /// | `fabs` | `abs` | 1 | - /// | `copysign` | `copysign` | 1 | - /// - /// `round` is the one exception, and deliberately so. It rounds halfway - /// cases away from zero, which is not one of the four hardware `cvt` modes - /// (`rni` is ties-to-even), so no single instruction can express it. LLVM - /// expands it inline into compare/select over `cvt.rzi` rather than - /// emitting a `roundf` libcall, so routing it here still removes the - /// libdevice call without introducing an unresolved symbol. - /// - /// Using LLVM intrinsics avoids a libdevice function call and removes the - /// libNVVM dependency for kernels that only use these operations. - /// - /// Scope: `max`/`min` deliberately return `None` here. They cannot use - /// `llvm.maxnum`/`llvm.minnum` at all, because under LLVM 21 those - /// propagate signaling NaNs and so contradict Rust's contract; they lower - /// as an ordered compare/select instead (`lower_float_minmax_nsz`, see - /// #390) and never reach this table. - /// - /// Naming: pliron identifiers cannot contain dots, so intrinsic symbols - /// use the underscore encoding (`llvm_floor_f32`) that the textual `.ll` - /// exporter decodes back to the dotted LLVM name (`llvm.floor.f32`), - /// exactly like the `llvm_ctpop_i*` bit-intrinsic family above. fn llvm_intrinsic_name( self, ctx: &Context, result_ty: TypeHandle, loc: pliron::location::Location, ) -> Result> { - match self { - // Rounding: each maps to a single PTX `cvt` instruction, except - // `round`; see the note above. - Self::FloorF32 => Ok(Some("llvm_floor_f32")), - Self::FloorF64 => Ok(Some("llvm_floor_f64")), - Self::CeilF32 => Ok(Some("llvm_ceil_f32")), - Self::CeilF64 => Ok(Some("llvm_ceil_f64")), - Self::TruncF32 => Ok(Some("llvm_trunc_f32")), - Self::TruncF64 => Ok(Some("llvm_trunc_f64")), - Self::RoundF32 => Ok(Some("llvm_round_f32")), - Self::RoundF64 => Ok(Some("llvm_round_f64")), - Self::RoundevenF32 => Ok(Some("llvm_roundeven_f32")), - Self::RoundevenF64 => Ok(Some("llvm_roundeven_f64")), - // Sign operations: `llvm.fabs.*` / `llvm.copysign.*` select to - // single PTX `abs` / `copysign` instructions. `fabs` is the one - // placeholder that is generic over the float width, so its name - // dispatches on the result type. - Self::Fabs => fabs_llvm_intrinsic_name(ctx, result_ty, loc).map(Some), - Self::CopysignF32 => Ok(Some("llvm_copysign_f32")), - Self::CopysignF64 => Ok(Some("llvm_copysign_f64")), - _ => Ok(None), + if self == Self::Fabs { + return fabs_llvm_intrinsic_name(ctx, result_ty, loc).map(Some); } + Ok(self.llvm_symbol(false)) } - /// Return the equivalent fast-math binop kind, if this intrinsic is one. fn fast_binop(self) -> Option { match self { Self::FaddFast => Some(FastFloatBinop::Add), @@ -1727,17 +1392,10 @@ fn fabs_llvm_intrinsic_name( ty: TypeHandle, loc: pliron::location::Location, ) -> Result<&'static str> { - let ty_ref = ty.deref(ctx); - if ty_ref.is::() { - Ok("llvm_fabs_f32") - } else if ty_ref.is::() { - Ok("llvm_fabs_f64") - } else { - pliron::input_err!( - loc, - "expected f32 or f64 type for Rust float math intrinsic" - ) - } + let is_f64 = float_math_is_f64(ctx, ty, loc)?; + Ok(RustFloatMathIntrinsic::Fabs + .llvm_symbol(is_f64) + .expect("generated fabs LLVM symbol")) } /// Return the libdevice `fabs` entry point for the concrete float type. @@ -1746,11 +1404,23 @@ fn fabs_libdevice_name( ty: TypeHandle, loc: pliron::location::Location, ) -> Result<&'static str> { + let is_f64 = float_math_is_f64(ctx, ty, loc)?; + Ok(RustFloatMathIntrinsic::Fabs + .libdevice_symbol(is_f64) + .expect("generated fabs libdevice symbol")) +} + +/// Generic math placeholders recover their width from the concrete float type. +fn float_math_is_f64( + ctx: &Context, + ty: TypeHandle, + loc: pliron::location::Location, +) -> Result { let ty_ref = ty.deref(ctx); if ty_ref.is::() { - Ok("__nv_fabsf") + Ok(false) } else if ty_ref.is::() { - Ok("__nv_fabs") + Ok(true) } else { pliron::input_err!( loc, diff --git a/cuda-oxide/crates/rustc-codegen-cuda/src/collector.rs b/cuda-oxide/crates/rustc-codegen-cuda/src/collector.rs index 239f34060b..c76c16e9d0 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/src/collector.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/src/collector.rs @@ -727,7 +727,7 @@ fn truncate_path_for_box(fn_path: &str, width: usize) -> String { /// We never actually run `std` here: mir-importer rewrites each of these /// names to the matching NVIDIA libdevice function (`__nv_tan`, `__nv_sinh`, /// ...). This list just tells the guard "these are fine, we handle them." -/// Keep it in sync with the `std::sys::cmath` matches in float_math.rs. +/// The same generated shard owns importer dispatch and this exact allow-list. /// /// A few functions (`sin`, `cos`, `exp`, ...) take a different, allowed route /// on this toolchain: the compiler lowers them to a builtin in `core`, so @@ -736,49 +736,7 @@ fn truncate_path_for_box(fn_path: &str, width: usize) -> String { /// case a build ever takes the `std` route too. Only `std::`-prefixed names /// belong here; `core`-based shims are already allowed. fn is_intrinsic_lowered_cmath_shim(fn_path: &str) -> bool { - matches!( - fn_path, - "std::sys::cmath::sinf" - | "std::sys::cmath::sin" - | "std::sys::cmath::cosf" - | "std::sys::cmath::cos" - | "std::sys::cmath::tanf" - | "std::sys::cmath::tan" - | "std::sys::cmath::asinf" - | "std::sys::cmath::asin" - | "std::sys::cmath::acosf" - | "std::sys::cmath::acos" - | "std::sys::cmath::atan2f" - | "std::sys::cmath::atan2" - | "std::sys::cmath::atanf" - | "std::sys::cmath::atan" - | "std::sys::cmath::cbrtf" - | "std::sys::cmath::cbrt" - | "std::sys::cmath::sinhf" - | "std::sys::cmath::sinh" - | "std::sys::cmath::coshf" - | "std::sys::cmath::cosh" - | "std::sys::cmath::tanhf" - | "std::sys::cmath::tanh" - // Inverse hyperbolics: on this nightly (rustc e457a7b0d) only - // asinh/acosh are `std::sys::cmath` shims; atanh is still the - // pure-Rust `ln_1p` formula and cmath declares no atanh, so its - // entries are defensive, for when std makes the same move. - | "std::sys::cmath::asinhf" - | "std::sys::cmath::asinh" - | "std::sys::cmath::acoshf" - | "std::sys::cmath::acosh" - | "std::sys::cmath::atanhf" - | "std::sys::cmath::atanh" - | "std::sys::cmath::expm1f" - | "std::sys::cmath::expm1" - | "std::sys::cmath::log1pf" - | "std::sys::cmath::log1p" - | "std::sys::cmath::hypotf" - | "std::sys::cmath::hypot" - | "std::sys::cmath::tgammaf" - | "std::sys::cmath::tgamma" - ) + dialect_mir::rust_intrinsics::is_intrinsic_lowered_cmath_shim(fn_path) } /// `std`'s inherent float methods: `std::f32::::atan` and friends. @@ -2423,7 +2381,7 @@ impl<'tcx> DeviceCollector<'tcx> { // implementations (e.g. `libm::math::generic::sqrt::sqrt_round`) would // both be wasted work and trip importer gaps. Skip the whole crate; any // libm function we don't yet intercept will surface as a missing symbol, - // signalling that `from_libm_path` needs another entry. + // signalling that the libdevice math shard needs another row. if name_str == "libm" { return CollectDecision::SkipIntentional; } diff --git a/cuda-oxide/intrinsics/evidence/libdevice_math/README.md b/cuda-oxide/intrinsics/evidence/libdevice_math/README.md index 4ca16f6c8e..c2b371d8ef 100644 --- a/cuda-oxide/intrinsics/evidence/libdevice_math/README.md +++ b/cuda-oxide/intrinsics/evidence/libdevice_math/README.md @@ -3,8 +3,9 @@ # Libdevice math ABI capture -`cuda-13.0.json` contains 64 distinct exported function signatures referenced by -the initial libdevice math shard. They were extracted from NVIDIA's CUDA +`cuda-13.0.json` contains 74 distinct exported function signatures referenced by +the compiler math shard, including tgamma, ilogb, ldexp, frexp, and core powi. +They were extracted from NVIDIA's CUDA 13.0.88 `libdevice.10.bc` with LLVM 23.1.1 from the pinned Rust 1.99.1 toolchain. Source: [NVIDIA nvidia-nvvm 13.0.88](https://pypi.org/project/nvidia-nvvm/13.0.88/), diff --git a/cuda-oxide/intrinsics/evidence/libdevice_math/cuda-13.0.json b/cuda-oxide/intrinsics/evidence/libdevice_math/cuda-13.0.json index 6c9e8b70f7..785990eb93 100644 --- a/cuda-oxide/intrinsics/evidence/libdevice_math/cuda-13.0.json +++ b/cuda-oxide/intrinsics/evidence/libdevice_math/cuda-13.0.json @@ -39,8 +39,14 @@ "__nv_floorf": "float(float)", "__nv_fma": "double(double, double, double)", "__nv_fmaf": "float(float, float, float)", + "__nv_frexp": "double(double, ptr)", + "__nv_frexpf": "float(float, ptr)", "__nv_hypot": "double(double, double)", "__nv_hypotf": "float(float, float)", + "__nv_ilogb": "i32(double)", + "__nv_ilogbf": "i32(float)", + "__nv_ldexp": "double(double, i32)", + "__nv_ldexpf": "float(float, i32)", "__nv_log": "double(double)", "__nv_log10": "double(double)", "__nv_log10f": "float(float)", @@ -51,6 +57,8 @@ "__nv_logf": "float(float)", "__nv_pow": "double(double, double)", "__nv_powf": "float(float, float)", + "__nv_powi": "double(double, i32)", + "__nv_powif": "float(float, i32)", "__nv_rint": "double(double)", "__nv_rintf": "float(float)", "__nv_round": "double(double)", @@ -65,6 +73,8 @@ "__nv_tanf": "float(float)", "__nv_tanh": "double(double)", "__nv_tanhf": "float(float)", + "__nv_tgamma": "double(double)", + "__nv_tgammaf": "float(float)", "__nv_trunc": "double(double)", "__nv_truncf": "float(float)" } diff --git a/cuda-oxide/intrinsics/generated/libdevice_math.rs b/cuda-oxide/intrinsics/generated/libdevice_math.rs index aedf6eebed..07c4f0c0a3 100644 --- a/cuda-oxide/intrinsics/generated/libdevice_math.rs +++ b/cuda-oxide/intrinsics/generated/libdevice_math.rs @@ -4,11 +4,14 @@ */ // @generated by cuda-intrinsics-gen from overlay/libdevice_math.toml. DO NOT EDIT. -// Staged metadata; compiler dispatch is still handwritten. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct MathMapping { - pub libm: &'static str, + pub libm: Option<&'static str>, + pub width: &'static str, + pub variant: Option<&'static str>, + pub constant: Option<&'static str>, + pub paths: &'static [&'static str], pub signature: &'static str, pub placeholders: &'static [&'static str], pub cmath: Option<&'static str>, @@ -20,7 +23,73 @@ pub struct MathMapping { pub const MAPPINGS: &[MathMapping] = &[ MathMapping { - libm: "acos", + libm: None, + width: "f32", + variant: Some("FmuladdF32"), + constant: Some("CALLEE_FMULADD_F32"), + paths: &[ + "core::intrinsics::fmuladdf32", + "std::intrinsics::fmuladdf32", + ], + signature: "(f32, f32, f32) -> f32", + placeholders: &["fmuladdf32"], + cmath: None, + libdevice: Some("__nv_fmaf"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: None, + width: "f64", + variant: Some("FmuladdF64"), + constant: Some("CALLEE_FMULADD_F64"), + paths: &[ + "core::intrinsics::fmuladdf64", + "std::intrinsics::fmuladdf64", + ], + signature: "(f64, f64, f64) -> f64", + placeholders: &["fmuladdf64"], + cmath: None, + libdevice: Some("__nv_fma"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: None, + width: "f32", + variant: Some("PowiF32"), + constant: Some("CALLEE_POWI_F32"), + paths: &["core::intrinsics::powif32", "std::intrinsics::powif32"], + signature: "(f32, i32) -> f32", + placeholders: &["powif32"], + cmath: None, + libdevice: Some("__nv_powif"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: None, + width: "f64", + variant: Some("PowiF64"), + constant: Some("CALLEE_POWI_F64"), + paths: &["core::intrinsics::powif64", "std::intrinsics::powif64"], + signature: "(f64, i32) -> f64", + placeholders: &["powif64"], + cmath: None, + libdevice: Some("__nv_powi"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: Some("acos"), + width: "f64", + variant: Some("AcosF64"), + constant: Some("CALLEE_ACOS_F64"), + paths: &["std::sys::cmath::acos"], signature: "(f64) -> f64", placeholders: &["acosf64"], cmath: Some("std::sys::cmath::acos"), @@ -30,7 +99,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "acosf", + libm: Some("acosf"), + width: "f32", + variant: Some("AcosF32"), + constant: Some("CALLEE_ACOS_F32"), + paths: &["std::sys::cmath::acosf"], signature: "(f32) -> f32", placeholders: &["acosf32"], cmath: Some("std::sys::cmath::acosf"), @@ -40,7 +113,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "acosh", + libm: Some("acosh"), + width: "f64", + variant: Some("AcoshF64"), + constant: Some("CALLEE_ACOSH_F64"), + paths: &["std::sys::cmath::acosh"], signature: "(f64) -> f64", placeholders: &["acoshf64"], cmath: Some("std::sys::cmath::acosh"), @@ -50,7 +127,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "acoshf", + libm: Some("acoshf"), + width: "f32", + variant: Some("AcoshF32"), + constant: Some("CALLEE_ACOSH_F32"), + paths: &["std::sys::cmath::acoshf"], signature: "(f32) -> f32", placeholders: &["acoshf32"], cmath: Some("std::sys::cmath::acoshf"), @@ -60,7 +141,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "asin", + libm: Some("asin"), + width: "f64", + variant: Some("AsinF64"), + constant: Some("CALLEE_ASIN_F64"), + paths: &["std::sys::cmath::asin"], signature: "(f64) -> f64", placeholders: &["asinf64"], cmath: Some("std::sys::cmath::asin"), @@ -70,7 +155,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "asinf", + libm: Some("asinf"), + width: "f32", + variant: Some("AsinF32"), + constant: Some("CALLEE_ASIN_F32"), + paths: &["std::sys::cmath::asinf"], signature: "(f32) -> f32", placeholders: &["asinf32"], cmath: Some("std::sys::cmath::asinf"), @@ -80,7 +169,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "asinh", + libm: Some("asinh"), + width: "f64", + variant: Some("AsinhF64"), + constant: Some("CALLEE_ASINH_F64"), + paths: &["std::sys::cmath::asinh"], signature: "(f64) -> f64", placeholders: &["asinhf64"], cmath: Some("std::sys::cmath::asinh"), @@ -90,7 +183,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "asinhf", + libm: Some("asinhf"), + width: "f32", + variant: Some("AsinhF32"), + constant: Some("CALLEE_ASINH_F32"), + paths: &["std::sys::cmath::asinhf"], signature: "(f32) -> f32", placeholders: &["asinhf32"], cmath: Some("std::sys::cmath::asinhf"), @@ -100,7 +197,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "atan", + libm: Some("atan"), + width: "f64", + variant: Some("AtanF64"), + constant: Some("CALLEE_ATAN_F64"), + paths: &["std::sys::cmath::atan"], signature: "(f64) -> f64", placeholders: &["atanf64"], cmath: Some("std::sys::cmath::atan"), @@ -110,7 +211,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "atan2", + libm: Some("atan2"), + width: "f64", + variant: Some("Atan2F64"), + constant: Some("CALLEE_ATAN2_F64"), + paths: &["std::sys::cmath::atan2"], signature: "(f64, f64) -> f64", placeholders: &["atan2f64"], cmath: Some("std::sys::cmath::atan2"), @@ -120,7 +225,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "atan2f", + libm: Some("atan2f"), + width: "f32", + variant: Some("Atan2F32"), + constant: Some("CALLEE_ATAN2_F32"), + paths: &["std::sys::cmath::atan2f"], signature: "(f32, f32) -> f32", placeholders: &["atan2f32"], cmath: Some("std::sys::cmath::atan2f"), @@ -130,7 +239,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "atanf", + libm: Some("atanf"), + width: "f32", + variant: Some("AtanF32"), + constant: Some("CALLEE_ATAN_F32"), + paths: &["std::sys::cmath::atanf"], signature: "(f32) -> f32", placeholders: &["atanf32"], cmath: Some("std::sys::cmath::atanf"), @@ -140,7 +253,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "atanh", + libm: Some("atanh"), + width: "f64", + variant: Some("AtanhF64"), + constant: Some("CALLEE_ATANH_F64"), + paths: &["std::sys::cmath::atanh"], signature: "(f64) -> f64", placeholders: &["atanhf64"], cmath: Some("std::sys::cmath::atanh"), @@ -150,7 +267,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "atanhf", + libm: Some("atanhf"), + width: "f32", + variant: Some("AtanhF32"), + constant: Some("CALLEE_ATANH_F32"), + paths: &["std::sys::cmath::atanhf"], signature: "(f32) -> f32", placeholders: &["atanhf32"], cmath: Some("std::sys::cmath::atanhf"), @@ -160,7 +281,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "cbrt", + libm: Some("cbrt"), + width: "f64", + variant: Some("CbrtF64"), + constant: Some("CALLEE_CBRT_F64"), + paths: &["core::num::imp::libm::cbrt", "std::sys::cmath::cbrt"], signature: "(f64) -> f64", placeholders: &["cbrtf64"], cmath: Some("std::sys::cmath::cbrt"), @@ -170,7 +295,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "cbrtf", + libm: Some("cbrtf"), + width: "f32", + variant: Some("CbrtF32"), + constant: Some("CALLEE_CBRT_F32"), + paths: &["core::num::imp::libm::cbrtf", "std::sys::cmath::cbrtf"], signature: "(f32) -> f32", placeholders: &["cbrtf32"], cmath: Some("std::sys::cmath::cbrtf"), @@ -180,7 +309,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "ceil", + libm: Some("ceil"), + width: "f64", + variant: Some("CeilF64"), + constant: Some("CALLEE_CEIL_F64"), + paths: &["core::intrinsics::ceilf64", "std::intrinsics::ceilf64"], signature: "(f64) -> f64", placeholders: &["ceilf64"], cmath: None, @@ -190,7 +323,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "libnvvm", }, MathMapping { - libm: "ceilf", + libm: Some("ceilf"), + width: "f32", + variant: Some("CeilF32"), + constant: Some("CALLEE_CEIL_F32"), + paths: &["core::intrinsics::ceilf32", "std::intrinsics::ceilf32"], signature: "(f32) -> f32", placeholders: &["ceilf32"], cmath: None, @@ -200,7 +337,14 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "libnvvm", }, MathMapping { - libm: "copysign", + libm: Some("copysign"), + width: "f64", + variant: Some("CopysignF64"), + constant: Some("CALLEE_COPYSIGN_F64"), + paths: &[ + "core::intrinsics::copysignf64", + "std::intrinsics::copysignf64", + ], signature: "(f64, f64) -> f64", placeholders: &["copysignf64"], cmath: None, @@ -210,7 +354,14 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "libnvvm", }, MathMapping { - libm: "copysignf", + libm: Some("copysignf"), + width: "f32", + variant: Some("CopysignF32"), + constant: Some("CALLEE_COPYSIGN_F32"), + paths: &[ + "core::intrinsics::copysignf32", + "std::intrinsics::copysignf32", + ], signature: "(f32, f32) -> f32", placeholders: &["copysignf32"], cmath: None, @@ -220,7 +371,15 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "libnvvm", }, MathMapping { - libm: "cos", + libm: Some("cos"), + width: "f64", + variant: Some("CosF64"), + constant: Some("CALLEE_COS_F64"), + paths: &[ + "core::intrinsics::cosf64", + "std::intrinsics::cosf64", + "std::sys::cmath::cos", + ], signature: "(f64) -> f64", placeholders: &["cosf64"], cmath: Some("std::sys::cmath::cos"), @@ -230,7 +389,15 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "cosf", + libm: Some("cosf"), + width: "f32", + variant: Some("CosF32"), + constant: Some("CALLEE_COS_F32"), + paths: &[ + "core::intrinsics::cosf32", + "std::intrinsics::cosf32", + "std::sys::cmath::cosf", + ], signature: "(f32) -> f32", placeholders: &["cosf32"], cmath: Some("std::sys::cmath::cosf"), @@ -240,7 +407,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "cosh", + libm: Some("cosh"), + width: "f64", + variant: Some("CoshF64"), + constant: Some("CALLEE_COSH_F64"), + paths: &["std::sys::cmath::cosh"], signature: "(f64) -> f64", placeholders: &["coshf64"], cmath: Some("std::sys::cmath::cosh"), @@ -250,7 +421,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "coshf", + libm: Some("coshf"), + width: "f32", + variant: Some("CoshF32"), + constant: Some("CALLEE_COSH_F32"), + paths: &["std::sys::cmath::coshf"], signature: "(f32) -> f32", placeholders: &["coshf32"], cmath: Some("std::sys::cmath::coshf"), @@ -260,7 +435,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "exp", + libm: Some("exp"), + width: "f64", + variant: Some("ExpF64"), + constant: Some("CALLEE_EXP_F64"), + paths: &["core::intrinsics::expf64", "std::intrinsics::expf64"], signature: "(f64) -> f64", placeholders: &["expf64"], cmath: None, @@ -270,7 +449,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "exp2", + libm: Some("exp2"), + width: "f64", + variant: Some("Exp2F64"), + constant: Some("CALLEE_EXP2_F64"), + paths: &["core::intrinsics::exp2f64", "std::intrinsics::exp2f64"], signature: "(f64) -> f64", placeholders: &["exp2f64"], cmath: None, @@ -280,7 +463,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "exp2f", + libm: Some("exp2f"), + width: "f32", + variant: Some("Exp2F32"), + constant: Some("CALLEE_EXP2_F32"), + paths: &["core::intrinsics::exp2f32", "std::intrinsics::exp2f32"], signature: "(f32) -> f32", placeholders: &["exp2f32"], cmath: None, @@ -290,7 +477,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "expf", + libm: Some("expf"), + width: "f32", + variant: Some("ExpF32"), + constant: Some("CALLEE_EXP_F32"), + paths: &["core::intrinsics::expf32", "std::intrinsics::expf32"], signature: "(f32) -> f32", placeholders: &["expf32"], cmath: None, @@ -300,7 +491,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "expm1", + libm: Some("expm1"), + width: "f64", + variant: Some("Expm1F64"), + constant: Some("CALLEE_EXPM1_F64"), + paths: &["std::sys::cmath::expm1"], signature: "(f64) -> f64", placeholders: &["expm1f64"], cmath: Some("std::sys::cmath::expm1"), @@ -310,7 +505,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "expm1f", + libm: Some("expm1f"), + width: "f32", + variant: Some("Expm1F32"), + constant: Some("CALLEE_EXPM1_F32"), + paths: &["std::sys::cmath::expm1f"], signature: "(f32) -> f32", placeholders: &["expm1f32"], cmath: Some("std::sys::cmath::expm1f"), @@ -320,7 +519,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "fabs", + libm: Some("fabs"), + width: "f64", + variant: Some("Fabs"), + constant: Some("CALLEE_FABS"), + paths: &["core::intrinsics::fabs", "std::intrinsics::fabs"], signature: "(f64) -> f64", placeholders: &["fabs"], cmath: None, @@ -330,7 +533,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "libnvvm", }, MathMapping { - libm: "fabsf", + libm: Some("fabsf"), + width: "f32", + variant: Some("Fabs"), + constant: Some("CALLEE_FABS"), + paths: &["core::intrinsics::fabs", "std::intrinsics::fabs"], signature: "(f32) -> f32", placeholders: &["fabs"], cmath: None, @@ -340,7 +547,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "libnvvm", }, MathMapping { - libm: "floor", + libm: Some("floor"), + width: "f64", + variant: Some("FloorF64"), + constant: Some("CALLEE_FLOOR_F64"), + paths: &["core::intrinsics::floorf64", "std::intrinsics::floorf64"], signature: "(f64) -> f64", placeholders: &["floorf64"], cmath: None, @@ -350,7 +561,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "libnvvm", }, MathMapping { - libm: "floorf", + libm: Some("floorf"), + width: "f32", + variant: Some("FloorF32"), + constant: Some("CALLEE_FLOOR_F32"), + paths: &["core::intrinsics::floorf32", "std::intrinsics::floorf32"], signature: "(f32) -> f32", placeholders: &["floorf32"], cmath: None, @@ -360,7 +575,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "libnvvm", }, MathMapping { - libm: "fma", + libm: Some("fma"), + width: "f64", + variant: Some("FmaF64"), + constant: Some("CALLEE_FMA_F64"), + paths: &["core::intrinsics::fmaf64", "std::intrinsics::fmaf64"], signature: "(f64, f64, f64) -> f64", placeholders: &["fmaf64"], cmath: None, @@ -370,7 +589,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "fmaf", + libm: Some("fmaf"), + width: "f32", + variant: Some("FmaF32"), + constant: Some("CALLEE_FMA_F32"), + paths: &["core::intrinsics::fmaf32", "std::intrinsics::fmaf32"], signature: "(f32, f32, f32) -> f32", placeholders: &["fmaf32"], cmath: None, @@ -380,7 +603,14 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "fmax", + libm: Some("fmax"), + width: "f64", + variant: Some("MaxNumNszF64"), + constant: Some("CALLEE_MAXNUM_NSZ_F64"), + paths: &[ + "core::intrinsics::maximum_number_nsz_f64", + "std::intrinsics::maximum_number_nsz_f64", + ], signature: "(f64, f64) -> f64", placeholders: &["maximum_number_nsz_f64"], cmath: None, @@ -390,7 +620,14 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "never", }, MathMapping { - libm: "fmaxf", + libm: Some("fmaxf"), + width: "f32", + variant: Some("MaxNumNszF32"), + constant: Some("CALLEE_MAXNUM_NSZ_F32"), + paths: &[ + "core::intrinsics::maximum_number_nsz_f32", + "std::intrinsics::maximum_number_nsz_f32", + ], signature: "(f32, f32) -> f32", placeholders: &["maximum_number_nsz_f32"], cmath: None, @@ -400,7 +637,14 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "never", }, MathMapping { - libm: "fmin", + libm: Some("fmin"), + width: "f64", + variant: Some("MinNumNszF64"), + constant: Some("CALLEE_MINNUM_NSZ_F64"), + paths: &[ + "core::intrinsics::minimum_number_nsz_f64", + "std::intrinsics::minimum_number_nsz_f64", + ], signature: "(f64, f64) -> f64", placeholders: &["minimum_number_nsz_f64"], cmath: None, @@ -410,7 +654,14 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "never", }, MathMapping { - libm: "fminf", + libm: Some("fminf"), + width: "f32", + variant: Some("MinNumNszF32"), + constant: Some("CALLEE_MINNUM_NSZ_F32"), + paths: &[ + "core::intrinsics::minimum_number_nsz_f32", + "std::intrinsics::minimum_number_nsz_f32", + ], signature: "(f32, f32) -> f32", placeholders: &["minimum_number_nsz_f32"], cmath: None, @@ -420,7 +671,39 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "never", }, MathMapping { - libm: "hypot", + libm: Some("frexp"), + width: "f64", + variant: Some("FrexpF64"), + constant: Some("CALLEE_FREXP_F64"), + paths: &[], + signature: "(f64) -> (f64, i32)", + placeholders: &["frexpf64"], + cmath: None, + libdevice: Some("__nv_frexp"), + llvm: None, + lowering: Some("handwritten:emit_frexp"), + link: "always", + }, + MathMapping { + libm: Some("frexpf"), + width: "f32", + variant: Some("FrexpF32"), + constant: Some("CALLEE_FREXP_F32"), + paths: &[], + signature: "(f32) -> (f32, i32)", + placeholders: &["frexpf32"], + cmath: None, + libdevice: Some("__nv_frexpf"), + llvm: None, + lowering: Some("handwritten:emit_frexp"), + link: "always", + }, + MathMapping { + libm: Some("hypot"), + width: "f64", + variant: Some("HypotF64"), + constant: Some("CALLEE_HYPOT_F64"), + paths: &["std::sys::cmath::hypot"], signature: "(f64, f64) -> f64", placeholders: &["hypotf64"], cmath: Some("std::sys::cmath::hypot"), @@ -430,7 +713,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "hypotf", + libm: Some("hypotf"), + width: "f32", + variant: Some("HypotF32"), + constant: Some("CALLEE_HYPOT_F32"), + paths: &["std::sys::cmath::hypotf"], signature: "(f32, f32) -> f32", placeholders: &["hypotf32"], cmath: Some("std::sys::cmath::hypotf"), @@ -440,7 +727,67 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "log", + libm: Some("ilogb"), + width: "f64", + variant: Some("IlogbF64"), + constant: Some("CALLEE_ILOGB_F64"), + paths: &[], + signature: "(f64) -> i32", + placeholders: &["ilogbf64"], + cmath: None, + libdevice: Some("__nv_ilogb"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: Some("ilogbf"), + width: "f32", + variant: Some("IlogbF32"), + constant: Some("CALLEE_ILOGB_F32"), + paths: &[], + signature: "(f32) -> i32", + placeholders: &["ilogbf32"], + cmath: None, + libdevice: Some("__nv_ilogbf"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: Some("ldexp"), + width: "f64", + variant: Some("LdexpF64"), + constant: Some("CALLEE_LDEXP_F64"), + paths: &[], + signature: "(f64, i32) -> f64", + placeholders: &["ldexpf64"], + cmath: None, + libdevice: Some("__nv_ldexp"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: Some("ldexpf"), + width: "f32", + variant: Some("LdexpF32"), + constant: Some("CALLEE_LDEXP_F32"), + paths: &[], + signature: "(f32, i32) -> f32", + placeholders: &["ldexpf32"], + cmath: None, + libdevice: Some("__nv_ldexpf"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: Some("log"), + width: "f64", + variant: Some("LogF64"), + constant: Some("CALLEE_LOG_F64"), + paths: &["core::intrinsics::logf64", "std::intrinsics::logf64"], signature: "(f64) -> f64", placeholders: &["logf64"], cmath: None, @@ -450,7 +797,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "log10", + libm: Some("log10"), + width: "f64", + variant: Some("Log10F64"), + constant: Some("CALLEE_LOG10_F64"), + paths: &["core::intrinsics::log10f64", "std::intrinsics::log10f64"], signature: "(f64) -> f64", placeholders: &["log10f64"], cmath: None, @@ -460,7 +811,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "log10f", + libm: Some("log10f"), + width: "f32", + variant: Some("Log10F32"), + constant: Some("CALLEE_LOG10_F32"), + paths: &["core::intrinsics::log10f32", "std::intrinsics::log10f32"], signature: "(f32) -> f32", placeholders: &["log10f32"], cmath: None, @@ -470,7 +825,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "log1p", + libm: Some("log1p"), + width: "f64", + variant: Some("Log1pF64"), + constant: Some("CALLEE_LOG1P_F64"), + paths: &["std::sys::cmath::log1p"], signature: "(f64) -> f64", placeholders: &["log1pf64"], cmath: Some("std::sys::cmath::log1p"), @@ -480,7 +839,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "log1pf", + libm: Some("log1pf"), + width: "f32", + variant: Some("Log1pF32"), + constant: Some("CALLEE_LOG1P_F32"), + paths: &["std::sys::cmath::log1pf"], signature: "(f32) -> f32", placeholders: &["log1pf32"], cmath: Some("std::sys::cmath::log1pf"), @@ -490,7 +853,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "log2", + libm: Some("log2"), + width: "f64", + variant: Some("Log2F64"), + constant: Some("CALLEE_LOG2_F64"), + paths: &["core::intrinsics::log2f64", "std::intrinsics::log2f64"], signature: "(f64) -> f64", placeholders: &["log2f64"], cmath: None, @@ -500,7 +867,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "log2f", + libm: Some("log2f"), + width: "f32", + variant: Some("Log2F32"), + constant: Some("CALLEE_LOG2_F32"), + paths: &["core::intrinsics::log2f32", "std::intrinsics::log2f32"], signature: "(f32) -> f32", placeholders: &["log2f32"], cmath: None, @@ -510,7 +881,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "logf", + libm: Some("logf"), + width: "f32", + variant: Some("LogF32"), + constant: Some("CALLEE_LOG_F32"), + paths: &["core::intrinsics::logf32", "std::intrinsics::logf32"], signature: "(f32) -> f32", placeholders: &["logf32"], cmath: None, @@ -520,7 +895,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "pow", + libm: Some("pow"), + width: "f64", + variant: Some("PowfF64"), + constant: Some("CALLEE_POWF_F64"), + paths: &["core::intrinsics::powf64", "std::intrinsics::powf64"], signature: "(f64, f64) -> f64", placeholders: &["powf64"], cmath: None, @@ -530,7 +909,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "powf", + libm: Some("powf"), + width: "f32", + variant: Some("PowfF32"), + constant: Some("CALLEE_POWF_F32"), + paths: &["core::intrinsics::powf32", "std::intrinsics::powf32"], signature: "(f32, f32) -> f32", placeholders: &["powf32"], cmath: None, @@ -540,7 +923,14 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "rint", + libm: Some("rint"), + width: "f64", + variant: Some("RoundevenF64"), + constant: Some("CALLEE_ROUNDEVEN_F64"), + paths: &[ + "core::intrinsics::round_ties_even_f64", + "std::intrinsics::round_ties_even_f64", + ], signature: "(f64) -> f64", placeholders: &["round_ties_even_f64"], cmath: None, @@ -550,7 +940,14 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "libnvvm", }, MathMapping { - libm: "rintf", + libm: Some("rintf"), + width: "f32", + variant: Some("RoundevenF32"), + constant: Some("CALLEE_ROUNDEVEN_F32"), + paths: &[ + "core::intrinsics::round_ties_even_f32", + "std::intrinsics::round_ties_even_f32", + ], signature: "(f32) -> f32", placeholders: &["round_ties_even_f32"], cmath: None, @@ -560,7 +957,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "libnvvm", }, MathMapping { - libm: "round", + libm: Some("round"), + width: "f64", + variant: Some("RoundF64"), + constant: Some("CALLEE_ROUND_F64"), + paths: &["core::intrinsics::roundf64", "std::intrinsics::roundf64"], signature: "(f64) -> f64", placeholders: &["roundf64"], cmath: None, @@ -570,7 +971,14 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "libnvvm", }, MathMapping { - libm: "roundeven", + libm: Some("roundeven"), + width: "f64", + variant: Some("RoundevenF64"), + constant: Some("CALLEE_ROUNDEVEN_F64"), + paths: &[ + "core::intrinsics::round_ties_even_f64", + "std::intrinsics::round_ties_even_f64", + ], signature: "(f64) -> f64", placeholders: &["round_ties_even_f64"], cmath: None, @@ -580,7 +988,14 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "libnvvm", }, MathMapping { - libm: "roundevenf", + libm: Some("roundevenf"), + width: "f32", + variant: Some("RoundevenF32"), + constant: Some("CALLEE_ROUNDEVEN_F32"), + paths: &[ + "core::intrinsics::round_ties_even_f32", + "std::intrinsics::round_ties_even_f32", + ], signature: "(f32) -> f32", placeholders: &["round_ties_even_f32"], cmath: None, @@ -590,7 +1005,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "libnvvm", }, MathMapping { - libm: "roundf", + libm: Some("roundf"), + width: "f32", + variant: Some("RoundF32"), + constant: Some("CALLEE_ROUND_F32"), + paths: &["core::intrinsics::roundf32", "std::intrinsics::roundf32"], signature: "(f32) -> f32", placeholders: &["roundf32"], cmath: None, @@ -600,7 +1019,43 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "libnvvm", }, MathMapping { - libm: "sin", + libm: Some("scalbn"), + width: "f64", + variant: Some("LdexpF64"), + constant: Some("CALLEE_LDEXP_F64"), + paths: &[], + signature: "(f64, i32) -> f64", + placeholders: &["ldexpf64"], + cmath: None, + libdevice: Some("__nv_ldexp"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: Some("scalbnf"), + width: "f32", + variant: Some("LdexpF32"), + constant: Some("CALLEE_LDEXP_F32"), + paths: &[], + signature: "(f32, i32) -> f32", + placeholders: &["ldexpf32"], + cmath: None, + libdevice: Some("__nv_ldexpf"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: Some("sin"), + width: "f64", + variant: Some("SinF64"), + constant: Some("CALLEE_SIN_F64"), + paths: &[ + "core::intrinsics::sinf64", + "std::intrinsics::sinf64", + "std::sys::cmath::sin", + ], signature: "(f64) -> f64", placeholders: &["sinf64"], cmath: Some("std::sys::cmath::sin"), @@ -610,7 +1065,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "sincos", + libm: Some("sincos"), + width: "f64", + variant: None, + constant: None, + paths: &[], signature: "(f64) -> (f64, f64)", placeholders: &["sinf64", "cosf64"], cmath: None, @@ -620,7 +1079,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "sincosf", + libm: Some("sincosf"), + width: "f32", + variant: None, + constant: None, + paths: &[], signature: "(f32) -> (f32, f32)", placeholders: &["sinf32", "cosf32"], cmath: None, @@ -630,7 +1093,15 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "sinf", + libm: Some("sinf"), + width: "f32", + variant: Some("SinF32"), + constant: Some("CALLEE_SIN_F32"), + paths: &[ + "core::intrinsics::sinf32", + "std::intrinsics::sinf32", + "std::sys::cmath::sinf", + ], signature: "(f32) -> f32", placeholders: &["sinf32"], cmath: Some("std::sys::cmath::sinf"), @@ -640,7 +1111,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "sinh", + libm: Some("sinh"), + width: "f64", + variant: Some("SinhF64"), + constant: Some("CALLEE_SINH_F64"), + paths: &["std::sys::cmath::sinh"], signature: "(f64) -> f64", placeholders: &["sinhf64"], cmath: Some("std::sys::cmath::sinh"), @@ -650,7 +1125,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "sinhf", + libm: Some("sinhf"), + width: "f32", + variant: Some("SinhF32"), + constant: Some("CALLEE_SINH_F32"), + paths: &["std::sys::cmath::sinhf"], signature: "(f32) -> f32", placeholders: &["sinhf32"], cmath: Some("std::sys::cmath::sinhf"), @@ -660,7 +1139,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "sqrt", + libm: Some("sqrt"), + width: "f64", + variant: Some("SqrtF64"), + constant: Some("CALLEE_SQRT_F64"), + paths: &["core::intrinsics::sqrtf64", "std::intrinsics::sqrtf64"], signature: "(f64) -> f64", placeholders: &["sqrtf64"], cmath: None, @@ -670,7 +1153,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "sqrtf", + libm: Some("sqrtf"), + width: "f32", + variant: Some("SqrtF32"), + constant: Some("CALLEE_SQRT_F32"), + paths: &["core::intrinsics::sqrtf32", "std::intrinsics::sqrtf32"], signature: "(f32) -> f32", placeholders: &["sqrtf32"], cmath: None, @@ -680,7 +1167,15 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "tan", + libm: Some("tan"), + width: "f64", + variant: Some("TanF64"), + constant: Some("CALLEE_TAN_F64"), + paths: &[ + "core::intrinsics::tanf64", + "std::intrinsics::tanf64", + "std::sys::cmath::tan", + ], signature: "(f64) -> f64", placeholders: &["tanf64"], cmath: Some("std::sys::cmath::tan"), @@ -690,7 +1185,15 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "tanf", + libm: Some("tanf"), + width: "f32", + variant: Some("TanF32"), + constant: Some("CALLEE_TAN_F32"), + paths: &[ + "core::intrinsics::tanf32", + "std::intrinsics::tanf32", + "std::sys::cmath::tanf", + ], signature: "(f32) -> f32", placeholders: &["tanf32"], cmath: Some("std::sys::cmath::tanf"), @@ -700,7 +1203,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "tanh", + libm: Some("tanh"), + width: "f64", + variant: Some("TanhF64"), + constant: Some("CALLEE_TANH_F64"), + paths: &["std::sys::cmath::tanh"], signature: "(f64) -> f64", placeholders: &["tanhf64"], cmath: Some("std::sys::cmath::tanh"), @@ -710,7 +1217,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "tanhf", + libm: Some("tanhf"), + width: "f32", + variant: Some("TanhF32"), + constant: Some("CALLEE_TANH_F32"), + paths: &["std::sys::cmath::tanhf"], signature: "(f32) -> f32", placeholders: &["tanhf32"], cmath: Some("std::sys::cmath::tanhf"), @@ -720,7 +1231,39 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "always", }, MathMapping { - libm: "trunc", + libm: Some("tgamma"), + width: "f64", + variant: Some("TgammaF64"), + constant: Some("CALLEE_TGAMMA_F64"), + paths: &["std::sys::cmath::tgamma"], + signature: "(f64) -> f64", + placeholders: &["tgammaf64"], + cmath: Some("std::sys::cmath::tgamma"), + libdevice: Some("__nv_tgamma"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: Some("tgammaf"), + width: "f32", + variant: Some("TgammaF32"), + constant: Some("CALLEE_TGAMMA_F32"), + paths: &["std::sys::cmath::tgammaf"], + signature: "(f32) -> f32", + placeholders: &["tgammaf32"], + cmath: Some("std::sys::cmath::tgammaf"), + libdevice: Some("__nv_tgammaf"), + llvm: None, + lowering: None, + link: "always", + }, + MathMapping { + libm: Some("trunc"), + width: "f64", + variant: Some("TruncF64"), + constant: Some("CALLEE_TRUNC_F64"), + paths: &["core::intrinsics::truncf64", "std::intrinsics::truncf64"], signature: "(f64) -> f64", placeholders: &["truncf64"], cmath: None, @@ -730,7 +1273,11 @@ pub const MAPPINGS: &[MathMapping] = &[ link: "libnvvm", }, MathMapping { - libm: "truncf", + libm: Some("truncf"), + width: "f32", + variant: Some("TruncF32"), + constant: Some("CALLEE_TRUNC_F32"), + paths: &["core::intrinsics::truncf32", "std::intrinsics::truncf32"], signature: "(f32) -> f32", placeholders: &["truncf32"], cmath: None, diff --git a/cuda-oxide/intrinsics/libdevice-math.md b/cuda-oxide/intrinsics/libdevice-math.md index c6ddf489c9..60280018a8 100644 --- a/cuda-oxide/intrinsics/libdevice-math.md +++ b/cuda-oxide/intrinsics/libdevice-math.md @@ -2,9 +2,13 @@ -# Staged libm mappings +# Compiler math mappings -72 mapped f32/f64 libm exports. Compiler dispatch remains handwritten. +82 mapped f32/f64 libm exports, plus core-only compiler routes. + +The same shard generates compiler dispatch, placeholder constants, symbol lookup, +operand counts, cmath interception, and libdevice link requirements. Handwritten +semantic expansions remain responsible for sincos, frexp, min/max, and fast binops. Libdevice bitcode SHA-256: `91334d6e12748f6cb5bbf0a1cd965a56bcd93dc4f496d2e5c5f8c6e523094356`. @@ -13,6 +17,10 @@ a handwritten expansion. Aliases are separate exports sharing a mapping. | libm export | Rust signature | libdevice symbol | LLVM NVPTX alternative | Handwritten lowering | Linking | | --- | --- | --- | --- | --- | --- | +| | `(f32, f32, f32) -> f32` | `__nv_fmaf` | | | `always` | +| | `(f64, f64, f64) -> f64` | `__nv_fma` | | | `always` | +| | `(f32, i32) -> f32` | `__nv_powif` | | | `always` | +| | `(f64, i32) -> f64` | `__nv_powi` | | | `always` | | `acos` | `(f64) -> f64` | `__nv_acos` | | | `always` | | `acosf` | `(f32) -> f32` | `__nv_acosf` | | | `always` | | `acosh` | `(f64) -> f64` | `__nv_acosh` | | | `always` | @@ -53,8 +61,14 @@ a handwritten expansion. Aliases are separate exports sharing a mapping. | `fmaxf` | `(f32, f32) -> f32` | | | `handwritten:lower_float_minmax_nsz` | `never` | | `fmin` | `(f64, f64) -> f64` | | | `handwritten:lower_float_minmax_nsz` | `never` | | `fminf` | `(f32, f32) -> f32` | | | `handwritten:lower_float_minmax_nsz` | `never` | +| `frexp` | `(f64) -> (f64, i32)` | `__nv_frexp` | | `handwritten:emit_frexp` | `always` | +| `frexpf` | `(f32) -> (f32, i32)` | `__nv_frexpf` | | `handwritten:emit_frexp` | `always` | | `hypot` | `(f64, f64) -> f64` | `__nv_hypot` | | | `always` | | `hypotf` | `(f32, f32) -> f32` | `__nv_hypotf` | | | `always` | +| `ilogb` | `(f64) -> i32` | `__nv_ilogb` | | | `always` | +| `ilogbf` | `(f32) -> i32` | `__nv_ilogbf` | | | `always` | +| `ldexp` | `(f64, i32) -> f64` | `__nv_ldexp` | | | `always` | +| `ldexpf` | `(f32, i32) -> f32` | `__nv_ldexpf` | | | `always` | | `log` | `(f64) -> f64` | `__nv_log` | | | `always` | | `log10` | `(f64) -> f64` | `__nv_log10` | | | `always` | | `log10f` | `(f32) -> f32` | `__nv_log10f` | | | `always` | @@ -71,6 +85,8 @@ a handwritten expansion. Aliases are separate exports sharing a mapping. | `roundeven` | `(f64) -> f64` | `__nv_rint` | `llvm_roundeven_f64` | | `libnvvm` | | `roundevenf` | `(f32) -> f32` | `__nv_rintf` | `llvm_roundeven_f32` | | `libnvvm` | | `roundf` | `(f32) -> f32` | `__nv_roundf` | `llvm_round_f32` | | `libnvvm` | +| `scalbn` | `(f64, i32) -> f64` | `__nv_ldexp` | | | `always` | +| `scalbnf` | `(f32, i32) -> f32` | `__nv_ldexpf` | | | `always` | | `sin` | `(f64) -> f64` | `__nv_sin` | | | `always` | | `sincos` | `(f64) -> (f64, f64)` | | | `handwritten:emit_sincos` | `always` | | `sincosf` | `(f32) -> (f32, f32)` | | | `handwritten:emit_sincos` | `always` | @@ -83,5 +99,7 @@ a handwritten expansion. Aliases are separate exports sharing a mapping. | `tanf` | `(f32) -> f32` | `__nv_tanf` | | | `always` | | `tanh` | `(f64) -> f64` | `__nv_tanh` | | | `always` | | `tanhf` | `(f32) -> f32` | `__nv_tanhf` | | | `always` | +| `tgamma` | `(f64) -> f64` | `__nv_tgamma` | | | `always` | +| `tgammaf` | `(f32) -> f32` | `__nv_tgammaf` | | | `always` | | `trunc` | `(f64) -> f64` | `__nv_trunc` | `llvm_trunc_f64` | | `libnvvm` | | `truncf` | `(f32) -> f32` | `__nv_truncf` | `llvm_trunc_f32` | | `libnvvm` | diff --git a/cuda-oxide/intrinsics/overlay/libdevice_math.toml b/cuda-oxide/intrinsics/overlay/libdevice_math.toml index ceb976e8d4..1e4c4e894d 100644 --- a/cuda-oxide/intrinsics/overlay/libdevice_math.toml +++ b/cuda-oxide/intrinsics/overlay/libdevice_math.toml @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 # Separate libdevice evidence domain; not part of the hardware ABI shard index. -# PR3 stages the existing 72 libm exports. Compiler tables are migrated later. -schema = 1 +# Source of compiler math dispatch, placeholders, symbols, and link requirements. +schema = 2 family = "libdevice_math" [libdevice_math] @@ -49,23 +49,27 @@ cmath = "atanh{suffix}" [[libdevice_math.function]] name = "cbrt" signature = "(F) -> F" +paths = ["core::num::imp::libm::cbrt{suffix}"] cmath = "cbrt{suffix}" [[libdevice_math.function]] name = "ceil" signature = "(F) -> F" +core = ["ceil{width}"] llvm = "llvm_ceil_{width}" link = "libnvvm" [[libdevice_math.function]] name = "copysign" signature = "(F, F) -> F" +core = ["copysign{width}"] llvm = "llvm_copysign_{width}" link = "libnvvm" [[libdevice_math.function]] name = "cos" signature = "(F) -> F" +core = ["cos{width}"] cmath = "cos{suffix}" [[libdevice_math.function]] @@ -76,10 +80,12 @@ cmath = "cosh{suffix}" [[libdevice_math.function]] name = "exp" signature = "(F) -> F" +core = ["exp{width}"] [[libdevice_math.function]] name = "exp2" signature = "(F) -> F" +core = ["exp2{width}"] [[libdevice_math.function]] name = "expm1" @@ -89,6 +95,9 @@ cmath = "expm1{suffix}" [[libdevice_math.function]] name = "fabs" signature = "(F) -> F" +variant = "Fabs" +constant = "CALLEE_FABS" +core = ["fabs"] placeholders = ["fabs"] llvm = "llvm_fabs_{width}" link = "libnvvm" @@ -96,16 +105,21 @@ link = "libnvvm" [[libdevice_math.function]] name = "floor" signature = "(F) -> F" +core = ["floor{width}"] llvm = "llvm_floor_{width}" link = "libnvvm" [[libdevice_math.function]] name = "fma" signature = "(F, F, F) -> F" +core = ["fma{width}"] [[libdevice_math.function]] name = "fmax" signature = "(F, F) -> F" +variant = "MaxNumNsz{Width}" +constant = "CALLEE_MAXNUM_NSZ_{WIDTH}" +core = ["maximum_number_nsz_{width}"] placeholders = ["maximum_number_nsz_{width}"] lowering = "handwritten:lower_float_minmax_nsz" link = "never" @@ -113,6 +127,9 @@ link = "never" [[libdevice_math.function]] name = "fmin" signature = "(F, F) -> F" +variant = "MinNumNsz{Width}" +constant = "CALLEE_MINNUM_NSZ_{WIDTH}" +core = ["minimum_number_nsz_{width}"] placeholders = ["minimum_number_nsz_{width}"] lowering = "handwritten:lower_float_minmax_nsz" link = "never" @@ -125,10 +142,12 @@ cmath = "hypot{suffix}" [[libdevice_math.function]] name = "log" signature = "(F) -> F" +core = ["log{width}"] [[libdevice_math.function]] name = "log10" signature = "(F) -> F" +core = ["log10{width}"] [[libdevice_math.function]] name = "log1p" @@ -138,14 +157,21 @@ cmath = "log1p{suffix}" [[libdevice_math.function]] name = "log2" signature = "(F) -> F" +core = ["log2{width}"] [[libdevice_math.function]] name = "pow" signature = "(F, F) -> F" +variant = "Powf{Width}" +constant = "CALLEE_POWF_{WIDTH}" +core = ["pow{width}"] [[libdevice_math.function]] name = "rint" signature = "(F) -> F" +variant = "Roundeven{Width}" +constant = "CALLEE_ROUNDEVEN_{WIDTH}" +core = ["round_ties_even_{width}"] libm = ["rint", "roundeven"] placeholders = ["round_ties_even_{width}"] llvm = "llvm_roundeven_{width}" @@ -154,12 +180,14 @@ link = "libnvvm" [[libdevice_math.function]] name = "round" signature = "(F) -> F" +core = ["round{width}"] llvm = "llvm_round_{width}" link = "libnvvm" [[libdevice_math.function]] name = "sin" signature = "(F) -> F" +core = ["sin{width}"] cmath = "sin{suffix}" [[libdevice_math.function]] @@ -176,10 +204,12 @@ cmath = "sinh{suffix}" [[libdevice_math.function]] name = "sqrt" signature = "(F) -> F" +core = ["sqrt{width}"] [[libdevice_math.function]] name = "tan" signature = "(F) -> F" +core = ["tan{width}"] cmath = "tan{suffix}" [[libdevice_math.function]] @@ -190,5 +220,39 @@ cmath = "tanh{suffix}" [[libdevice_math.function]] name = "trunc" signature = "(F) -> F" +core = ["trunc{width}"] llvm = "llvm_trunc_{width}" link = "libnvvm" + +[[libdevice_math.function]] +name = "tgamma" +signature = "(F) -> F" +cmath = "tgamma{suffix}" + +[[libdevice_math.function]] +name = "ilogb" +signature = "(F) -> i32" + +[[libdevice_math.function]] +name = "ldexp" +signature = "(F, i32) -> F" +libm = ["ldexp", "scalbn"] + +[[libdevice_math.function]] +name = "frexp" +signature = "(F) -> (F, i32)" +lowering = "handwritten:emit_frexp" +libdevice = "__nv_frexp{suffix}" + +[[libdevice_math.function]] +name = "powi" +signature = "(F, i32) -> F" +core = ["powi{width}"] +libm = [] + +[[libdevice_math.function]] +name = "fmuladd" +signature = "(F, F, F) -> F" +core = ["fmuladd{width}"] +libm = [] +libdevice = "__nv_fma{suffix}"