Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion DESCRIPTION
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
Package: diffuseR
Title: Functional Interface to Diffusion Models in R
Version: 0.2.2
Version: 0.2.2.1
Authors@R: c(
person("Troy", "Hernandez", email = "troy@cornball.ai", role = c("aut", "cre"),
comment = c(ORCID = "0009-0005-4248-604X")),
Expand Down
10 changes: 10 additions & 0 deletions NEWS.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,13 @@
# diffuseR 0.2.2.1

* `txt2img()` and `img2img()` now `match.arg()` their `model_name`, so
the bare calls work (defaulting to sd21) instead of erroring on the
choices vector.
* README fixes for the CRAN-rendered page: closed the unclosed fence
that swallowed the LTX section, replaced the LTX example with the
working call shape, and pointed the example images at GitHub URLs
(the files are .Rbuildignore'd).

# diffuseR 0.2.2

* Every precision `recommend()` can return is now reachable. `bf16` was
Expand Down
16 changes: 10 additions & 6 deletions R/download_ltx23.R
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,8 @@ NULL
#'
#' @export
download_ltx2 <- function(quantize = TRUE, precision = c("nf4", "fp8"),
output_dir = NULL,
text_encoder = TRUE, verbose = TRUE) {
output_dir = NULL, text_encoder = TRUE,
verbose = TRUE) {
if (!requireNamespace("hfhub", quietly = TRUE)) {
stop("The hfhub package is required to download model weights.")
}
Expand All @@ -83,7 +83,11 @@ download_ltx2 <- function(quantize = TRUE, precision = c("nf4", "fp8"),
output_dir <- file.path(tools::R_user_dir("diffuseR", "data"),
paste0("ltx2.3-", precision))
}
art_gb <- if (identical(precision, "fp8")) 26 else 19
if (identical(precision, "fp8")) {
art_gb <- 26
} else {
art_gb <- 19
}
result <- list(checkpoint = NULL, artifact_dir = output_dir,
precision = precision,
fp8_dir = if (identical(precision, "fp8")) output_dir,
Expand All @@ -109,9 +113,9 @@ download_ltx2 <- function(quantize = TRUE, precision = c("nf4", "fp8"),
free, precision, 46 + art_gb))
}
ok <- .ltx23_consent(sprintf(paste0(
"the LTX-2.3 distilled checkpoint (46 GB) plus a ~%d GB local %s ",
"artifact from HuggingFace (weights under the LTX-2 Community License)"),
art_gb, precision))
"the LTX-2.3 distilled checkpoint (46 GB) plus a ~%d GB local %s ",
"artifact from HuggingFace (weights under the LTX-2 Community License)"),
art_gb, precision))
if (!ok) {
stop("Download cancelled.", call. = FALSE)
}
Expand Down
1 change: 1 addition & 0 deletions R/img2img.R
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ img2img <- function(input_image, prompt, negative_prompt = NULL,
use_native_decoder = FALSE,
use_native_text_encoder = FALSE, use_native_unet = FALSE,
...) {
model_name <- match.arg(model_name)
if (model_name %in% c("sd21", "sdxl")) {
num_train_timesteps <- 1000
} else {
Expand Down
3 changes: 1 addition & 2 deletions R/quantize_flux.R
Original file line number Diff line number Diff line change
Expand Up @@ -97,8 +97,7 @@ NULL
zimage = list(repo = .zimage_repo,
files = .zimage_transformer_files,
fn = "download_zimage_turbo"),
stop("No bf16 source known for model '", model, "'",
call. = FALSE))
stop("No bf16 source known for model '", model, "'", call. = FALSE))
paths <- lapply(spec$files, function(f) {
tryCatch(hfhub::hub_download(spec$repo, f, local_files_only = TRUE),
error = function(e) NULL)
Expand Down
8 changes: 4 additions & 4 deletions R/recommend.R
Original file line number Diff line number Diff line change
Expand Up @@ -132,10 +132,10 @@ recommend <- function(model = c("sd21", "sdxl", "flux1", "flux2", "zimage",
host_ram_gb = host_ram_gb,
fork_suggested = fork,
note = if (fork) {
.st_fork_note(want$precision)
} else {
.bf16_note(model, chosen$precision)
}
.st_fork_note(want$precision)
} else {
.bf16_note(model, chosen$precision)
}
)
}

Expand Down
36 changes: 18 additions & 18 deletions R/resident.R
Original file line number Diff line number Diff line change
Expand Up @@ -126,16 +126,18 @@
# as.character() on a torch_dtype returns ("Float", "Half", "Byte",
# "Long", ...), NOT the R constructor alias. Unknown dtypes fall back to
# 4, which only affects a reported number.
.dtype_widths <- c(double = 8, long = 8, complexfloat = 8,
float = 4, int = 4,
half = 2, bfloat16 = 2, short = 2,
byte = 1, char = 1, bool = 1,
float8_e4m3fn = 1, float8_e5m2 = 1)
.dtype_widths <- c(double = 8, long = 8, complexfloat = 8, float = 4,
int = 4, half = 2, bfloat16 = 2, short = 2, byte = 1,
char = 1, bool = 1, float8_e4m3fn = 1, float8_e5m2 = 1)

.dtype_bytes <- function(dtype) {
nm <- tolower(tryCatch(as.character(dtype), error = function(e) ""))
w <- .dtype_widths[[nm, exact = TRUE]]
if (is.null(w)) 4 else w
if (is.null(w)) {
4
} else {
w
}
}

#' TRUE when every staged tensor sits on the expected device type
Expand All @@ -149,7 +151,8 @@
.resident_all_on <- function(staging, type) {
for (st in staging) {
for (pair in st) {
dev <- tryCatch(pair$live$device$type, error = function(e) NA_character_)
dev <- tryCatch(pair$live$device$type,
error = function(e) NA_character_)
if (!identical(dev, type)) {
return(FALSE)
}
Expand Down Expand Up @@ -192,8 +195,9 @@
stop("cannot ", verb, ": this handle is unloaded", call. = FALSE)
}
if (identical(res$state, "broken")) {
stop("cannot ", verb, ": this handle is broken (", res$last_error %||%
"no detail recorded", "). Only resident_status() and ",
stop("cannot ", verb, ": this handle is broken (",
res$last_error %||% "no detail recorded",
"). Only resident_status() and ",
"resident_unload() work from here.", call. = FALSE)
}
invisible(TRUE)
Expand Down Expand Up @@ -324,7 +328,7 @@ resident_load <- function(model = c("flux2", "flux1", "zimage", "ltx"),
free_gb <- tryCatch(.detect_vram(use_free = TRUE),
error = function(e) NA_real_)
}
need_gb <- res$pinned_bytes / 1024^3
need_gb <- res$pinned_bytes / 1024 ^ 3
if (!is.na(free_gb) && free_gb > 0 && need_gb > free_gb) {
stop(sprintf(paste0("%s needs %.2f GB resident but only %.2f GB of ",
"VRAM is free. Load the pipeline with ",
Expand Down Expand Up @@ -532,15 +536,11 @@ resident_generate <- function(res, prompt, ...) {
resident_status <- function(res) {
stopifnot(inherits(res, "diffuseR_resident"))
mem <- .cuda_bytes()
list(model = res$model,
state = res$state,
device = res$device,
list(model = res$model, state = res$state, device = res$device,
components = names(res$staging),
components_on_gpu = .resident_on_gpu_count(res$staging),
pinned_bytes = res$pinned_bytes,
gpu_allocated = mem$allocated,
gpu_reserved = mem$reserved,
loaded_at = res$loaded_at,
pinned_bytes = res$pinned_bytes, gpu_allocated = mem$allocated,
gpu_reserved = mem$reserved, loaded_at = res$loaded_at,
last_error = res$last_error)
}

Expand Down Expand Up @@ -623,5 +623,5 @@ print.diffuseR_resident <- function(x, ...) {
if (is.null(b) || is.na(b) || b <= 0) {
return("0 GB")
}
sprintf("%.2f GB", b / 1024^3)
sprintf("%.2f GB", b / 1024 ^ 3)
}
3 changes: 1 addition & 2 deletions R/st_caps.R
Original file line number Diff line number Diff line change
Expand Up @@ -113,8 +113,7 @@ NULL
} else {
sprintf("%s needs", precision)
}
sprintf(paste0(
"%s a safetensors newer than the one on CRAN: %s is ",
sprintf(paste0("%s a safetensors newer than the one on CRAN: %s is ",
"merged upstream but not yet released. Install the ",
"development version from the mlverse/safetensors ",
"repository on GitHub, or press on with nf4: same ",
Expand Down
1 change: 1 addition & 0 deletions R/txt2img.R
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
txt2img <- function(prompt,
model_name = c("sd21", "sdxl", "flux1", "flux2", "zimage"),
...) {
model_name <- match.arg(model_name)
switch(model_name,
# "sd15" = txt2img_sd15(prompt, ...),
"sd21" = txt2img_sd21(prompt, ...),
Expand Down
3 changes: 1 addition & 2 deletions R/txt2img_flux2.R
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,7 @@ flux2_load_pipeline <- function(model_dir = NULL, device = "cuda",
}
if (is.null(model_dir)) {
model_dir <- .flux_model_dir("flux2", precision,
file.path(tools::R_user_dir("diffuseR", "data"),
"flux2-klein-4b-"))
file.path(tools::R_user_dir("diffuseR", "data"), "flux2-klein-4b-"))
}

ckpt <- if (file.exists(file.path(model_dir, "manifest.json"))) {
Expand Down
3 changes: 1 addition & 2 deletions R/txt2img_zimage.R
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,7 @@ zimage_load_pipeline <- function(model_dir = NULL, device = "cuda",
}
if (is.null(model_dir)) {
model_dir <- .flux_model_dir("zimage", precision,
file.path(tools::R_user_dir("diffuseR", "data"),
"zimage-turbo-"))
file.path(tools::R_user_dir("diffuseR", "data"), "zimage-turbo-"))
}

ckpt <- if (file.exists(file.path(model_dir, "manifest.json"))) {
Expand Down
16 changes: 11 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -143,8 +143,8 @@ gambling_cat <- img2img(
pipeline <- NULL
torch::cuda_empty_cache()
```
![](cat2.png)
![](gambling_cat.png)
![](https://raw.githubusercontent.com/cornball-ai/diffuseR/main/cat2.png)
![](https://raw.githubusercontent.com/cornball-ai/diffuseR/main/gambling_cat.png)


### FLUX and Z-Image
Expand Down Expand Up @@ -176,6 +176,7 @@ txt2img_flux2("a red fox sitting in a snowy forest, digital art",
download_zimage_turbo()
txt2img_zimage(paste("A storefront with a large wooden sign that reads",
"\"DIFFUSER\" in bold carved letters"), seed = 42)
```

### Text-to-Video: LTX-2.3

Expand All @@ -185,10 +186,14 @@ steps. NF4-quantized it renders 768x512x49 in ~44s warm on an RTX
are supported (see ?txt2vid_ltx2).

```r
download_ltx2() # ~46GB download, one-time fp8 quantize
paths <- download_ltx2() # ~46GB download, one-time nf4 quantize
pipe <- ltx23_load_pipeline(paths$artifact_dir)
te <- load_gemma3_text_encoder(paths$text_encoder_dir, device = "cpu")
tok <- gemma3_tokenizer(dirname(hfhub::hub_download("Lightricks/LTX-2",
"tokenizer/tokenizer.json")))
txt2vid_ltx2("A river winding through a misty forest at dawn",
pipeline = ltx23_load_pipeline(),
filename = "river.mp4")
pipeline = pipe, text_encoder = te, tokenizer = tok,
num_frames = 73L, seed = 11L, filename = "river.mp4")
```

### Serving over HTTP
Expand All @@ -201,6 +206,7 @@ serve(model = "flux2", port = 7812L, token = "my-secret")
# http://localhost:7812/v1/images/generations
```

```r
# Or through the common dispatcher
txt2img("a lighthouse at dusk", model_name = "flux2")
```
Expand Down
8 changes: 8 additions & 0 deletions inst/tinytest/test_dispatch.R
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
# Dispatcher argument validation: an unknown model_name must fail fast
# with match.arg's message instead of reaching switch() or a vector
# if() condition (bare txt2img("prompt") used to error with "EXPR must
# be a length 1 vector"; img2img errored on the length-2 default).

expect_error(txt2img("a cat", model_name = "nope"), "should be one of")
expect_error(img2img("cat.png", "a cat", model_name = "nope"),
"should be one of")
10 changes: 6 additions & 4 deletions man/CLIPTokenizer.Rd
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,12 @@
\alias{CLIPTokenizer}
\title{Tokenize a prompt}
\usage{
CLIPTokenizer(prompt,
merges = system.file("tokenizer/merges.txt", package = "diffuseR"),
vocab_file = system.file("tokenizer/vocab.json", package = "diffuseR"),
pad_token = 0L)
CLIPTokenizer(
prompt,
merges = system.file("tokenizer/merges.txt", package = "diffuseR"),
vocab_file = system.file("tokenizer/vocab.json", package = "diffuseR"),
pad_token = 0L
)
}
\arguments{
\item{prompt}{A character string prompt describing the image to generate.}
Expand Down
8 changes: 7 additions & 1 deletion man/SpatialTransformer.Rd
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,13 @@
\alias{SpatialTransformer}
\title{Spatial Transformer (Attention Block)}
\usage{
SpatialTransformer(in_channels, n_heads, d_head, depth = 1L, context_dim = NULL)
SpatialTransformer(
in_channels,
n_heads,
d_head,
depth = 1L,
context_dim = NULL
)
}
\description{
Spatial Transformer (Attention Block)
Expand Down
9 changes: 7 additions & 2 deletions man/VAEUpBlock.Rd
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,13 @@
\alias{VAEUpBlock}
\title{VAE Up Block}
\usage{
VAEUpBlock(in_channels, out_channels, num_resnets = 3, add_upsample = TRUE,
norm_groups = 32)
VAEUpBlock(
in_channels,
out_channels,
num_resnets = 3,
add_upsample = TRUE,
norm_groups = 32
)
}
\arguments{
\item{in_channels}{Input channels}
Expand Down
8 changes: 6 additions & 2 deletions man/convert_sd21_pt_to_diffusers.Rd
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,12 @@
\alias{convert_sd21_pt_to_diffusers}
\title{Convert cornball SD 2.1 TorchScript weights to a diffusers artifact}
\usage{
convert_sd21_pt_to_diffusers(pt_dir = NULL, output_dir = NULL,
dtype = c("float16", "float32"), verbose = TRUE)
convert_sd21_pt_to_diffusers(
pt_dir = NULL,
output_dir = NULL,
dtype = c("float16", "float32"),
verbose = TRUE
)
}
\arguments{
\item{pt_dir}{Directory holding \code{unet-cpu.pt},
Expand Down
18 changes: 11 additions & 7 deletions man/ddim_scheduler_create.Rd
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,17 @@
\alias{ddim_scheduler_create}
\title{Create a DDIM Scheduler}
\usage{
ddim_scheduler_create(num_train_timesteps = 1000, num_inference_steps = 50,
eta = 0,
beta_schedule = c("linear", "scaled_linear", "cosine"),
beta_start = 0.00085, beta_end = 0.012,
rescale_betas_zero_snr = FALSE,
dtype = torch::torch_float32(),
device = torch::torch_device("cpu"))
ddim_scheduler_create(
num_train_timesteps = 1000,
num_inference_steps = 50,
eta = 0,
beta_schedule = c("linear", "scaled_linear", "cosine"),
beta_start = 0.00085,
beta_end = 0.012,
rescale_betas_zero_snr = FALSE,
dtype = torch::torch_float32(),
device = torch::torch_device("cpu")
)
}
\arguments{
\item{num_train_timesteps}{Integer. The number of diffusion steps used to
Expand Down
22 changes: 16 additions & 6 deletions man/ddim_scheduler_step.Rd
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,22 @@
\alias{ddim_scheduler_step}
\title{Perform a DDIM scheduler step}
\usage{
ddim_scheduler_step(model_output, timestep, sample, schedule, eta = 0,
use_clipped_model_output = FALSE, thresholding = FALSE,
generator = NULL, variance_noise = NULL,
clip_sample = FALSE, set_alpha_to_one = FALSE,
prediction_type = c("epsilon", "sample", "v_prediction"),
dtype = torch::torch_float32(), device = "cpu")
ddim_scheduler_step(
model_output,
timestep,
sample,
schedule,
eta = 0,
use_clipped_model_output = FALSE,
thresholding = FALSE,
generator = NULL,
variance_noise = NULL,
clip_sample = FALSE,
set_alpha_to_one = FALSE,
prediction_type = c("epsilon", "sample", "v_prediction"),
dtype = torch::torch_float32(),
device = "cpu"
)
}
\arguments{
\item{model_output}{Numeric array. The output from the diffusion model, typically
Expand Down
Loading
Loading