From 9771eb3e3910fec79a58e41325b4604e1a065983 Mon Sep 17 00:00:00 2001 From: Chaser Huang Date: Fri, 11 Sep 2026 17:33:52 -0400 Subject: [PATCH 1/6] feat(runtime): add LanPaint Langevin-dynamics inpainting core engine Assisted-by: prime-agent deepseek-v4-flash-0731 Assisted-by: prime-agent glm-5.3-flash Signed-off-by: Chaser Huang --- src/core/tensor.hpp | 9 ++ src/runtime/lanpaint.hpp | 341 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 350 insertions(+) create mode 100644 src/runtime/lanpaint.hpp diff --git a/src/core/tensor.hpp b/src/core/tensor.hpp index ba5dc137a..bef0a9952 100644 --- a/src/core/tensor.hpp +++ b/src/core/tensor.hpp @@ -1073,6 +1073,15 @@ namespace sd { return output; } + template + inline Tensor sqrt(const Tensor& input) { + Tensor output(input.shape()); + for (int64_t i = 0; i < input.numel(); ++i) { + output[i] = static_cast(std::sqrt(static_cast(input[i]))); + } + return output; + } + template inline Tensor clamp(const Tensor& input, const T& min_value, const T& max_value) { if (min_value > max_value) { diff --git a/src/runtime/lanpaint.hpp b/src/runtime/lanpaint.hpp new file mode 100644 index 000000000..5df741d03 --- /dev/null +++ b/src/runtime/lanpaint.hpp @@ -0,0 +1,341 @@ +#ifndef __SD_RUNTIME_LANPAINT_HPP__ +#define __SD_RUNTIME_LANPAINT_HPP__ + +/* + * LanPaint: Langevin-dynamics inpainting, following the ComfyUI LanPaint + * algorithm (LanPaint/src/LanPaint/lanpaint.py + nodes.py). + * + * The underlying model evaluation is abstracted behind `lanpaint_eval_t` + * (the unblended (x0, x0_BIG) pair), bundled with its Denoiser in + * `LanPaintInnerModel`. This header includes `runtime/denoiser.hpp`; + * `denoiser.hpp` must not include this header. + * + * Mask convention (comfy `latent_mask = 1 - denoise_mask`): + * `latent_mask` is a float tensor, 1 = KEEP the original image, 0 = EDIT. + * sd.cpp's `denoise_mask` is a latent-space tensor of shape {W,H,1,1} + * (images) / {W,H,T,1,1} (video) against latents {W,H,C,1} / {W,H,T,C,1}, + * so the caller passes `latent_mask = 1 - denoise_mask` and broadcasting + * is automatic. + * + * Sigma convention: `sigma` is the native denoiser sigma -- flow time t in + * [0,1] for the DiscreteFlowDenoiser family, the VE sigma (e.g. + * ~0.03..14.6 for SD1.5) for the CompVisDenoiser family. `compute_times()` + * derives the (VE_Sigma, abt, Flow_t) triple from it. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "core/rng.hpp" +#include "core/tensor.hpp" +#include "runtime/denoiser.hpp" +#include "runtime/guidance.h" + +// Parameters (defaults from the ComfyUI node, v2.1.0). +struct LanPaintParams { + int n_steps = 5; // NSteps: inner Langevin steps per outer step + float friction = 15.f; // Friction: unused by the overdamped scheme + // implemented here + float lambda = 5.f; // Lambda: strength of the keep-region score_y + float beta = 1.f; // Beta: time-scale ratio of the y branch + float step_size = 0.2f; // StepSize + int early_stop = 1; // EarlyStop: drop the inner loop in the last N outer steps + float min_step_frac = 1.f; // MinStepFrac: pin step size below this, ramp n_eff down + float cfg_big = INFINITY; // BIG-CFG scale; sentinel INFINITY == "Image First" (cfg_BIG == cfg). + // Consumed by the evaluation, not by the cycle. +}; + +// Unified time triple (comfy: `current_times = (VE_Sigma, abt, Flow_t)`). +struct LanPaintModelTimes { + float ve_sigma; // variance-exploding sigma + float abt; // the "remaining signal" fraction in [0,1] + float flow_t; // flow time t +}; + +// Result of one diffusion forward: the CFG denoised x0 and its BIG-CFG +// variant. Both must be in NATIVE latent coordinates and UNBLENDED (the +// LanPaint cycle does the mask blending itself). +struct LanPaintEval { + sd::Tensor x0; // pred (full CFG, unblended) + sd::Tensor x0_big; // BIG-CFG denoised; may equal x0 +}; + +// The underlying network evaluation. `x` is in native latent coordinates, +// `sigma` in native units (flow t for flow models, VE sigma otherwise) -- +// i.e. exactly the sigma the outer sampler passes at this step. Must be +// pure (no side effects): the caller owns `x`. +using lanpaint_eval_t = std::function& x, float sigma)>; + +// The model LanPaint paints with: the unblended network eval plus the +// Denoiser it belongs to (noise-scaling formulas, flow/VE mode) -- comfy's +// `inner_model` + `inner_model.inner_model.model_sampling`. `denoiser` is a +// borrowed, non-owning pointer (non-const because noise_scaling() is a +// non-const virtual) that must outlive the LanPaint engine and its +// callback. +struct LanPaintInnerModel { + lanpaint_eval_t eval; + Denoiser* denoiser; + bool is_flow; +}; + +// Builds a LanPaintInnerModel from a runtime denoiser: derives the flow/VE +// mode from the denoiser type and refuses the denoisers whose latent +// scaling / time conventions are unsupported (MiniT2I: noise*2 start, +// reversed time; SeFi: dual timesteps). Returns nullopt when refused. +inline std::optional make_lanpaint_inner_model(lanpaint_eval_t eval, + const std::shared_ptr& denoiser) { + if (!denoiser) { + LOG_ERROR("LanPaint requires a denoiser"); + return std::nullopt; + } + if (std::dynamic_pointer_cast(denoiser) || std::dynamic_pointer_cast(denoiser)) { + LOG_ERROR("LanPaint does not support this denoiser's latent scaling / time conventions"); + return std::nullopt; + } + return LanPaintInnerModel{std::move(eval), + denoiser.get(), + (bool)std::dynamic_pointer_cast(denoiser)}; +} + +// One LanPaint engine. Holds the per-sample context (the inner model, the +// replace-step noise, the target latent, the keep mask and the RNG); `run()` +// executes one full outer-step cycle on the sampler's latent in place and +// returns the blended denoised x0 (comfy's `KSamplerX0Inpaint.__call__` + +// `LanPaint.LanPaint` for one step). `make_callback()` wraps `run()` into +// the callback shape the sampler kernels call (outer index == |step|-1; `x` +// by non-const reference so the evolved latent is written back to the +// sampler's local state, comfy's `input_x.copy_(x)`). +struct LanPaint { + LanPaint(const LanPaintParams& params, + LanPaintInnerModel inner_model, + const std::shared_ptr& rng, + const sd::Tensor& noise, + const sd::Tensor& latent_image, + const sd::Tensor& latent_mask) + : params_(params), + inner_model_(std::move(inner_model)), + rng_(rng), + noise_(noise), + latent_image_(latent_image), + latent_mask_(latent_mask) { + } + + static LanPaintModelTimes compute_times(bool is_flow, float sigma) { + LanPaintModelTimes t; + if (is_flow) { + t.flow_t = sigma; + float f = t.flow_t; + t.abt = (1.f - f) * (1.f - f) / ((1.f - f) * (1.f - f) + f * f); + t.ve_sigma = f / (1.f - f); + } else { + t.ve_sigma = sigma; + t.abt = 1.f / (1.f + sigma * sigma); + t.flow_t = std::sqrt(1.f - t.abt) / (std::sqrt(1.f - t.abt) + std::sqrt(t.abt)); + } + return t; + } + + // One full LanPaint cycle (comfy `KSamplerX0Inpaint.__call__` + + // `LanPaint.LanPaint`) for a single outer step. + // + // In: `x` = the sampler's latent in NATIVE coordinates. + // Out: `x` is overwritten with the post-Langevin native latent (comfy's + // in-place `input_x.copy_(x)`), and the returned tensor is the + // blended denoised x0 (`out`). + // + // `sigma` is the native sigma of this outer step (flow t or VE sigma); + // `sigmas` is the full schedule (for total_steps); `outer_step` is the + // 0-based outer index (== |step|-1 in the sd.cpp kernel convention). + sd::Tensor run(sd::Tensor& x, float sigma, const std::vector& sigmas, int outer_step) const { + if (latent_mask_.empty()) { + // No mask: LanPaint is a no-op -- return the plain denoised x0. + LanPaintEval ev = inner_model_.eval(x, sigma); + return ev.x0.empty() ? sd::Tensor() : std::move(ev.x0); + } + + const LanPaintModelTimes ct = compute_times(inner_model_.is_flow, sigma); + const float abt = ct.abt; + + // 1. Step size: StepSize * clamp(1 - abt, min = MinStepFrac). + const float step_size = params_.step_size * std::max(1.f - abt, params_.min_step_frac); + + // 2. Effective inner-step count (comfy `min_step_frac_effective_steps`). + const int total_steps = static_cast(sigmas.size()) - 1; + int n_eff = params_.n_steps; + if (total_steps - outer_step <= params_.early_stop) { + n_eff = 0; + } else if (params_.min_step_frac > 0.f && (1.f - abt) < params_.min_step_frac && params_.n_steps > 0.f) { + n_eff = std::max(0, static_cast(std::lround((float)params_.n_steps * (1.f - abt) / params_.min_step_frac))); + } + + // 3. Replace step: the keep region is re-noised around the current + // noise level, at every outer step (even when n_eff == 0): + // x <- x*(1-m) + model_sampling.noise_scaling(sigma, noise, y)*m + // with the noise scaling delegated to the Denoiser. + const bool is_flow = inner_model_.is_flow; + x = x * (1.f - latent_mask_) + + inner_model_.denoiser->noise_scaling(sigma, noise_, latent_image_) * latent_mask_; + + // 4. To variance-preserving form. + sd::Tensor x_t; + if (is_flow) { + const float s = std::sqrt(abt) + std::sqrt(1.f - abt); + x_t = x * s; + } else { + x_t = x / std::sqrt(1.f + ct.ve_sigma * ct.ve_sigma); + } + + // 5. Langevin inner loop. Scalar coefficients (sigma is a scalar, + // hence so are abt and step_size): + // sigma_x = abt^0 = 1 ; sigma_y = beta*abt^0 = beta + // dtx = 2*step_size*sigma_x ; dty = 2*step_size*sigma_y + // (prepare_step_size returns dtx/2, dty/2) + // dt = (dtx/2)*(1-m) + (dty/2)*m + // A_x = 1/(1-abt) ; A_y = (1+lambda)/(1-abt) + // A = A_x*(1-m) + A_y*m + // D^2 = 2 + const float one_minus_abt = 1.f - abt; + const float A_x = 1.f / one_minus_abt; + const float A_y = (1.f + params_.lambda) / one_minus_abt; + const float half_dtx = step_size; // step_size * sigma_x + const float half_dty = step_size * params_.beta; // step_size * sigma_y + const float D2 = 2.f; + + sd::Tensor dt = half_dtx * (1.f - latent_mask_) + half_dty * latent_mask_; + sd::Tensor A = A_x * (1.f - latent_mask_) + A_y * latent_mask_; + + // The inner loop runs only when the effective step count is positive + // (comfy: KSamplerX0Inpaint sets n_eff = 0 for EarlyStop / the + // MinStepFrac ramp, making `range(n_eff)` empty) and the time step is + // positive (comfy: `if mean(dtx) <= 0: return` in langevin_dynamics; + // dtx == step_size here). + const bool inner_ok = (n_eff > 0) && (step_size > 0.f); + + // Coef_C(x_t): x0 = x_t + score(x_t); + // C = (sqrt(abt)*x0 - x_t)/(1-abt) + A*x_t + // The score (comfy `score_model`): + // deconvert x_t -> native x, eval model -> (x0, x0_big) + // score_x = -(x_t - x0) + // score_y = -(1+lambda)*(x_t - y) + lambda*(x_t - x0_big) + // score = score_x*(1-m) + score_y*m + // Only returns C since x0 is not used nor updated by further pipeline + auto score_and_C = [&](const sd::Tensor& xt) -> sd::Tensor { + sd::Tensor x_native; + if (is_flow) { + const float s = std::sqrt(abt) + std::sqrt(1.f - abt); + x_native = xt / s; + } else { + x_native = xt * std::sqrt(1.f + ct.ve_sigma * ct.ve_sigma); + } + LanPaintEval ev = inner_model_.eval(x_native, sigma); + if (ev.x0.empty()) { + return sd::Tensor(); + } + sd::Tensor score_x = -(xt - ev.x0); + sd::Tensor score_y = -(1.f + params_.lambda) * (xt - latent_image_) + params_.lambda * (xt - ev.x0_big); + sd::Tensor score = score_x * (1.f - latent_mask_) + score_y * latent_mask_; + sd::Tensor x0 = xt + score; + return (std::sqrt(abt) * x0 - xt) / one_minus_abt + A * xt; + }; + + // comfy `run_overdamped` + sd::Tensor C; + bool have_C = false; + if (inner_ok) { + const sd::Tensor dt_half = dt * 0.5f; + for (int i = 0; i < n_eff; ++i) { + if (!have_C) { + C = score_and_C(x_t); + if (C.empty()) { + return sd::Tensor(); + } + x_t = advance_overdamped(x_t, dt, A, C, D2, rng_); + have_C = true; + } else { + x_t = advance_overdamped(x_t, dt_half, A, C, D2, rng_); + sd::Tensor C_new = score_and_C(x_t); + if (C_new.empty()) { + return sd::Tensor(); + } + x_t = x_t + (C_new - C) * dt; + x_t = advance_overdamped(x_t, dt_half, A, C_new, D2, rng_); + C = C_new; + } + } + } + + // 6. Back to native. + if (is_flow) { + const float s = std::sqrt(abt) + std::sqrt(1.f - abt); + x = x_t / s; + } else { + x = x_t * std::sqrt(1.f + ct.ve_sigma * ct.ve_sigma); + } + + // 7. Final model eval at (x, sigma) -> out (blended). + // comfy: out = out*(1-m) + latent_image*m + LanPaintEval out_ev = inner_model_.eval(x, sigma); + if (out_ev.x0.empty()) { + return sd::Tensor(); + } + sd::Tensor out = out_ev.x0 * (1.f - latent_mask_) + latent_image_ * latent_mask_; + return out; + } + + // Wraps `run()` into the callback shape the sampler kernels call. The + // returned function must not outlive the engine, its inner model, or the + // referenced tensors. + std::function&, float, int)> + make_callback(const std::vector& sigmas) const { + return [this, sigmas](sd::Tensor& x, float sigma, int step) -> sd::guidance::GuiderOutput { + const int outer_step = std::abs(step) - 1; + sd::guidance::GuiderOutput result; + result.pred = run(x, sigma, sigmas, outer_step); + return result; + }; + } + +private: + // Overdamped (Gamma -> infinity) Langevin substep (comfy + // `advance_time_overdamped`): + // dx = -A x dt + C dt + D dW_t (C treated as constant over the substep) + // k = (1 - exp(-A dt)) / A (-> dt as A -> 0) + // k2 = (1 - exp(-2 A dt)) / (2 A) + // x = exp(-A dt) x + k C + sqrt(D^2 k2) eps + // `A` and `C` are latent-shaped masked tensors; `dtv` is the (also + // masked, per-pixel) time step `dt` or `dt/2`. + static sd::Tensor advance_overdamped(const sd::Tensor& xin, + const sd::Tensor& dtv, + const sd::Tensor& A, + const sd::Tensor& C, + float D2, + const std::shared_ptr& rng) { + const float eps = 1e-8f; + // A > 0 over the valid domain (A = A_x*(1-m)+A_y*m with A_x, A_y > + // 0); the clamp only guards a divide-by-zero if A were exactly 0. + sd::Tensor A_safe = sd::ops::clamp(A, eps, std::numeric_limits::max()); + sd::Tensor A_dt = A * dtv; + sd::Tensor exp_neg = sd::ops::exp(-A_dt); + sd::Tensor k = (1.f - exp_neg) / A_safe; + sd::Tensor k2 = (1.f - sd::ops::exp(-2.f * A_dt)) / (2.f * A_safe); + + sd::Tensor mean = exp_neg * xin + k * C; + sd::Tensor var = D2 * k2; + sd::Tensor eps_draw = sd::Tensor::randn_like(xin, rng); + return mean + eps_draw * sd::ops::sqrt(sd::ops::clamp(var, 0.f, std::numeric_limits::max())); + } + + LanPaintParams params_; + LanPaintInnerModel inner_model_; + std::shared_ptr rng_; + const sd::Tensor& noise_; + const sd::Tensor& latent_image_; + const sd::Tensor& latent_mask_; +}; + +#endif // __SD_RUNTIME_LANPAINT_HPP__ From b5066ca4b5370f8e14ae7eea561d50cc0ca34a34 Mon Sep 17 00:00:00 2001 From: Chaser Huang Date: Fri, 11 Sep 2026 20:34:28 -0400 Subject: [PATCH 2/6] fix(lanpaint): reject SenseNova U1 and MiniMax H3 AV denoisers Assisted-by: prime-agent glm-5.3-flash Signed-off-by: Chaser Huang --- src/runtime/lanpaint.hpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/runtime/lanpaint.hpp b/src/runtime/lanpaint.hpp index 5df741d03..eb78b1086 100644 --- a/src/runtime/lanpaint.hpp +++ b/src/runtime/lanpaint.hpp @@ -84,19 +84,27 @@ struct LanPaintInnerModel { }; // Builds a LanPaintInnerModel from a runtime denoiser: derives the flow/VE -// mode from the denoiser type and refuses the denoisers whose latent -// scaling / time conventions are unsupported (MiniT2I: noise*2 start, -// reversed time; SeFi: dual timesteps). Returns nullopt when refused. +// mode from the denoiser type and refuses the denoisers whose conventions +// the cycle cannot reproduce (MiniT2I: replace step starts from noise and +// ignores sigma; SeFi: dual timesteps; SenseNova U1.5: noise_scaling +// ignores the latent, so the replace step would overwrite the keep region +// with noise; MiniMax H3 AV: audio rows follow a shifted noise schedule). +// Returns nullopt when refused. inline std::optional make_lanpaint_inner_model(lanpaint_eval_t eval, const std::shared_ptr& denoiser) { if (!denoiser) { LOG_ERROR("LanPaint requires a denoiser"); return std::nullopt; } - if (std::dynamic_pointer_cast(denoiser) || std::dynamic_pointer_cast(denoiser)) { + if (std::dynamic_pointer_cast(denoiser) || std::dynamic_pointer_cast(denoiser) || + std::dynamic_pointer_cast(denoiser)) { LOG_ERROR("LanPaint does not support this denoiser's latent scaling / time conventions"); return std::nullopt; } + if (std::dynamic_pointer_cast(denoiser)) { + LOG_ERROR("LanPaint does not support models with a per-stream audio noise schedule"); + return std::nullopt; + } return LanPaintInnerModel{std::move(eval), denoiser.get(), (bool)std::dynamic_pointer_cast(denoiser)}; From e85afbb8c82efaff4b32d3dc6e0101cec44879d2 Mon Sep 17 00:00:00 2001 From: Chaser Huang Date: Sat, 12 Sep 2026 01:00:14 -0400 Subject: [PATCH 3/6] feat(sampling): add LanPaint Langevin-dynamics inpainting support Assisted-by: prime-agent glm-5.3-flash Signed-off-by: Chaser Huang --- examples/common/common.cpp | 38 ++++++++++ include/stable-diffusion.h | 16 +++++ src/pipeline/diffusion_engine.cpp | 113 +++++++++++++++++++++++++++++- src/pipeline/diffusion_engine.h | 3 +- src/pipeline/image.cpp | 8 ++- src/pipeline/video.cpp | 9 ++- src/runtime/denoiser.hpp | 2 +- src/runtime/guidance.h | 1 + src/runtime/lanpaint.hpp | 18 ++--- src/stable-diffusion.cpp | 32 ++++++++- 10 files changed, 221 insertions(+), 19 deletions(-) diff --git a/examples/common/common.cpp b/examples/common/common.cpp index e8ec40295..992963d31 100644 --- a/examples/common/common.cpp +++ b/examples/common/common.cpp @@ -1137,6 +1137,14 @@ ArgOptions SDGenerationParams::get_options() { "--steps", "number of sample steps (default: 20)", &sample_params.sample_steps}, + {"", + "--lanpaint-n-steps", + "LanPaint number of Langevin steps per sample step, i.e. turns of thinking: (default: 5)", + &sample_params.lanpaint.n_steps}, + {"", + "--lanpaint-early-stop", + "LanPaint drops the inner loop in the last N outer steps: (default: 1)", + &sample_params.lanpaint.early_stop}, {"", "--high-noise-steps", "(high noise) number of sample steps (default: -1 = auto)", @@ -1226,6 +1234,26 @@ ArgOptions SDGenerationParams::get_options() { "--flow-shift", "shift value for Flow models like SD3.x or WAN (default: auto)", &sample_params.flow_shift}, + {"", + "--lanpaint-lambda", + "LanPaint bidirectional guidance scale for the known region: (default: 5.0)", + &sample_params.lanpaint.lambda}, + {"", + "--lanpaint-beta", + "LanPaint time-scale ratio of the y branch: (default: 1.0)", + &sample_params.lanpaint.beta}, + {"", + "--lanpaint-step-size", + "LanPaint Langevin step size: (default: 0.2)", + &sample_params.lanpaint.step_size}, + {"", + "--lanpaint-min-step-frac", + "LanPaint minimum noise fraction below which the step size is pinned: (default: 1.0)", + &sample_params.lanpaint.min_step_frac}, + {"", + "--lanpaint-cfg-big", + "LanPaint BIG guidance scale; default auto resolves to --cfg-scale (Image First) or -0.5 (Prompt First)", + &sample_params.lanpaint.cfg_big}, {"", "--high-noise-cfg-scale", "(high noise) unconditional guidance scale: (default: 7.0)", @@ -1297,6 +1325,16 @@ ArgOptions SDGenerationParams::get_options() { }; options.bool_options = { + {"", + "--lanpaint", + "enable LanPaint Langevin-dynamics inpainting (recommended sampler: euler, requires a mask)", + true, + &sample_params.lanpaint.enabled}, + {"", + "--lanpaint-prompt-first", + "LanPaint Prompt First mode: emphasis prompt following over image quality (default: Image First)", + true, + &sample_params.lanpaint.prompt_first}, {"", "--increase-ref-index", "automatically increase the indices of references images based on the order they are listed (starting with 1).", diff --git a/include/stable-diffusion.h b/include/stable-diffusion.h index 9bbf8c757..be8a743e9 100644 --- a/include/stable-diffusion.h +++ b/include/stable-diffusion.h @@ -283,7 +283,23 @@ typedef struct { sd_slg_params_t slg; } sd_guidance_params_t; +// LanPaint: Langevin-dynamics inpainting (ComfyUI LanPaint port). typedef struct { + bool enabled; // --lanpaint + int n_steps; // inner Langevin steps per outer step (default 5) + float lambda; // keep-region score strength (default 5) + float beta; // time-scale ratio of the y branch (default 1) + float step_size; // Langevin step size (default 0.2) + int early_stop; // drop the inner loop in the last N outer steps (default 1) + float min_step_frac; // pin the step size below this noise fraction (default 1) + bool prompt_first; // Prompt First mode (default false = Image First); + // resolves the BIG-CFG scale to -0.5 instead of txt_cfg + float cfg_big; // explicit BIG-CFG scale override (INFINITY = auto from + // prompt_first / txt_cfg, matching the comfy node) +} sd_lanpaint_params_t; + +typedef struct { + sd_lanpaint_params_t lanpaint; sd_guidance_params_t guidance; enum scheduler_t scheduler; enum sample_method_t sample_method; diff --git a/src/pipeline/diffusion_engine.cpp b/src/pipeline/diffusion_engine.cpp index ecad31a53..c593cafb8 100644 --- a/src/pipeline/diffusion_engine.cpp +++ b/src/pipeline/diffusion_engine.cpp @@ -43,6 +43,7 @@ #include "runtime/audio_processing.h" #include "runtime/denoiser.hpp" #include "runtime/guidance.h" +#include "runtime/lanpaint.hpp" #include "runtime/preview_interval.h" #include "runtime/sample-cache.h" @@ -2145,7 +2146,8 @@ sd::Tensor StableDiffusionGGML::sample(const std::shared_ptr& video_positions) { + const sd::Tensor& video_positions, + const sd_lanpaint_params_t& lanpaint) { struct RunnerEndOnExit { GGMLRunner* runner = nullptr; ~RunnerEndOnExit() { @@ -2194,6 +2196,12 @@ sd::Tensor StableDiffusionGGML::sample(const std::shared_ptr StableDiffusionGGML::sample(const std::shared_ptr(); + } + switch (method) { + case EULER_SAMPLE_METHOD: + case EULER_A_SAMPLE_METHOD: + case HEUN_SAMPLE_METHOD: + case DPM2_SAMPLE_METHOD: + case DPMPP2M_SAMPLE_METHOD: + case DPMPP2Mv2_SAMPLE_METHOD: + break; + default: + LOG_ERROR("LanPaint supports the euler, euler_a, heun, dpm2, dpmpp_2m and dpmpp_2m_v2 samplers"); + return sd::Tensor(); + } + lanpaint_active = true; + lanpaint_cfg_big = std::isfinite(lanpaint.cfg_big) ? lanpaint.cfg_big : (lanpaint.prompt_first ? -0.5f : cfg_scale); + lanpaint_params.n_steps = lanpaint.n_steps; + lanpaint_params.friction = 15.f; + lanpaint_params.lambda = lanpaint.lambda; + lanpaint_params.beta = lanpaint.beta; + lanpaint_params.step_size = lanpaint.step_size; + lanpaint_params.early_stop = lanpaint.early_stop; + lanpaint_params.min_step_frac = lanpaint.min_step_frac; + lanpaint_params.cfg_big = lanpaint_cfg_big; + } + if (version == VERSION_HIDREAM_O1 && !noise.empty()) { noise *= eta; } @@ -2248,7 +2291,7 @@ sd::Tensor StableDiffusionGGML::sample(const std::shared_ptr StableDiffusionGGML::sample(const std::shared_ptr StableDiffusionGGML::sample(const std::shared_ptr lanpaint_engine; + if (lanpaint_active) { + // The LanPaint engine wraps the denoise evaluation into the callback + // shape the sampler kernels call: each kernel model() call runs one + // full Langevin cycle and evolves `x` in place. + const sd::Tensor lanpaint_keep_mask = + denoise_mask.empty() ? sd::Tensor() : 1.f - denoise_mask; + lanpaint_eval_t lanpaint_eval = [&denoise](const sd::Tensor& x, float sigma, int step) -> LanPaintEval { + sd::guidance::GuiderOutput g = denoise(x, sigma, step); + return LanPaintEval{std::move(g.pred), std::move(g.pred_big)}; + }; + auto lanpaint_inner = make_lanpaint_inner_model(std::move(lanpaint_eval), denoiser); + if (!lanpaint_inner.has_value()) { + return sd::Tensor(); + } + lanpaint_engine.emplace(lanpaint_params, + std::move(*lanpaint_inner), + sampler_rng, + noise, + sampling_init_latent, + lanpaint_keep_mask); + denoise_cb_t lanpaint_cb = lanpaint_engine->make_callback(sigmas); + const SDVersion lanpaint_version = version; + effective_denoise = [this, + lanpaint_cb = std::move(lanpaint_cb), + &preview, + steps, + terminal_sigma_is_zero, + &last_progress_us, + lanpaint_version, + preview_final_step](sd::Tensor& x, + float sigma, + int step) -> sd::guidance::GuiderOutput { + sd::guidance::GuiderOutput out = lanpaint_cb(x, sigma, step); + if (out.pred.empty()) { + return out; + } + // One progress/preview update per outer step (the eval calls of + // the inner loop stay silent). + report_sample_progress(step, steps, terminal_sigma_is_zero, &last_progress_us); + if (preview.callback != nullptr && sd_should_preview_denoised() && + sd::preview::should_preview_sample_step(step, steps, terminal_sigma_is_zero, sd_get_preview_interval(), preview_final_step)) { + preview_image(step, out.pred, lanpaint_version, preview.mode, preview.callback, preview.data, false); + } + return out; + }; + } + + auto x0_opt = sample_k_diffusion(method, effective_denoise, x_t, sigmas, sampler_rng, eta, is_flow_denoiser, extra_sample_args, denoiser); if (x0_opt.empty()) { LOG_ERROR("Diffusion model sampling failed"); if (control_net) { diff --git a/src/pipeline/diffusion_engine.h b/src/pipeline/diffusion_engine.h index a3c530b21..df05cc500 100644 --- a/src/pipeline/diffusion_engine.h +++ b/src/pipeline/diffusion_engine.h @@ -442,7 +442,8 @@ class StableDiffusionGGML { float frame_rate, const sd_cache_params_t* cache_params, bool preview_final_step, - const sd::Tensor& video_positions = {}); + const sd::Tensor& video_positions = {}, + const sd_lanpaint_params_t& lanpaint = {}); int get_vae_scale_factor(); diff --git a/src/pipeline/image.cpp b/src/pipeline/image.cpp index aada0f452..c04f697ad 100644 --- a/src/pipeline/image.cpp +++ b/src/pipeline/image.cpp @@ -880,7 +880,9 @@ namespace sd::pipeline { 0, static_cast(request.fps), request.cache_params, - true); + true, + sd::Tensor(), + sd_img_gen_params->sample_params.lanpaint); int64_t sampling_end = ggml_time_ms(); if (!x_0.empty()) { LOG_INFO("sampling completed, taking %.2fs", (sampling_end - sampling_start) * 1.0f / 1000); @@ -1002,7 +1004,9 @@ namespace sd::pipeline { 0, static_cast(request.fps), request.cache_params, - false); + false, + sd::Tensor(), + sd_img_gen_params->sample_params.lanpaint); int64_t hires_sample_end = ggml_time_ms(); if (!x_0.empty()) { LOG_INFO("hires sampling %d/%d completed, taking %.2fs", diff --git a/src/pipeline/video.cpp b/src/pipeline/video.cpp index 503afa530..d919b2443 100644 --- a/src/pipeline/video.cpp +++ b/src/pipeline/video.cpp @@ -1646,7 +1646,8 @@ namespace sd::pipeline { static_cast(request.fps), request.cache_params, true, - latents.video_positions); + latents.video_positions, + sd_vid_gen_params->sample_params.lanpaint); int64_t sampling_end = ggml_time_ms(); if (x_t_sampled.empty()) { LOG_ERROR("sampling(high noise) failed after %.2fs", (sampling_end - sampling_start) * 1.0f / 1000); @@ -1689,7 +1690,8 @@ namespace sd::pipeline { static_cast(request.fps), request.cache_params, plan.high_noise_sample_steps <= 0, - latents.video_positions); + latents.video_positions, + sd_vid_gen_params->sample_params.lanpaint); int64_t sampling_end = ggml_time_ms(); if (final_latent.empty()) { @@ -1828,7 +1830,8 @@ namespace sd::pipeline { static_cast(hires_request.fps), hires_request.cache_params, false, - hires_video_positions); + hires_video_positions, + sd_vid_gen_params->sample_params.lanpaint); sampling_end = ggml_time_ms(); if (final_latent.empty()) { LOG_ERROR("sampling(latent upscale) failed after %.2fs", diff --git a/src/runtime/denoiser.hpp b/src/runtime/denoiser.hpp index 6e6f22d52..5b3195674 100644 --- a/src/runtime/denoiser.hpp +++ b/src/runtime/denoiser.hpp @@ -1566,7 +1566,7 @@ struct SenseNovaU1FlowDenoiser : public DiscreteFlowDenoiser { } }; -typedef std::function&, float, int)> denoise_cb_t; +typedef std::function&, float, int)> denoise_cb_t; static std::pair get_ancestral_step(float sigma_from, float sigma_to, diff --git a/src/runtime/guidance.h b/src/runtime/guidance.h index 3de337042..35097ab85 100644 --- a/src/runtime/guidance.h +++ b/src/runtime/guidance.h @@ -16,6 +16,7 @@ namespace sd::guidance { sd::Tensor pred_uncond; sd::Tensor pred_img_cond; sd::Tensor pred_skip_layer; + sd::Tensor pred_big; }; struct AdaptiveProjectedGuidanceParams { diff --git a/src/runtime/lanpaint.hpp b/src/runtime/lanpaint.hpp index eb78b1086..57c3eec9a 100644 --- a/src/runtime/lanpaint.hpp +++ b/src/runtime/lanpaint.hpp @@ -67,9 +67,12 @@ struct LanPaintEval { // The underlying network evaluation. `x` is in native latent coordinates, // `sigma` in native units (flow t for flow models, VE sigma otherwise) -- -// i.e. exactly the sigma the outer sampler passes at this step. Must be -// pure (no side effects): the caller owns `x`. -using lanpaint_eval_t = std::function& x, float sigma)>; +// i.e. exactly the sigma the outer sampler passes at this step. `step` is +// the outer sampler's step index (1-based, the value the sampler passes its +// own callback), so per-step evaluation state (guidance schedules, timestep +// preparation) behaves exactly as in the outer sampler's own callback. Must +// be pure (no side effects): the caller owns `x`. +using lanpaint_eval_t = std::function& x, float sigma, int step)>; // The model LanPaint paints with: the unblended network eval plus the // Denoiser it belongs to (noise-scaling formulas, flow/VE mode) -- comfy's @@ -162,7 +165,7 @@ struct LanPaint { sd::Tensor run(sd::Tensor& x, float sigma, const std::vector& sigmas, int outer_step) const { if (latent_mask_.empty()) { // No mask: LanPaint is a no-op -- return the plain denoised x0. - LanPaintEval ev = inner_model_.eval(x, sigma); + LanPaintEval ev = inner_model_.eval(x, sigma, outer_step + 1); return ev.x0.empty() ? sd::Tensor() : std::move(ev.x0); } @@ -240,7 +243,7 @@ struct LanPaint { } else { x_native = xt * std::sqrt(1.f + ct.ve_sigma * ct.ve_sigma); } - LanPaintEval ev = inner_model_.eval(x_native, sigma); + LanPaintEval ev = inner_model_.eval(x_native, sigma, outer_step + 1); if (ev.x0.empty()) { return sd::Tensor(); } @@ -287,7 +290,7 @@ struct LanPaint { // 7. Final model eval at (x, sigma) -> out (blended). // comfy: out = out*(1-m) + latent_image*m - LanPaintEval out_ev = inner_model_.eval(x, sigma); + LanPaintEval out_ev = inner_model_.eval(x, sigma, outer_step + 1); if (out_ev.x0.empty()) { return sd::Tensor(); } @@ -298,8 +301,7 @@ struct LanPaint { // Wraps `run()` into the callback shape the sampler kernels call. The // returned function must not outlive the engine, its inner model, or the // referenced tensors. - std::function&, float, int)> - make_callback(const std::vector& sigmas) const { + denoise_cb_t make_callback(const std::vector& sigmas) const { return [this, sigmas](sd::Tensor& x, float sigma, int step) -> sd::guidance::GuiderOutput { const int outer_step = std::abs(step) - 1; sd::guidance::GuiderOutput result; diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index 5a56f4029..16eff8098 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -426,6 +426,15 @@ char* sd_ctx_params_to_str(const sd_ctx_params_t* sd_ctx_params) { void sd_sample_params_init(sd_sample_params_t* sample_params) { *sample_params = {}; + sample_params->lanpaint.enabled = false; + sample_params->lanpaint.n_steps = 5; + sample_params->lanpaint.lambda = 5.0f; + sample_params->lanpaint.beta = 1.0f; + sample_params->lanpaint.step_size = 0.2f; + sample_params->lanpaint.early_stop = 1; + sample_params->lanpaint.min_step_frac = 1.0f; + sample_params->lanpaint.prompt_first = false; + sample_params->lanpaint.cfg_big = INFINITY; sample_params->guidance.txt_cfg = 7.0f; sample_params->guidance.img_cfg = INFINITY; sample_params->guidance.distilled_guidance = 3.5f; @@ -479,7 +488,28 @@ char* sd_sample_params_to_str(const sd_sample_params_t* sample_params) { sample_params->eta, sample_params->shifted_timestep, sample_params->flow_shift, - SAFE_STR(sample_params->extra_sample_args)); + SAFE_STR(sample_params->extra_sample_args), + sample_params->lanpaint.enabled ? "on" : "off"); + + if (sample_params->lanpaint.enabled) { + snprintf(buf + strlen(buf), 4096 - strlen(buf), + " (lanpaint n_steps: %d, " + "lambda: %.2f, " + "beta: %.2f, " + "step_size: %.2f, " + "early_stop: %d, " + "min_step_frac: %.2f, " + "prompt_first: %d, " + "cfg_big: %s)", + sample_params->lanpaint.n_steps, + sample_params->lanpaint.lambda, + sample_params->lanpaint.beta, + sample_params->lanpaint.step_size, + sample_params->lanpaint.early_stop, + sample_params->lanpaint.min_step_frac, + (int)sample_params->lanpaint.prompt_first, + std::isfinite(sample_params->lanpaint.cfg_big) ? "custom" : "auto"); + } return buf; } From 161decfde266835d33b1aa81805646f991f84ec4 Mon Sep 17 00:00:00 2001 From: Chaser Huang Date: Mon, 14 Sep 2026 19:06:56 -0400 Subject: [PATCH 4/6] feat(lanpaint): add schedule logging and extract effective_inner_steps helper Assisted-by: prime-agent glm-5.3-flash Signed-off-by: Chaser Huang --- src/pipeline/diffusion_engine.cpp | 27 +++++++++++++++++++++++++++ src/runtime/lanpaint.hpp | 27 +++++++++++++++++++++------ 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/src/pipeline/diffusion_engine.cpp b/src/pipeline/diffusion_engine.cpp index c593cafb8..25e6d4e75 100644 --- a/src/pipeline/diffusion_engine.cpp +++ b/src/pipeline/diffusion_engine.cpp @@ -2267,6 +2267,33 @@ sd::Tensor StableDiffusionGGML::sample(const std::shared_ptr(sigmas.size()) - 1, + static_cast(i)); + } + LOG_INFO("LanPaint: %d outer steps, up to %d inner steps per outer step, %d model evaluations planned (%d without LanPaint)", + static_cast(sigmas.size()) - 1, + lanpaint.n_steps, + lanpaint_inner_total + static_cast(sigmas.size()) - 1, + static_cast(sigmas.size()) - 1); } if (version == VERSION_HIDREAM_O1 && !noise.empty()) { diff --git a/src/runtime/lanpaint.hpp b/src/runtime/lanpaint.hpp index 57c3eec9a..2f417112b 100644 --- a/src/runtime/lanpaint.hpp +++ b/src/runtime/lanpaint.hpp @@ -136,6 +136,20 @@ struct LanPaint { latent_mask_(latent_mask) { } + // Effective inner-step count for one outer step (comfy EarlyStop gating + + // `min_step_frac_effective_steps`): 0 inside the early-stop window, the + // linear ramp below MinStepFrac, NSteps otherwise. Shared by run() and + // the sampler wiring so the logged schedule matches execution. + static int effective_inner_steps(int n_steps, int early_stop, float min_step_frac, float abt, int total_steps, int outer_step) { + if (total_steps - outer_step <= early_stop) { + return 0; + } + if (min_step_frac > 0.f && (1.f - abt) < min_step_frac && n_steps > 0) { + return std::max(0, static_cast(std::lround((float)n_steps * (1.f - abt) / min_step_frac))); + } + return n_steps; + } + static LanPaintModelTimes compute_times(bool is_flow, float sigma) { LanPaintModelTimes t; if (is_flow) { @@ -177,12 +191,7 @@ struct LanPaint { // 2. Effective inner-step count (comfy `min_step_frac_effective_steps`). const int total_steps = static_cast(sigmas.size()) - 1; - int n_eff = params_.n_steps; - if (total_steps - outer_step <= params_.early_stop) { - n_eff = 0; - } else if (params_.min_step_frac > 0.f && (1.f - abt) < params_.min_step_frac && params_.n_steps > 0.f) { - n_eff = std::max(0, static_cast(std::lround((float)params_.n_steps * (1.f - abt) / params_.min_step_frac))); - } + const int n_eff = effective_inner_steps(params_.n_steps, params_.early_stop, params_.min_step_frac, abt, total_steps, outer_step); // 3. Replace step: the keep region is re-noised around the current // noise level, at every outer step (even when n_eff == 0): @@ -257,9 +266,15 @@ struct LanPaint { // comfy `run_overdamped` sd::Tensor C; bool have_C = false; + if (inner_ok && n_eff == 0) { + LOG_VERBOSE("LanPaint: outer step %d/%d skips the inner loop, sigma %.4f (early stop or ramped-down count)", + outer_step + 1, total_steps, sigma); + } if (inner_ok) { const sd::Tensor dt_half = dt * 0.5f; for (int i = 0; i < n_eff; ++i) { + LOG_VERBOSE("LanPaint: outer step %d/%d, inner step %d/%d, sigma %.4f, abt %.4f", + outer_step + 1, total_steps, sigma, i + 1, n_eff, abt); if (!have_C) { C = score_and_C(x_t); if (C.empty()) { From 90b7c04f0b746d26ac343c52a59d3a2148ad9334 Mon Sep 17 00:00:00 2001 From: Chaser Huang Date: Mon, 14 Sep 2026 20:21:44 -0400 Subject: [PATCH 5/6] fix(lanpaint): fix mask tensor lifetime issue Assisted-by: prime-agent glm-5.3-flash Signed-off-by: Chaser Huang --- src/pipeline/diffusion_engine.cpp | 10 ++++----- src/runtime/lanpaint.hpp | 34 ++++++++++++++++++++----------- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/src/pipeline/diffusion_engine.cpp b/src/pipeline/diffusion_engine.cpp index 25e6d4e75..182015f4c 100644 --- a/src/pipeline/diffusion_engine.cpp +++ b/src/pipeline/diffusion_engine.cpp @@ -2611,15 +2611,15 @@ sd::Tensor StableDiffusionGGML::sample(const std::shared_ptr lanpaint_engine; if (lanpaint_active) { // The LanPaint engine wraps the denoise evaluation into the callback // shape the sampler kernels call: each kernel model() call runs one // full Langevin cycle and evolves `x` in place. - const sd::Tensor lanpaint_keep_mask = - denoise_mask.empty() ? sd::Tensor() : 1.f - denoise_mask; lanpaint_eval_t lanpaint_eval = [&denoise](const sd::Tensor& x, float sigma, int step) -> LanPaintEval { sd::guidance::GuiderOutput g = denoise(x, sigma, step); return LanPaintEval{std::move(g.pred), std::move(g.pred_big)}; @@ -2633,7 +2633,7 @@ sd::Tensor StableDiffusionGGML::sample(const std::shared_ptrmake_callback(sigmas); const SDVersion lanpaint_version = version; effective_denoise = [this, diff --git a/src/runtime/lanpaint.hpp b/src/runtime/lanpaint.hpp index 2f417112b..b844f5900 100644 --- a/src/runtime/lanpaint.hpp +++ b/src/runtime/lanpaint.hpp @@ -11,11 +11,12 @@ * `denoiser.hpp` must not include this header. * * Mask convention (comfy `latent_mask = 1 - denoise_mask`): - * `latent_mask` is a float tensor, 1 = KEEP the original image, 0 = EDIT. - * sd.cpp's `denoise_mask` is a latent-space tensor of shape {W,H,1,1} - * (images) / {W,H,T,1,1} (video) against latents {W,H,C,1} / {W,H,T,C,1}, - * so the caller passes `latent_mask = 1 - denoise_mask` and broadcasting - * is automatic. + * The engine takes the user-facing sd.cpp `denoise_mask` -- a latent-space + * float tensor of shape {W,H,1,1} (images) / {W,H,T,1,1} (video) against + * latents {W,H,C,1} / {W,H,T,C,1}, 1 = EDIT (repaint), empty = no mask -- + * and derives its internal keep mask `latent_mask_ = 1 - denoise_mask` + * (1 = KEEP the original image, 0 = EDIT) used throughout the cycle; + * broadcasting against the latents is automatic. * * Sigma convention: `sigma` is the native denoiser sigma -- flow time t in * [0,1] for the DiscreteFlowDenoiser family, the VE sigma (e.g. @@ -113,8 +114,9 @@ inline std::optional make_lanpaint_inner_model(lanpaint_eval (bool)std::dynamic_pointer_cast(denoiser)}; } -// One LanPaint engine. Holds the per-sample context (the inner model, the -// replace-step noise, the target latent, the keep mask and the RNG); `run()` +// One LanPaint engine. Owns the derived keep mask and borrows the +// replace-step noise, the target latent, the RNG and the inner model; +// `run()` // executes one full outer-step cycle on the sampler's latent in place and // returns the blended denoised x0 (comfy's `KSamplerX0Inpaint.__call__` + // `LanPaint.LanPaint` for one step). `make_callback()` wraps `run()` into @@ -122,18 +124,22 @@ inline std::optional make_lanpaint_inner_model(lanpaint_eval // by non-const reference so the evolved latent is written back to the // sampler's local state, comfy's `input_x.copy_(x)`). struct LanPaint { + // `noise` and `latent_image` are borrowed views and must outlive the + // engine (the sampler wiring binds them to caller-scope tensors); + // `denoise_mask` is the user-facing mask (1 = repaint, empty = no mask), + // from which the engine derives its owned keep mask. LanPaint(const LanPaintParams& params, LanPaintInnerModel inner_model, const std::shared_ptr& rng, const sd::Tensor& noise, const sd::Tensor& latent_image, - const sd::Tensor& latent_mask) + const sd::Tensor& denoise_mask) : params_(params), inner_model_(std::move(inner_model)), rng_(rng), noise_(noise), latent_image_(latent_image), - latent_mask_(latent_mask) { + latent_mask_(1.f - denoise_mask) { } // Effective inner-step count for one outer step (comfy EarlyStop gating + @@ -314,8 +320,9 @@ struct LanPaint { } // Wraps `run()` into the callback shape the sampler kernels call. The - // returned function must not outlive the engine, its inner model, or the - // referenced tensors. + // returned function must not outlive the engine, its inner model, the + // Denoiser, the evaluation closure's captures, or the tensors the engine + // borrows (noise, latent image). denoise_cb_t make_callback(const std::vector& sigmas) const { return [this, sigmas](sd::Tensor& x, float sigma, int step) -> sd::guidance::GuiderOutput { const int outer_step = std::abs(step) - 1; @@ -358,9 +365,12 @@ struct LanPaint { LanPaintParams params_; LanPaintInnerModel inner_model_; std::shared_ptr rng_; + // Borrowed views of the caller's tensors; they must outlive the engine. const sd::Tensor& noise_; const sd::Tensor& latent_image_; - const sd::Tensor& latent_mask_; + // Owned keep mask derived from the caller's denoise mask (see the + // constructor): 1 = keep, 0 = repaint, empty = no-mask passthrough. + sd::Tensor latent_mask_; }; #endif // __SD_RUNTIME_LANPAINT_HPP__ From 0e74bae59fc1b2a016b4a4190860e74ab8895ef0 Mon Sep 17 00:00:00 2001 From: Chaser Huang Date: Tue, 15 Sep 2026 04:44:19 -0400 Subject: [PATCH 6/6] docs(lanpaint): add LanPaint inpainting documentation Assisted-by: prime-agent glm-5.3-flash Signed-off-by: Chaser Huang --- README.md | 2 + docs/lanpaint.md | 97 ++++++++++++++++++++++++++++++++++++++++++ examples/cli/README.md | 3 ++ 3 files changed, 102 insertions(+) create mode 100644 docs/lanpaint.md diff --git a/README.md b/README.md index d99348cbf..8a07d0b42 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,7 @@ API and command-line option may change frequently.*** - [ADetailer](./docs/adetailer.md) - LoRA support, same as [stable-diffusion-webui](https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Features#lora) - Latent Consistency Models support (LCM/LCM-LoRA) + - [LanPaint](./docs/lanpaint.md) training-free inpainting sampler with inner Langevin iterations ("think mode") for all supported models - Faster and memory efficient latent decoding with [TAESD](./docs/taesd.md) - Upscale images generated with [ESRGAN](./docs/esrgan.md) - Supported backends @@ -157,6 +158,7 @@ For runtime and parameter backend placement, see the [backend selection guide](. - [Quantization and GGUF](./docs/quantization_and_gguf.md) - [INT8 convrot safetensors](./docs/int8_convrot.md) - [Inference acceleration via caching](./docs/caching.md) +- [LanPaint inpainting](./docs/lanpaint.md) ## Bindings diff --git a/docs/lanpaint.md b/docs/lanpaint.md new file mode 100644 index 000000000..4e205e383 --- /dev/null +++ b/docs/lanpaint.md @@ -0,0 +1,97 @@ +## LanPaint Inpainting + +LanPaint is a training-free inpainting sampler that gives diffusion models +"think mode": instead of denoising in one pass, it runs up to N inner Langevin +steps per sampling step, letting the model reconcile the prompt-driven region +with the known (kept) region before committing to the update. It is a port of +the [ComfyUI LanPaint extension](https://github.com/scraed/LanPaint) +(official implementation of +["LanPaint: Training-Free Diffusion Inpainting with Asymptotically Exact and Fast Conditional Sampling"](https://arxiv.org/abs/2502.03491), +TMLR 2025), with the parameter defaults of the v2.1.0 ComfyUI node. + +In essence, the inpainting problem is modeled as a conditional stochastic +process, and LanPaint integrates it numerically with +[Langevin dynamics](https://en.wikipedia.org/wiki/Langevin_dynamics). The +drift of the process is assembled from the underlying model's own denoising +solutions, i.e. its inferred solutions of the unconditional version of the +same process, so the conditional problem is solved without retraining. Each +inner iteration is one integration step of this process, and each step costs +one model evaluation; that is the exchange of extra compute for inpainting +quality: up to `1 + n_steps` model evaluations per sampling step instead of one. + +### Usage + +Provide an init image and a mask and enable LanPaint: + +```bash +sd-cli -m model.safetensors -i images/input.png --mask images/mask.png \ + -p "a cozy living room, photorealistic" \ + --strength 1.0 --sampling-method euler --lanpaint -v +``` + +Mask convention (same as the rest of `stable-diffusion.cpp`): **white (255) +marks the region that gets repainted, black (0) marks the region that is kept.** +Masks are binarized at load time and evaluated at latent resolution. When no +mask is given, LanPaint is inactive and sampling proceeds like the plain +sampler. + +The inner loop needs both a conditional and an unconditional model branch. +Guidance-distilled models without an unconditional branch (for example +turbo/schnell variants) run, but the keep-region guidance degenerates and the +LanPaint benefit largely disappears. + +### Parameters + +| Flag | Description | Default | +|------|-------------|---------| +| `--lanpaint` | enable LanPaint | off | +| `--lanpaint-n-steps` | number of inner Langevin steps per sampling step ("turns of thinking"); the effective count ramps down near the end of the schedule | 5 | +| `--lanpaint-lambda` | strength of the keep-region (bidirectional) guidance | 5.0 | +| `--lanpaint-beta` | time-scale ratio of the keep-region branch | 1.0 | +| `--lanpaint-step-size` | Langevin step size; scaled by the remaining noise fraction of the current step | 0.2 | +| `--lanpaint-early-stop` | skip the inner loop for the last N sampling steps (they only polish with a plain evaluation) | 1 | +| `--lanpaint-min-step-frac` | when the remaining noise fraction drops below this value, the step size is pinned there and the inner-step count ramps down to zero | 1.0 | +| `--lanpaint-prompt-first` | Prompt First mode: sets the BIG guidance scale to -0.5, emphasizing prompt following over mask-boundary coherence | off (Image First) | +| `--lanpaint-cfg-big` | explicit BIG guidance scale override; by default it resolves to `--cfg-scale` (Image First) or -0.5 (Prompt First) | auto | + +`Image First` (default) and `Prompt First` change how strongly the inner loop +is anchored to the already-known image versus the prompt. Use Image First for +seamless object insertion in the known surroundings; use Prompt First when the +repainted region should follow the prompt even at the cost of boundary +coherence. + +LanPaint inherits every per-step feature of the outer sampler: conditioning, Control +Net, IP-Adapter, reference latents, video masks, previews and cancellation. + +### Cost + +Each inner step costs one model evaluation (conditional + unconditional), so a +run needs up to `steps x (1 + n_steps)` evaluations. At INFO log level the +planned count is printed before sampling starts: + +``` +LanPaint: 16 outer steps, up to 5 inner steps per outer step, 68 model evaluations planned (16 without LanPaint) +``` + +At VERBOSE level every inner step is logged. With the defaults, `early_stop` +and the `min_step_frac` ramp already keep the tail of the schedule cheap. +Recommended `--lanpaint-n-steps` range: 2-8 (the upstream default of 5 is a +good quality/speed balance). + +### Supported samplers and models + +- Samplers: `euler` (recommended), `euler_a`, `heun`, `dpm2`, `dpm++2m`, + `dpm++2mv2`. Other samplers are rejected with an error. +- Models: everything using the standard denoiser noise scaling, including + SD1.x/SD2.x (eps, v-prediction, EDM), SDXL, SD3/SD3.5, FLUX, Chroma, + Qwen-Image, Wan, HunyuanVideo, and LTX video models. +- Not supported (rejected with an error): + - MiniT2I and SeFi (custom latent scaling / dual time conventions), + - SenseNova U1.5 (its noise scaling discards the latent, which would + overwrite the kept region), + - MiniMax-H3 and LTX-AV (audio rows follow a per-stream noise schedule the + inner loop does not model). +- Sampling caches (`--cache-mode`, for example) are disabled under LanPaint: + the inner loop's repeated evaluations at one step index break their reuse + accounting. + diff --git a/examples/cli/README.md b/examples/cli/README.md index fe47122b6..839b0280d 100644 --- a/examples/cli/README.md +++ b/examples/cli/README.md @@ -14,6 +14,9 @@ equivalent to `--log-level verbose`. If repeated, the last logging option wins. For direct image repair or automatic post-generation YOLOv8 detection followed by cropped inpainting, see [ADetailer](../../docs/adetailer.md). +For high-quality inpainting with the Langevin-based LanPaint sampler (`--lanpaint`), see +[LanPaint](../../docs/lanpaint.md). + Metadata mode inspects PNG/JPEG container metadata without loading any model: ```bash