From ab156c781733f161307c14b95fa39179fd692457 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Sun, 23 Aug 2026 23:05:19 +0200 Subject: [PATCH 01/19] add sd-fit-params: fit placement to free device memory using measured dry runs Runs the generation pipeline in a metadata-only measure mode: graphs are built and compute buffers measured with ggml_gallocr_reserve_n_size, no weights are read and nothing is allocated. The measured per-module memory is packed against per-device budgets and emitted as --backend / --params-backend / --vae-tiling arguments, via the new sd_fit_params() API and the sd-fit-params tool. Assisted-by: Claude Fable 5 --- docs/backend.md | 28 +++ examples/CMakeLists.txt | 1 + examples/fit-params/CMakeLists.txt | 26 +++ examples/fit-params/README.md | 53 +++++ examples/fit-params/main.cpp | 126 +++++++++++ include/stable-diffusion.h | 32 +++ src/core/fit_params.cpp | 342 +++++++++++++++++++++++++++++ src/core/fit_params.h | 38 ++++ src/core/ggml_extend.hpp | 97 ++++++++ src/stable-diffusion.cpp | 220 ++++++++++++++++++- 10 files changed, 961 insertions(+), 2 deletions(-) create mode 100644 examples/fit-params/CMakeLists.txt create mode 100644 examples/fit-params/README.md create mode 100644 examples/fit-params/main.cpp create mode 100644 src/core/fit_params.cpp create mode 100644 src/core/fit_params.h diff --git a/docs/backend.md b/docs/backend.md index c936b2232..ee9de9132 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -142,6 +142,34 @@ GPUs with the layer/row split mechanism (`--split-mode` selects which, layer by default). Components that fit nowhere fall back to the CPU. If a VAE decode still runs out of memory, tiling is enabled and the decode retried once. +## Measured fitting (`sd-fit-params`) + +`sd-fit-params` is a standalone tool that derives the same kind of placement, +but from *measured* memory instead of auto-fit's fixed compute reserves. It +runs the real generation pipeline in a metadata-only dry run: every module's +compute graphs are built for the requested width/height/frames and their +compute buffer sizes are measured without allocating anything or reading any +weight data, so a fit takes seconds even for very large models. Because +compute memory depends on the generation parameters, they are inputs to the +tool, and the printed arguments are valid for workloads up to that size. + +Logs go to stderr, the fitted arguments go to stdout: + +```shell +sd-fit-params -m model.gguf -W 1024 -H 1024 | tee args.txt +cat args.txt | xargs sd-cli -m model.gguf -p "a cat" -W 1024 -H 1024 +``` + +`--fit-print` prints the measured per-device / per-module memory table instead +of arguments. Budgets reuse `--max-vram` with the same semantics as auto-fit. +If the current parameters already fit, nothing needs to change and the tool +prints an empty line. If `--backend` / `--params-backend` are already set and +changes would be needed, the tool fails instead of overriding them. + +The same measurement is available to library users through `sd_fit_params()` +in `stable-diffusion.h`, which takes the context params plus an +`sd_fit_workload_t` and returns the derived specs and a report. + ## Modules | Module | Purpose | Accepted names | diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index b347f1785..e0dbb75cf 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -1,6 +1,7 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}) add_subdirectory(cli) +add_subdirectory(fit-params) add_subdirectory(server) if(NOT SD_BUILD_SHARED_LIBS) # ABot validation harnesses link engine internals not exported by the shared library. diff --git a/examples/fit-params/CMakeLists.txt b/examples/fit-params/CMakeLists.txt new file mode 100644 index 000000000..2fea96f59 --- /dev/null +++ b/examples/fit-params/CMakeLists.txt @@ -0,0 +1,26 @@ +set(TARGET sd-fit-params) + +add_executable(${TARGET} + ../common/common.cpp + ../common/log.cpp + ../common/media_io.cpp + main.cpp +) +if(APPLE) + sd_set_macos_rpaths(${TARGET}) +endif() +target_include_directories(${TARGET} PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/.." + "${PROJECT_SOURCE_DIR}/src" +) +install(TARGETS ${TARGET} RUNTIME) +target_link_libraries(${TARGET} PRIVATE stable-diffusion zip ${CMAKE_THREAD_LIBS_INIT}) +if(SD_WEBP) + target_compile_definitions(${TARGET} PRIVATE SD_USE_WEBP) + target_link_libraries(${TARGET} PRIVATE webp libwebpmux) +endif() +if(SD_WEBM) + target_compile_definitions(${TARGET} PRIVATE SD_USE_WEBM) + target_link_libraries(${TARGET} PRIVATE webm) +endif() +target_compile_features(${TARGET} PUBLIC c_std_11 cxx_std_17) diff --git a/examples/fit-params/README.md b/examples/fit-params/README.md new file mode 100644 index 000000000..1582aeb05 --- /dev/null +++ b/examples/fit-params/README.md @@ -0,0 +1,53 @@ +# sd-fit-params + +`sd-fit-params` computes the CLI arguments that make a model fit into free +device memory, using measured metadata-only dry runs: the real generation +pipeline is executed once with graph building and memory measurement only, so +no weight data is read and no buffers are allocated. The measured per-module +memory is then packed against the free memory of every GPU device (minus a +512 MiB margin, or the `--max-vram` budgets) and the resulting placement is +printed to stdout as `--backend` / `--params-backend` / `--vae-tiling` +arguments. + +Because compute memory depends on the generation parameters, pass the same +width/height (and video frames) you intend to generate with. Example usage: + +``` bash +# First, run sd-fit-params and store the results in a file: +> ./build/bin/sd-fit-params -m sd_v1-5.gguf -W 1024 -H 1024 --max-vram 4 | tee args.txt +[INFO ] fit_params.cpp:93 - fit-params: measured memory plan +[INFO ] fit_params.cpp:93 - devices: +[INFO ] fit_params.cpp:93 - MTL0 Apple M4 free 12123 MiB, budget 4096 MiB +[INFO ] fit_params.cpp:93 - modules (measured for this workload): +[INFO ] fit_params.cpp:93 - diffusion params 1398 MiB, compute 8360 MiB +[INFO ] fit_params.cpp:93 - te params 125 MiB, compute 1 MiB +[INFO ] fit_params.cpp:93 - vae params 159 MiB, compute 6656 MiB (tiled 416 MiB) +[INFO ] fit_params.cpp:93 - placement (time-share: params load per phase and free after): +[INFO ] fit_params.cpp:93 - diffusion -> cpu +[INFO ] fit_params.cpp:93 - te -> MTL0, params on disk +[INFO ] fit_params.cpp:93 - vae -> MTL0, params on disk, vae tiling +[INFO ] stable-diffusion.cpp - fit-params: fitting params to free memory took 0.26s +printing fitted CLI arguments to stdout... +--backend "diffusion=cpu,te=MTL0,vae=MTL0" --params-backend "te=disk,vae=disk" --vae-tiling + +# Next, use those results for sd-cli: +> cat args.txt | xargs ./build/bin/sd-cli -m sd_v1-5.gguf -p "a cat" -W 1024 -H 1024 +``` + +Useful flags: + +- `-W` / `-H` / `--video-frames`: the workload the fit must accommodate +- `--max-vram ` or `--max-vram cuda0=8,cuda1=14`: per-device budgets + (default: free memory minus 512 MiB per device) +- `--fit-print`: print the measured memory table to stdout instead of arguments +- `-p`: representative prompt (token count affects text encoder memory) +- model placement inputs such as `--type`, `--diffusion-fa`, `--vae-tiling` + flow into the measurement exactly as they would into a real run + +If the current parameters already fit, the tool prints an empty line and +reports that no changes are needed. If `--backend` / `--params-backend` are +already set and changes would be needed, the tool fails instead of overriding +them. + +See `docs/backend.md` for the placement spec syntax and the heuristic +`--auto-fit` alternative built into `sd-cli`. diff --git a/examples/fit-params/main.cpp b/examples/fit-params/main.cpp new file mode 100644 index 000000000..eda0e13f0 --- /dev/null +++ b/examples/fit-params/main.cpp @@ -0,0 +1,126 @@ +#include +#include +#include + +#include "stable-diffusion.h" + +#include "common/common.h" + +struct SDFitCliParams { + bool verbose = false; + bool color = false; + bool fit_print = false; + + ArgOptions get_options() { + ArgOptions options; + options.bool_options = { + {"-v", "--verbose", "print extra info", true, &verbose}, + {"", "--color", "colors the logging tags", true, &color}, + {"", "--fit-print", "print the measured memory report to stdout instead of fitted arguments", true, &fit_print}, + }; + return options; + } +}; + +static void print_usage(int argc, const char* argv[], const std::vector& options_list) { + fprintf(stderr, "usage: %s [arguments]\n", argv[0]); + fprintf(stderr, "\n"); + fprintf(stderr, "Measures the memory the model needs for the requested generation parameters using\n"); + fprintf(stderr, "metadata-only dry runs (no weight data is read), then prints the CLI arguments that\n"); + fprintf(stderr, "make it fit into free device memory. Logs go to stderr, arguments to stdout:\n"); + fprintf(stderr, "\n"); + fprintf(stderr, " %s -m model.gguf -W 1024 -H 1024 | tee args.txt\n", argv[0]); + fprintf(stderr, " cat args.txt | xargs sd-cli -m model.gguf -p \"a cat\" -W 1024 -H 1024\n"); + fprintf(stderr, "\n"); + fprintf(stderr, "arguments:\n"); + for (const auto& options : options_list) { + options.print(); + } +} + +// keep stdout clean for the fitted arguments +static void fit_log_cb(enum sd_log_level_t level, const char* log, void* data) { + SDFitCliParams* params = (SDFitCliParams*)data; + if (!params->verbose && level == SD_LOG_DEBUG) { + return; + } + const char* level_str[] = {"DEBUG", "INFO", "WARN", "ERROR"}; + fprintf(stderr, "[%-5s] %s", level >= 0 && level <= 3 ? level_str[level] : "?", SAFE_STR(log)); + fflush(stderr); +} + +int main(int argc, const char* argv[]) { + if (argc > 1 && std::string(argv[1]) == "--version") { + printf("%s\n", version_string().c_str()); + return 0; + } + + SDFitCliParams fit_params; + SDContextParams ctx_params; + SDGenerationParams gen_params; + + std::vector options_vec = {fit_params.get_options(), ctx_params.get_options(), gen_params.get_options()}; + if (!parse_options(argc, argv, options_vec)) { + print_usage(argc, argv, options_vec); + return 1; + } + + sd_set_log_callback(fit_log_cb, (void*)&fit_params); + + SDMode mode = gen_params.video_frames > 1 ? VID_GEN : IMG_GEN; + if (!ctx_params.resolve_and_validate(mode) || + !gen_params.resolve_and_validate(mode, ctx_params.lora_model_dir, ctx_params.hires_upscalers_dir)) { + print_usage(argc, argv, options_vec); + return 1; + } + + sd_ctx_params_t sd_ctx_params = ctx_params.to_sd_ctx_params_t(false); + + sd_fit_workload_t workload; + sd_fit_workload_init(&workload); + workload.prompt = gen_params.prompt.c_str(); + workload.width = gen_params.get_resolved_width(); + workload.height = gen_params.get_resolved_height(); + workload.video_frames = gen_params.video_frames; + workload.vae_tiling_params = gen_params.vae_tiling_params; + + sd_fit_result_t result; + enum sd_fit_status_t status = sd_fit_params(&sd_ctx_params, &workload, &result); + if (status != SD_FIT_SUCCESS) { + if (fit_params.fit_print && result.report != nullptr) { + printf("%s", result.report); + } + fprintf(stderr, "failed to fit CLI arguments to free memory, exiting...\n"); + sd_fit_result_free(&result); + return 1; + } + + if (fit_params.fit_print) { + printf("%s", SAFE_STR(result.report)); + } else if (result.changed) { + std::string args; + if (result.backend != nullptr) { + args += std::string("--backend \"") + result.backend + "\""; + } + if (result.params_backend != nullptr) { + if (!args.empty()) { + args += " "; + } + args += std::string("--params-backend \"") + result.params_backend + "\""; + } + if (result.vae_tiling && !workload.vae_tiling_params.enabled) { + if (!args.empty()) { + args += " "; + } + args += "--vae-tiling"; + } + fprintf(stderr, "printing fitted CLI arguments to stdout...\n"); + printf("%s\n", args.c_str()); + } else { + fprintf(stderr, "current parameters already fit into free device memory, no changes needed\n"); + printf("\n"); + } + + sd_fit_result_free(&result); + return 0; +} diff --git a/include/stable-diffusion.h b/include/stable-diffusion.h index 258465976..019ce2327 100644 --- a/include/stable-diffusion.h +++ b/include/stable-diffusion.h @@ -500,6 +500,38 @@ SD_API void sd_hires_params_init(sd_hires_params_t* hires_params); SD_API void sd_ctx_params_init(sd_ctx_params_t* sd_ctx_params); SD_API char* sd_ctx_params_to_str(const sd_ctx_params_t* sd_ctx_params); +// the workload the fitted parameters must accommodate: compute memory depends on it +typedef struct { + const char* prompt; // representative prompt, token count affects text encoder memory + int width; + int height; + int video_frames; // <= 1 for image generation + sd_tiling_params_t vae_tiling_params; +} sd_fit_workload_t; + +enum sd_fit_status_t { + SD_FIT_SUCCESS = 0, // found a placement projected to fit (or no changes needed) + SD_FIT_FAILURE = 1, // could not find a placement projected to fit + SD_FIT_ERROR = 2, // a hard error occurred, e.g. the model could not be read +}; + +typedef struct { + bool changed; // false = current/default placement already fits, specs below are NULL + char* backend; // fitted runtime placement spec for --backend, owned by the result + char* params_backend; // fitted params placement spec for --params-backend, owned by the result + bool vae_tiling; // recommend enabling VAE tiling + char* report; // human readable per-device / per-module memory table +} sd_fit_result_t; + +SD_API void sd_fit_workload_init(sd_fit_workload_t* workload); + +// fit model/context placement to free device memory using measured no-alloc dry runs; +// reads only model metadata, never weight data +SD_API enum sd_fit_status_t sd_fit_params(const sd_ctx_params_t* sd_ctx_params, + const sd_fit_workload_t* workload, + sd_fit_result_t* result); +SD_API void sd_fit_result_free(sd_fit_result_t* result); + SD_API sd_ctx_t* new_sd_ctx(const sd_ctx_params_t* sd_ctx_params); SD_API void free_sd_ctx(sd_ctx_t* sd_ctx); SD_API void free_sd_audio(sd_audio_t* audio); diff --git a/src/core/fit_params.cpp b/src/core/fit_params.cpp new file mode 100644 index 000000000..918bfdda1 --- /dev/null +++ b/src/core/fit_params.cpp @@ -0,0 +1,342 @@ +#include "fit_params.h" + +#include +#include +#include +#include + +#include "core/util.h" +#include "ggml-backend.h" + +namespace sd::fit_params { + namespace { + + constexpr int64_t MiB = 1024ll * 1024; + + struct Device { + ggml_backend_dev_t dev = nullptr; + std::string name; + std::string description; + int64_t free_bytes = 0; + int64_t total_bytes = 0; + int64_t budget_bytes = 0; + }; + + struct Decision { + bool placed = false; + bool on_cpu = false; + bool disk_params = false; + bool tiled = false; + std::vector device_idxs; + }; + + std::vector enumerate_gpu_devices(sd::ggml_graph_cut::MaxVramAssignment& budgets) { + std::vector out; + for (size_t i = 0; i < ggml_backend_dev_count(); i++) { + ggml_backend_dev_t dev = ggml_backend_dev_get(i); + if (ggml_backend_dev_type(dev) != GGML_BACKEND_DEVICE_TYPE_GPU) { + continue; + } + Device d; + d.dev = dev; + d.name = ggml_backend_dev_name(dev); + d.description = ggml_backend_dev_description(dev); + size_t free_bytes = 0, total_bytes = 0; + ggml_backend_dev_memory(dev, &free_bytes, &total_bytes); + d.free_bytes = (int64_t)free_bytes; + d.total_bytes = (int64_t)total_bytes; + + ggml_backend_t backend = nullptr; // bytes_for_backend needs a backend, resolve via name instead + (void)backend; + float gib = budgets.default_gib; + { + std::string budget_key = d.name; + std::transform(budget_key.begin(), budget_key.end(), budget_key.begin(), [](unsigned char c) { return (char)std::tolower(c); }); + auto it = budgets.backend_gib.find(budget_key); + if (it != budgets.backend_gib.end()) { + gib = it->second; + } + } + if (gib > 0.f) { + d.budget_bytes = std::min((int64_t)(gib * 1024.0 * 1024.0 * 1024.0), d.free_bytes); + } else if (gib < 0.f) { + d.budget_bytes = d.free_bytes + (int64_t)(gib * 1024.0 * 1024.0 * 1024.0); + } else { + d.budget_bytes = d.free_bytes - 512 * MiB; + } + d.budget_bytes = std::max(d.budget_bytes, 0); + out.push_back(d); + } + return out; + } + + std::string module_spec_key(SDBackendModule module) { + // sd_backend_module_name returns tokens sd_parse_backend_assignment accepts + return sd_backend_module_name(module); + } + + void append_assignment(std::string& spec, const std::string& key, const std::string& value) { + if (!spec.empty()) { + spec += ","; + } + spec += key; + spec += "="; + spec += value; + } + + void report_line(std::string& report, const char* fmt, ...) { + char line[512]; + va_list args; + va_start(args, fmt); + vsnprintf(line, sizeof(line), fmt, args); + va_end(args); + LOG_INFO("%s", line); + report += line; + report += "\n"; + } + + } // namespace + + bool plan_placement(const std::vector& modules, + sd::ggml_graph_cut::MaxVramAssignment& budgets, + FitPlan* plan) { + if (plan == nullptr) { + return false; + } + *plan = {}; + + { + std::string error; + if (!budgets.canonicalize_backend_keys(&error)) { + LOG_ERROR("%s", error.c_str()); + return false; + } + } + + std::vector devices = enumerate_gpu_devices(budgets); + + report_line(plan->report, "fit-params: measured memory plan"); + report_line(plan->report, " devices:"); + for (const Device& d : devices) { + report_line(plan->report, " %-12s %-32s free %6lld MiB, budget %6lld MiB", + d.name.c_str(), d.description.c_str(), + (long long)(d.free_bytes / MiB), (long long)(d.budget_bytes / MiB)); + } + report_line(plan->report, " modules (measured for this workload):"); + for (const ModuleMemory& m : modules) { + if (m.params_bytes == 0 && m.compute_bytes == 0) { + continue; + } + if (m.compute_bytes_tiled > 0) { + report_line(plan->report, " %-12s params %6lld MiB, compute %6lld MiB (tiled %6lld MiB)", + module_spec_key(m.module).c_str(), + (long long)(m.params_bytes / MiB), + (long long)(m.compute_bytes / MiB), + (long long)(m.compute_bytes_tiled / MiB)); + } else { + report_line(plan->report, " %-12s params %6lld MiB, compute %6lld MiB", + module_spec_key(m.module).c_str(), + (long long)(m.params_bytes / MiB), + (long long)(m.compute_bytes / MiB)); + } + } + + if (devices.empty()) { + report_line(plan->report, " no usable GPU devices; keeping the default backend"); + plan->valid = true; + plan->changed = false; + return true; + } + + // check-first: the default placement puts every module on the default (first GPU) device + { + int64_t params_sum = 0; + int64_t compute_max = 0; + for (const ModuleMemory& m : modules) { + params_sum += (int64_t)m.params_bytes; + compute_max = std::max(compute_max, (int64_t)m.compute_bytes); + } + if (params_sum + compute_max <= devices[0].budget_bytes) { + report_line(plan->report, " projected use %lld MiB <= budget %lld MiB on %s, no changes needed", + (long long)((params_sum + compute_max) / MiB), + (long long)(devices[0].budget_bytes / MiB), + devices[0].name.c_str()); + plan->valid = true; + plan->changed = false; + return true; + } + } + + std::vector order(modules.size()); + for (size_t i = 0; i < order.size(); i++) { + order[i] = i; + } + std::sort(order.begin(), order.end(), [&](size_t a, size_t b) { + return modules[a].params_bytes > modules[b].params_bytes; + }); + + std::vector decisions(modules.size()); + bool time_share = false; + + // resident plan: every module keeps its params loaded, compute buffers coexist per device + { + std::vector params_sum(devices.size(), 0); + std::vector max_compute(devices.size(), 0); + bool ok = true; + std::vector resident(modules.size()); + for (size_t mi : order) { + const ModuleMemory& m = modules[mi]; + if (m.params_bytes == 0 && m.compute_bytes == 0) { + resident[mi].placed = true; + continue; + } + int best = -1; + for (size_t di = 0; di < devices.size(); di++) { + int64_t need = params_sum[di] + (int64_t)m.params_bytes + + std::max(max_compute[di], (int64_t)m.compute_bytes); + if (need <= devices[di].budget_bytes && + (best < 0 || devices[di].budget_bytes - params_sum[di] > devices[best].budget_bytes - params_sum[best])) { + best = (int)di; + } + } + if (best < 0) { + ok = false; + break; + } + params_sum[best] += (int64_t)m.params_bytes; + max_compute[best] = std::max(max_compute[best], (int64_t)m.compute_bytes); + resident[mi].placed = true; + resident[mi].device_idxs.push_back((size_t)best); + } + if (ok) { + decisions = std::move(resident); + } else { + time_share = true; + } + } + + // time-share plan: phases run sequentially, heavy modules load per phase and free after + if (time_share) { + for (size_t mi : order) { + const ModuleMemory& m = modules[mi]; + Decision& decision = decisions[mi]; + decision = {}; + if (m.params_bytes == 0 && m.compute_bytes == 0) { + decision.placed = true; + continue; + } + int best = -1; + for (size_t di = 0; di < devices.size(); di++) { + if ((int64_t)m.params_bytes + (int64_t)m.compute_bytes <= devices[di].budget_bytes && + (best < 0 || devices[di].budget_bytes > devices[best].budget_bytes)) { + best = (int)di; + } + } + if (best >= 0) { + decision.placed = true; + decision.disk_params = true; + decision.device_idxs.push_back((size_t)best); + continue; + } + if (m.compute_bytes_tiled > 0) { + for (size_t di = 0; di < devices.size(); di++) { + if ((int64_t)m.params_bytes + (int64_t)m.compute_bytes_tiled <= devices[di].budget_bytes && + (best < 0 || devices[di].budget_bytes > devices[best].budget_bytes)) { + best = (int)di; + } + } + if (best >= 0) { + decision.placed = true; + decision.disk_params = true; + decision.tiled = true; + plan->vae_tiling = true; + decision.device_idxs.push_back((size_t)best); + continue; + } + } + if (m.splittable && devices.size() > 1) { + int64_t capacity = 0; + for (const Device& d : devices) { + capacity += std::max(d.budget_bytes - (int64_t)m.compute_bytes, 0); + } + if ((int64_t)m.params_bytes <= capacity) { + decision.placed = true; + decision.disk_params = true; + std::vector idxs(devices.size()); + for (size_t i = 0; i < idxs.size(); i++) { + idxs[i] = i; + } + std::sort(idxs.begin(), idxs.end(), [&](size_t a, size_t b) { + return devices[a].budget_bytes > devices[b].budget_bytes; + }); + decision.device_idxs = std::move(idxs); + continue; + } + } + decision.placed = true; + decision.on_cpu = true; + } + } + + report_line(plan->report, " placement%s:", time_share ? " (time-share: params load per phase and free after)" : ""); + for (size_t mi = 0; mi < modules.size(); mi++) { + const ModuleMemory& m = modules[mi]; + const Decision& decision = decisions[mi]; + if (m.params_bytes == 0 && m.compute_bytes == 0) { + continue; + } + std::string target; + if (decision.on_cpu) { + target = "cpu"; + } else { + for (size_t k = 0; k < decision.device_idxs.size(); k++) { + if (k > 0) { + target += " & "; + } + target += devices[decision.device_idxs[k]].name; + } + if (decision.device_idxs.size() > 1) { + target += " (split)"; + } + } + report_line(plan->report, " %-12s -> %s%s%s", + module_spec_key(m.module).c_str(), + target.c_str(), + decision.disk_params ? ", params on disk" : "", + decision.tiled ? ", vae tiling" : ""); + } + + for (size_t mi = 0; mi < modules.size(); mi++) { + const ModuleMemory& m = modules[mi]; + const Decision& decision = decisions[mi]; + if (m.params_bytes == 0 && m.compute_bytes == 0) { + continue; + } + const std::string key = module_spec_key(m.module); + if (decision.on_cpu) { + append_assignment(plan->runtime_spec, key, "cpu"); + continue; + } + if (decision.device_idxs.empty()) { + continue; + } + std::string device_list; + for (size_t k = 0; k < decision.device_idxs.size(); k++) { + if (k > 0) { + device_list += "&"; + } + device_list += devices[decision.device_idxs[k]].name; + } + append_assignment(plan->runtime_spec, key, device_list); + if (decision.disk_params) { + append_assignment(plan->params_spec, key, "disk"); + } + } + + plan->valid = true; + plan->changed = true; + plan->time_share = time_share; + return true; + } + +} // namespace sd::fit_params diff --git a/src/core/fit_params.h b/src/core/fit_params.h new file mode 100644 index 000000000..b0ac83756 --- /dev/null +++ b/src/core/fit_params.h @@ -0,0 +1,38 @@ +#ifndef __SD_FIT_PARAMS_H__ +#define __SD_FIT_PARAMS_H__ + +#include +#include + +#include "core/ggml_extend_backend.h" +#include "core/ggml_graph_cut.h" + +namespace sd::fit_params { + + // measured memory requirements for one module at the requested workload + struct ModuleMemory { + SDBackendModule module; + size_t params_bytes = 0; // weights registered for the module + size_t compute_bytes = 0; // largest measured compute buffer among the module's graphs + size_t compute_bytes_tiled = 0; // VAE only: compute buffer with tiling enabled, 0 if not measured + bool splittable = false; + }; + + struct FitPlan { + bool valid = false; + bool changed = false; // false = current/default placement already fits + bool time_share = false; + bool vae_tiling = false; + std::string runtime_spec; + std::string params_spec; + std::string report; // human readable per-device / per-module table + }; + + // derive placement specs from measured module memory and per-device budgets + bool plan_placement(const std::vector& modules, + sd::ggml_graph_cut::MaxVramAssignment& budgets, + FitPlan* plan); + +} // namespace sd::fit_params + +#endif // __SD_FIT_PARAMS_H__ diff --git a/src/core/ggml_extend.hpp b/src/core/ggml_extend.hpp index 017d3d8f3..9aef30575 100644 --- a/src/core/ggml_extend.hpp +++ b/src/core/ggml_extend.hpp @@ -1747,6 +1747,15 @@ struct GGMLRunnerContext { }; struct GGMLRunner { + struct graph_memory_measurement { + std::string desc; + const GGMLRunner* runner = nullptr; + ggml_backend_t backend = nullptr; + size_t compute_bytes = 0; + size_t params_bytes = 0; + bool valid = false; + }; + protected: typedef std::function get_graph_cb_t; using GraphCutSegment = sd::ggml_graph_cut::Segment; @@ -1783,6 +1792,12 @@ struct GGMLRunner { std::unordered_set runner_param_tensor_set; bool params_tensor_set_dirty_ = true; + // static so nested runners (e.g. text encoders inside a conditioner) are also + // intercepted; measurement is single-threaded like the rest of param fitting + static inline bool measure_mode_ = false; + static inline std::vector* measure_collector_ = nullptr; + graph_memory_measurement last_measurement_; + std::vector one_vec = {1.f}; ggml_tensor* one_tensor = nullptr; @@ -2270,6 +2285,60 @@ struct GGMLRunner { return true; } + // measure the compute buffer size and used param bytes of a built graph without + // allocating anything: params are temporarily marked as externally owned so + // gallocr does not reserve them (same trick as measure_segment_compute_buffer) + void measure_graph_memory(ggml_cgraph* gf) { + last_measurement_ = {}; + + struct TensorRuntimeBinding { + ggml_backend_buffer_t buffer = nullptr; + void* data = nullptr; + void* extra = nullptr; + }; + std::unordered_map saved_bindings; + auto mark_external = [&](ggml_tensor* t) { + if (t == nullptr || saved_bindings.find(t) != saved_bindings.end()) { + return; + } + saved_bindings[t] = {t->buffer, t->data, t->extra}; + t->data = reinterpret_cast(static_cast(1)); + }; + + std::vector used_params = collect_used_param_tensors(gf); + for (ggml_tensor* param : used_params) { + last_measurement_.params_bytes += ggml_nbytes(param); + } + const int n_leafs = sd::ggml_graph_cut::leaf_count(gf); + for (int i = 0; i < n_leafs; ++i) { + ggml_tensor* leaf = sd::ggml_graph_cut::leaf_tensor(gf, i); + if (canonical_param_tensor(leaf) == nullptr) { + continue; + } + mark_external(leaf); + mark_external(leaf->view_src); + } + + ggml_gallocr_t allocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(runtime_backend)); + size_t sizes[1] = {0}; + ggml_gallocr_reserve_n_size(allocr, gf, nullptr, nullptr, sizes); + last_measurement_.compute_bytes = sizes[0]; + ggml_gallocr_free(allocr); + + for (const auto& kv : saved_bindings) { + kv.first->buffer = kv.second.buffer; + kv.first->data = kv.second.data; + kv.first->extra = kv.second.extra; + } + last_measurement_.valid = true; + last_measurement_.desc = get_desc(); + last_measurement_.runner = this; + last_measurement_.backend = runtime_backend; + if (measure_collector_ != nullptr) { + measure_collector_->push_back(last_measurement_); + } + } + void free_cache_buffer() { if (cache_buffer != nullptr) { ggml_backend_buffer_free(cache_buffer); @@ -3227,6 +3296,19 @@ struct GGMLRunner { GGML_ASSERT(gf != nullptr); rebuild_params_tensor_set(); + if (measure_mode_) { + measure_graph_memory(gf); + // return a correctly shaped zero tensor so downstream host-side code + // (condition assembly, samplers) keeps working without weight data + std::optional> result = sd::Tensor(); + if (!no_return && ggml_graph_n_nodes(gf) > 0) { + ggml_tensor* out = ggml_graph_node(gf, -1); + result = sd::zeros({out->ne[0], out->ne[1], out->ne[2], out->ne[3]}); + } + free_compute_ctx(); + return result; + } + if (!assign_graph_cut_layer_split_backends(gf)) { free_compute_ctx(); return std::nullopt; @@ -3398,6 +3480,21 @@ struct GGMLRunner { max_graph_vram_bytes = max_vram_bytes; } + // in measure mode compute() builds the graph, records memory requirements and + // returns a shaped zero tensor without loading weights or allocating buffers + static void set_measure_mode(bool enabled, std::vector* collector = nullptr) { + measure_mode_ = enabled; + measure_collector_ = enabled ? collector : nullptr; + } + + static bool measure_mode_enabled() { + return measure_mode_; + } + + graph_memory_measurement get_last_measurement() const { + return last_measurement_; + } + void set_stream_layers_enabled(bool enabled) { if (enabled && is_multi_device()) { LOG_WARN("%s: --stream-layers is not supported with multiple runtime backends; ignoring", diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index e89ecf492..38c84d5f5 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -21,6 +21,7 @@ #include "conditioning/conditioner.hpp" #include "core/backend_fit.h" +#include "core/fit_params.h" #include "extensions/generation_extension.h" #include "model/adapter/ip_adapter.hpp" #include "model/adapter/lora.hpp" @@ -267,6 +268,8 @@ class StableDiffusionGGML { std::string split_mode_spec; bool auto_fit_enabled = false; bool vae_auto_cpu_fallback_enabled = false; + bool fit_dry_run = false; // metadata-only init for memory measurement, never reads weight data + std::map fit_module_params_bytes; bool diffusion_conv_direct = false; @@ -348,6 +351,11 @@ class StableDiffusionGGML { if constexpr (std::is_base_of_v) { model->get_param_tensor_ops(tensor_ops); } + for (const auto& kv : group_tensors) { + if (kv.second != nullptr) { + fit_module_params_bytes[module] += ggml_nbytes(kv.second); + } + } if (model_manager == nullptr) { return true; } @@ -901,6 +909,10 @@ class StableDiffusionGGML { } } } + if (fit_dry_run) { + enable_mmap = false; + eager_load = false; + } max_vram_assignment.reset(0.f); { std::string error; @@ -1031,7 +1043,9 @@ class StableDiffusionGGML { LOG_WARN("in mode 'immediately', LoRAs will cause extra memory usage with mmap"); } model_loader.process_model_files(enable_mmap, needs_writable_mmap); - load_alphas_cumprod(model_loader); + if (!fit_dry_run) { + load_alphas_cumprod(model_loader); + } diffusion_conv_direct = sd_ctx_params->diffusion_conv_direct; @@ -1798,7 +1812,8 @@ class StableDiffusionGGML { if (pred_type == PREDICTION_COUNT) { if (sd_version_is_sd2(version)) { - pred_type = is_using_v_parameterization_for_sd2(sd_version_is_inpaint(version)) ? V_PRED : EPS_PRED; + // the v-pred probe runs a real compute with loaded weights; irrelevant for memory measurement + pred_type = !fit_dry_run && is_using_v_parameterization_for_sd2(sd_version_is_inpaint(version)) ? V_PRED : EPS_PRED; } else if (sd_version_is_sdxl(version)) { if (tensor_storage_map.find("edm_vpred.sigma_max") != tensor_storage_map.end()) { // CosXL models @@ -3970,6 +3985,207 @@ SD_API void sd_cancel_generation(sd_ctx_t* sd_ctx, enum sd_cancel_mode_t mode) { } } +void sd_fit_workload_init(sd_fit_workload_t* workload) { + if (workload == nullptr) { + return; + } + *workload = {}; + workload->width = 512; + workload->height = 512; + workload->video_frames = 1; + workload->vae_tiling_params = {false, false, 0, 0, 0.5f, 0, 0, nullptr}; +} + +enum sd_fit_status_t sd_fit_params(const sd_ctx_params_t* sd_ctx_params, + const sd_fit_workload_t* workload, + sd_fit_result_t* result) { + if (sd_ctx_params == nullptr || workload == nullptr || result == nullptr) { + return SD_FIT_ERROR; + } + *result = {}; + + int64_t t0 = ggml_time_ms(); + + sd_ctx_params_t dry_params = *sd_ctx_params; + dry_params.auto_fit = false; + dry_params.eager_load = false; + + sd_ctx_t sd_ctx_storage{}; + sd_ctx_t* sd_ctx = &sd_ctx_storage; + sd_ctx->sd = new StableDiffusionGGML(); + sd_ctx->sd->fit_dry_run = true; + if (!sd_ctx->sd->init(&dry_params)) { + LOG_ERROR("fit-params: dry-run model init failed"); + delete sd_ctx->sd; + sd_ctx->sd = nullptr; + return SD_FIT_ERROR; + } + + const char* prompt = workload->prompt != nullptr && workload->prompt[0] != '\0' + ? workload->prompt + : "a photo of an astronaut riding a horse on the moon"; + const bool video = sd_version_supports_video_generation(sd_ctx->sd->version); + + // silence step progress during measurement, it would pollute stdout + sd_progress_cb_t saved_progress_cb = sd_get_progress_callback(); + void* saved_progress_data = sd_get_progress_callback_data(); + sd_set_progress_callback([](int, int, float, void*) {}, nullptr); + + // run the real generation pipeline in measure mode: every runner builds its graphs, + // records memory requirements and returns shaped zero tensors, no weights are read + auto measure = [&](const sd_tiling_params_t& tiling, + std::vector& records) -> bool { + records.clear(); + GGMLRunner::set_measure_mode(true, &records); + bool ok = false; + sd_image_t* images = nullptr; + int num_images = 0; + if (video) { + sd_vid_gen_params_t gen; + sd_vid_gen_params_init(&gen); + gen.prompt = prompt; + gen.width = workload->width; + gen.height = workload->height; + gen.video_frames = std::max(workload->video_frames, 1); + gen.sample_params.sample_steps = 1; + gen.vae_tiling_params = tiling; + sd_audio_t* audio = nullptr; + ok = generate_video(sd_ctx, &gen, &images, &num_images, &audio); + free_sd_audio(audio); + } else { + sd_img_gen_params_t gen; + sd_img_gen_params_init(&gen); + gen.prompt = prompt; + gen.width = workload->width; + gen.height = workload->height; + gen.sample_params.sample_steps = 1; + gen.batch_count = 1; + gen.vae_tiling_params = tiling; + ok = generate_image(sd_ctx, &gen, &images, &num_images); + } + GGMLRunner::set_measure_mode(false); + if (images != nullptr) { + for (int i = 0; i < num_images; i++) { + free(images[i].data); + } + free(images); + } + return ok; + }; + + std::vector records; + if (!measure(workload->vae_tiling_params, records) || records.empty()) { + LOG_ERROR("fit-params: measurement dry run failed"); + sd_set_progress_callback(saved_progress_cb, saved_progress_data); + delete sd_ctx->sd; + sd_ctx->sd = nullptr; + return SD_FIT_ERROR; + } + + auto module_for_runner = [&](const GGMLRunner* runner) -> SDBackendModule { + StableDiffusionGGML* sd = sd_ctx->sd; + if (runner == static_cast(sd->diffusion_model.get()) || + runner == static_cast(sd->high_noise_diffusion_model.get())) { + return SDBackendModule::DIFFUSION; + } + if (runner == static_cast(sd->first_stage_model.get()) || + runner == static_cast(sd->preview_vae.get())) { + return SDBackendModule::VAE; + } + if (runner == static_cast(sd->control_net.get())) { + return SDBackendModule::CONTROL_NET; + } + if (runner == static_cast(sd->clip_vision.get())) { + return SDBackendModule::CLIP_VISION; + } + return SDBackendModule::TE; + }; + + std::map module_map; + for (const auto& kv : sd_ctx->sd->fit_module_params_bytes) { + auto& m = module_map[kv.first]; + m.module = kv.first; + m.params_bytes = kv.second; + } + for (const auto& record : records) { + SDBackendModule module = module_for_runner(record.runner); + auto& m = module_map[module]; + m.module = module; + m.compute_bytes = std::max(m.compute_bytes, record.compute_bytes); + } + for (auto module : {SDBackendModule::DIFFUSION, SDBackendModule::TE}) { + auto it = module_map.find(module); + if (it != module_map.end()) { + it->second.splittable = true; + } + } + + // price VAE tiling so the planner can fall back to it when full-resolution decode does not fit + { + auto it = module_map.find(SDBackendModule::VAE); + if (it != module_map.end() && !workload->vae_tiling_params.enabled) { + sd_tiling_params_t tiled = workload->vae_tiling_params; + tiled.enabled = true; + std::vector tiled_records; + if (measure(tiled, tiled_records)) { + for (const auto& record : tiled_records) { + if (module_for_runner(record.runner) == SDBackendModule::VAE) { + it->second.compute_bytes_tiled = std::max(it->second.compute_bytes_tiled, record.compute_bytes); + } + } + } + } + } + + sd_set_progress_callback(saved_progress_cb, saved_progress_data); + + const bool user_set_placement = strlen(SAFE_STR(sd_ctx_params->backend)) > 0 || + strlen(SAFE_STR(sd_ctx_params->params_backend)) > 0; + + std::vector modules; + for (const auto& kv : module_map) { + modules.push_back(kv.second); + } + sd::fit_params::FitPlan plan; + bool planned = sd::fit_params::plan_placement(modules, sd_ctx->sd->max_vram_assignment, &plan); + + delete sd_ctx->sd; + sd_ctx->sd = nullptr; + + if (!planned || !plan.valid) { + return SD_FIT_FAILURE; + } + + result->report = strdup(plan.report.c_str()); + result->vae_tiling = plan.vae_tiling; + if (plan.changed && user_set_placement) { + LOG_WARN("fit-params: changes needed but --backend/--params-backend already set by user, abort"); + return SD_FIT_FAILURE; + } else if (plan.changed) { + result->changed = true; + if (!plan.runtime_spec.empty()) { + result->backend = strdup(plan.runtime_spec.c_str()); + } + if (!plan.params_spec.empty()) { + result->params_backend = strdup(plan.params_spec.c_str()); + } + } + + int64_t t1 = ggml_time_ms(); + LOG_INFO("fit-params: fitting params to free memory took %.2fs", (t1 - t0) * 1.0f / 1000); + return SD_FIT_SUCCESS; +} + +void sd_fit_result_free(sd_fit_result_t* result) { + if (result == nullptr) { + return; + } + free(result->backend); + free(result->params_backend); + free(result->report); + *result = {}; +} + static sd_audio_t* waveform_to_sd_audio(const StableDiffusionGGML* sd, const sd::Tensor& waveform) { if (sd == nullptr || waveform.empty()) { From 8ff25d323a80903926b1fd8f13f59fad1a5f9b19 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Mon, 24 Aug 2026 23:39:45 +0200 Subject: [PATCH 02/19] vae: remove obsolete video spatial-tiling bypass The !encode_video/!decode_video guards worked around the old tiling path that pre-allocated output with an image-shaped layout, breaking Wan video latents (transposed concat latent on encode, black frames on decode). The current process_tiles_2d derives the output shape from the first tile and handles video tensors correctly, so the bypass only disabled a working feature. Remove the guards and the encode_video plumbing; decode_video stays (used for the temporal-tiling retry decision). Verified on Apple Silicon: Wan2.2 TI2V 5B tiled t2v decode and Wan2.1 I2V 14B tiled concat-latent encode both match untiled output with no seams; tiled Wan VAE decode drops from 18522 MiB to 3136 MiB at 832x480x33. Assisted-by: Claude Fable 5 --- src/model/vae/minimax_h3_vae.hpp | 9 ++++----- src/model/vae/vae.hpp | 13 ++++--------- src/stable-diffusion.cpp | 10 +++++----- 3 files changed, 13 insertions(+), 19 deletions(-) diff --git a/src/model/vae/minimax_h3_vae.hpp b/src/model/vae/minimax_h3_vae.hpp index 9cf04c874..a89cb3f7f 100644 --- a/src/model/vae/minimax_h3_vae.hpp +++ b/src/model/vae/minimax_h3_vae.hpp @@ -602,13 +602,12 @@ namespace MiniMaxH3VAE { sd::Tensor encode(int n_threads, const sd::Tensor& x, sd_tiling_params_t tiling_params, - bool encode_video = false, - bool circular_x = false, - bool circular_y = false) override { + bool circular_x = false, + bool circular_y = false) override { auto input = ensure_video_shape(x); auto tiling = h3_tiling(tiling_params); if (input.shape()[2] == 1) { - auto encoded = VAE::encode(n_threads, input, tiling, encode_video, circular_x, circular_y); + auto encoded = VAE::encode(n_threads, input, tiling, circular_x, circular_y); if (!encoded.empty() && encoded.shape()[2] > 1) { encoded = sd::ops::slice(encoded, 2, @@ -628,7 +627,7 @@ namespace MiniMaxH3VAE { sd::Tensor result; for (int64_t start = 0; start < input.shape()[2]; start += 17) { auto chunk = sd::ops::slice(input, 2, start, start + 17); - auto encoded = VAE::encode(n_threads, chunk, tiling, encode_video, circular_x, circular_y); + auto encoded = VAE::encode(n_threads, chunk, tiling, circular_x, circular_y); if (encoded.empty()) { return {}; } diff --git a/src/model/vae/vae.hpp b/src/model/vae/vae.hpp index e6cb82978..5a06a509c 100644 --- a/src/model/vae/vae.hpp +++ b/src/model/vae/vae.hpp @@ -118,9 +118,8 @@ struct VAE : public GGMLRunner { virtual sd::Tensor encode(int n_threads, const sd::Tensor& x, sd_tiling_params_t tiling_params, - bool encode_video = false, - bool circular_x = false, - bool circular_y = false) { + bool circular_x = false, + bool circular_y = false) { int64_t t0 = ggml_time_ms(); sd::Tensor input = x; sd::Tensor output; @@ -129,9 +128,7 @@ struct VAE : public GGMLRunner { scale_tensor_to_minus1_1(&input); } - // Video VAEs produce 4D/5D tensors that the 2D spatial tiling path cannot - // handle correctly; bypass spatial tiling for video encode. - if (tiling_params.enabled && !encode_video) { + if (tiling_params.enabled) { const int scale_factor = get_scale_factor(); int64_t W = input.shape()[0] / scale_factor; int64_t H = input.shape()[1] / scale_factor; @@ -183,9 +180,7 @@ struct VAE : public GGMLRunner { sd::Tensor output; set_tiling_params(tiling_params); - // Video VAEs produce 4D/5D tensors that the 2D spatial tiling path cannot - // handle correctly; bypass spatial tiling for video decode. - if (tiling_params.enabled && !decode_video) { + if (tiling_params.enabled) { const int scale_factor = get_scale_factor(); int64_t W = input.shape()[0] * scale_factor; int64_t H = input.shape()[1] * scale_factor; diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index 38c84d5f5..9e4222d25 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -3135,8 +3135,8 @@ class StableDiffusionGGML { return latent_frames_to_video_frames(video_frames_to_latent_frames(frames)); } - sd::Tensor encode_to_vae_latents(const sd::Tensor& x, bool encode_video = false) { - auto latents = first_stage_model->encode(n_threads, x, vae_tiling_params, encode_video, circular_x, circular_y); + sd::Tensor encode_to_vae_latents(const sd::Tensor& x) { + auto latents = first_stage_model->encode(n_threads, x, vae_tiling_params, circular_x, circular_y); if (latents.empty()) { return {}; } @@ -3144,8 +3144,8 @@ class StableDiffusionGGML { return latents; } - sd::Tensor encode_first_stage(const sd::Tensor& x, bool encode_video = false) { - auto latents = encode_to_vae_latents(x, encode_video); + sd::Tensor encode_first_stage(const sd::Tensor& x) { + auto latents = encode_to_vae_latents(x); if (latents.empty()) { return {}; } @@ -6724,7 +6724,7 @@ static std::optional prepare_video_generation_latents(sd sd::ops::slice_assign(&image, 2, request->frames - 1, request->frames, end_image.unsqueeze(2)); } - auto concat_latent = sd_ctx->sd->encode_first_stage(image, /*encode_video=*/true); // [b, c, t, h/vae_scale_factor, w/vae_scale_factor]; encode_video bypasses spatial tiling + auto concat_latent = sd_ctx->sd->encode_first_stage(image); // [b, c, t, h/vae_scale_factor, w/vae_scale_factor] if (concat_latent.empty()) { LOG_ERROR("failed to encode video conditioning frames"); return std::nullopt; From d7f550212b38ecdcfe1d68facef9991c82e4ed91 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Mon, 24 Aug 2026 23:55:27 +0200 Subject: [PATCH 03/19] model: silence ggml gguf reader during probe with GGUFReader fallback Wan GGUFs carry a 5-D patch_embedding.weight that ggml's gguf reader rejects with two error logs before the GGUFReader fallback loads the file correctly. Suppress ggml logging for the probe and re-run it with logging restored only when the fallback also fails, so real diagnostics still print. Assisted-by: Claude Fable 5 --- src/model_io/gguf_io.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/model_io/gguf_io.cpp b/src/model_io/gguf_io.cpp index cd22312d5..9c4cd28d3 100644 --- a/src/model_io/gguf_io.cpp +++ b/src/model_io/gguf_io.cpp @@ -47,10 +47,25 @@ bool read_gguf_file(const std::string& file_path, gguf_context* ctx_gguf_ = nullptr; ggml_context* ctx_meta_ = nullptr; + // ggml's reader rejects tensors with more than GGML_MAX_DIMS dimensions (e.g. + // the 5-D Wan patch_embedding.weight) that the GGUFReader fallback handles, + // so silence its log during the probe to avoid spurious error messages + ggml_log_callback saved_log_cb = nullptr; + void* saved_log_data = nullptr; + ggml_log_get(&saved_log_cb, &saved_log_data); + ggml_log_set([](ggml_log_level, const char*, void*) {}, nullptr); ctx_gguf_ = gguf_init_from_file(file_path.c_str(), {true, &ctx_meta_}); + ggml_log_set(saved_log_cb, saved_log_data); + if (!ctx_gguf_) { GGUFReader gguf_reader; if (!gguf_reader.load(file_path)) { + // re-run the ggml reader with logging restored so its diagnostics are printed + ctx_gguf_ = gguf_init_from_file(file_path.c_str(), {true, &ctx_meta_}); + if (ctx_gguf_) { + gguf_free(ctx_gguf_); + ggml_free(ctx_meta_); + } set_error(error, "failed to open '" + file_path + "' with GGUFReader"); return false; } From 54b4f1378663dd55c5fdcac10e56a83d088e9585 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Tue, 25 Aug 2026 00:15:10 +0200 Subject: [PATCH 04/19] fit-params: measure image-conditioned graphs, add planner device simulation Feed a dummy input image during the measurement dry run when the model has a clip_vision tower, so the vision encoder and the VAE concat-latent encode graphs are built and measured too (both measured 0 before). Add SD_FIT_DEBUG_DEVICES to plan against simulated devices, which lets the multi-device resident/split/cpu planner paths be exercised on any machine. Mark the dry-run params memory log as projected since nothing is allocated. Assisted-by: Claude Fable 5 --- examples/fit-params/README.md | 9 +++++ src/core/fit_params.cpp | 73 +++++++++++++++++++++++++---------- src/stable-diffusion.cpp | 14 ++++++- 3 files changed, 75 insertions(+), 21 deletions(-) diff --git a/examples/fit-params/README.md b/examples/fit-params/README.md index 1582aeb05..7b0eceb3f 100644 --- a/examples/fit-params/README.md +++ b/examples/fit-params/README.md @@ -51,3 +51,12 @@ them. See `docs/backend.md` for the placement spec syntax and the heuristic `--auto-fit` alternative built into `sd-cli`. + +## Debugging the planner + +Set `SD_FIT_DEBUG_DEVICES` to plan against simulated devices instead of the +real ones, e.g. `SD_FIT_DEBUG_DEVICES="CUDA0:24,CUDA1:16"` (`name:free_gib`). +Measurement still runs on the real machine; only device enumeration is +replaced. Useful to preview placements for other hardware and to exercise +multi-device planning paths. Debug only: the emitted specs reference the +simulated device names. diff --git a/src/core/fit_params.cpp b/src/core/fit_params.cpp index 918bfdda1..6ca869b0e 100644 --- a/src/core/fit_params.cpp +++ b/src/core/fit_params.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include "core/util.h" #include "ggml-backend.h" @@ -30,7 +31,58 @@ namespace sd::fit_params { std::vector device_idxs; }; + void apply_device_budget(Device& d, sd::ggml_graph_cut::MaxVramAssignment& budgets) { + float gib = budgets.default_gib; + { + std::string budget_key = d.name; + std::transform(budget_key.begin(), budget_key.end(), budget_key.begin(), [](unsigned char c) { return (char)std::tolower(c); }); + auto it = budgets.backend_gib.find(budget_key); + if (it != budgets.backend_gib.end()) { + gib = it->second; + } + } + if (gib > 0.f) { + d.budget_bytes = std::min((int64_t)(gib * 1024.0 * 1024.0 * 1024.0), d.free_bytes); + } else if (gib < 0.f) { + d.budget_bytes = d.free_bytes + (int64_t)(gib * 1024.0 * 1024.0 * 1024.0); + } else { + d.budget_bytes = d.free_bytes - 512 * MiB; + } + d.budget_bytes = std::max(d.budget_bytes, 0); + } + + // debug override to exercise multi-device planning on any machine, + // e.g. SD_FIT_DEBUG_DEVICES="CUDA0:24,CUDA1:16" (name:free_gib) + std::vector simulated_devices(const char* spec, sd::ggml_graph_cut::MaxVramAssignment& budgets) { + std::vector out; + std::string s = spec; + size_t pos = 0; + while (pos < s.size()) { + size_t comma = s.find(',', pos); + std::string entry = s.substr(pos, comma == std::string::npos ? std::string::npos : comma - pos); + pos = comma == std::string::npos ? s.size() : comma + 1; + size_t colon = entry.find(':'); + if (colon == std::string::npos) { + continue; + } + Device d; + d.name = entry.substr(0, colon); + d.description = "simulated device"; + d.free_bytes = (int64_t)(std::stof(entry.substr(colon + 1)) * 1024.0 * 1024.0 * 1024.0); + d.total_bytes = d.free_bytes; + apply_device_budget(d, budgets); + out.push_back(d); + } + return out; + } + std::vector enumerate_gpu_devices(sd::ggml_graph_cut::MaxVramAssignment& budgets) { + const char* debug_devices = getenv("SD_FIT_DEBUG_DEVICES"); + if (debug_devices != nullptr && debug_devices[0] != '\0') { + LOG_WARN("fit-params: planning against simulated devices (SD_FIT_DEBUG_DEVICES)"); + return simulated_devices(debug_devices, budgets); + } + std::vector out; for (size_t i = 0; i < ggml_backend_dev_count(); i++) { ggml_backend_dev_t dev = ggml_backend_dev_get(i); @@ -45,26 +97,7 @@ namespace sd::fit_params { ggml_backend_dev_memory(dev, &free_bytes, &total_bytes); d.free_bytes = (int64_t)free_bytes; d.total_bytes = (int64_t)total_bytes; - - ggml_backend_t backend = nullptr; // bytes_for_backend needs a backend, resolve via name instead - (void)backend; - float gib = budgets.default_gib; - { - std::string budget_key = d.name; - std::transform(budget_key.begin(), budget_key.end(), budget_key.begin(), [](unsigned char c) { return (char)std::tolower(c); }); - auto it = budgets.backend_gib.find(budget_key); - if (it != budgets.backend_gib.end()) { - gib = it->second; - } - } - if (gib > 0.f) { - d.budget_bytes = std::min((int64_t)(gib * 1024.0 * 1024.0 * 1024.0), d.free_bytes); - } else if (gib < 0.f) { - d.budget_bytes = d.free_bytes + (int64_t)(gib * 1024.0 * 1024.0 * 1024.0); - } else { - d.budget_bytes = d.free_bytes - 512 * MiB; - } - d.budget_bytes = std::max(d.budget_bytes, 0); + apply_device_budget(d, budgets); out.push_back(d); } return out; diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index 9e4222d25..8ee920050 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -1789,8 +1789,9 @@ class StableDiffusionGGML { size_t total_params_size = total_params_ram_size + total_params_vram_size; LOG_INFO( - "total params memory size = %.2fMB (VRAM %.2fMB, RAM %.2fMB): " + "%stotal params memory size = %.2fMB (VRAM %.2fMB, RAM %.2fMB): " "text_encoders %.2fMB(%s), diffusion_model %.2fMB(%s), vae %.2fMB(%s), controlnet %.2fMB(%s), extensions %.2fMB(%s)", + fit_dry_run ? "projected " : "", // in a fit dry run nothing is allocated total_params_size / 1024.0 / 1024.0, total_params_vram_size / 1024.0 / 1024.0, total_params_ram_size / 1024.0 / 1024.0, @@ -4033,6 +4034,15 @@ enum sd_fit_status_t sd_fit_params(const sd_ctx_params_t* sd_ctx_params, // run the real generation pipeline in measure mode: every runner builds its graphs, // records memory requirements and returns shaped zero tensors, no weights are read + // models with a vision tower condition on an input image; feed a dummy one so + // the clip_vision and VAE encode graphs are built and measured too + std::vector dummy_image_data; + sd_image_t dummy_init_image = {0, 0, 3, nullptr}; + if (sd_ctx->sd->clip_vision != nullptr) { + dummy_image_data.assign((size_t)workload->width * workload->height * 3, 128); + dummy_init_image = {(uint32_t)workload->width, (uint32_t)workload->height, 3, dummy_image_data.data()}; + } + auto measure = [&](const sd_tiling_params_t& tiling, std::vector& records) -> bool { records.clear(); @@ -4049,6 +4059,7 @@ enum sd_fit_status_t sd_fit_params(const sd_ctx_params_t* sd_ctx_params, gen.video_frames = std::max(workload->video_frames, 1); gen.sample_params.sample_steps = 1; gen.vae_tiling_params = tiling; + gen.init_image = dummy_init_image; sd_audio_t* audio = nullptr; ok = generate_video(sd_ctx, &gen, &images, &num_images, &audio); free_sd_audio(audio); @@ -4061,6 +4072,7 @@ enum sd_fit_status_t sd_fit_params(const sd_ctx_params_t* sd_ctx_params, gen.sample_params.sample_steps = 1; gen.batch_count = 1; gen.vae_tiling_params = tiling; + gen.ip_adapter_image = dummy_init_image; // image models have clip_vision only for ip-adapter ok = generate_image(sd_ctx, &gen, &images, &num_images); } GGMLRunner::set_measure_mode(false); From af036eb03569b9d2908cde7629074a8ec3686f17 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Tue, 25 Aug 2026 01:04:08 +0200 Subject: [PATCH 05/19] fit-params: consider VAE tiling in the resident plan The resident tier only tried the full-resolution decode compute, so a VAE that fits resident with tiling still pushed the whole plan into time-share disk residency, reloading every module from disk each generation. Retry the device search with the measured tiled compute before giving up on residency; untiled placement is still preferred when it fits. Verified on 2x RTX 5090: klein at 1024px with 8 GiB budgets now plans diffusion=CUDA0,te=CUDA1,vae=CUDA1 --vae-tiling fully resident (previously all modules on disk) and the plan executes within budget. Assisted-by: Claude Fable 5 --- src/core/fit_params.cpp | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/src/core/fit_params.cpp b/src/core/fit_params.cpp index 6ca869b0e..6c07d98b5 100644 --- a/src/core/fit_params.cpp +++ b/src/core/fit_params.cpp @@ -215,34 +215,50 @@ namespace sd::fit_params { { std::vector params_sum(devices.size(), 0); std::vector max_compute(devices.size(), 0); - bool ok = true; + bool ok = true; + bool vae_tiling = false; std::vector resident(modules.size()); - for (size_t mi : order) { - const ModuleMemory& m = modules[mi]; - if (m.params_bytes == 0 && m.compute_bytes == 0) { - resident[mi].placed = true; - continue; - } + auto find_device = [&](const ModuleMemory& m, int64_t compute) -> int { int best = -1; for (size_t di = 0; di < devices.size(); di++) { int64_t need = params_sum[di] + (int64_t)m.params_bytes + - std::max(max_compute[di], (int64_t)m.compute_bytes); + std::max(max_compute[di], compute); if (need <= devices[di].budget_bytes && (best < 0 || devices[di].budget_bytes - params_sum[di] > devices[best].budget_bytes - params_sum[best])) { best = (int)di; } } + return best; + }; + for (size_t mi : order) { + const ModuleMemory& m = modules[mi]; + if (m.params_bytes == 0 && m.compute_bytes == 0) { + resident[mi].placed = true; + continue; + } + int64_t compute = (int64_t)m.compute_bytes; + int best = find_device(m, compute); + if (best < 0 && m.compute_bytes_tiled > 0 && m.compute_bytes_tiled < m.compute_bytes) { + // full-resolution decode does not fit anywhere, tiling may keep the module resident + compute = (int64_t)m.compute_bytes_tiled; + best = find_device(m, compute); + if (best >= 0) { + resident[mi].tiled = true; + vae_tiling = true; + } + } if (best < 0) { ok = false; break; } params_sum[best] += (int64_t)m.params_bytes; - max_compute[best] = std::max(max_compute[best], (int64_t)m.compute_bytes); + max_compute[best] = std::max(max_compute[best], compute); resident[mi].placed = true; resident[mi].device_idxs.push_back((size_t)best); } if (ok) { - decisions = std::move(resident); + decisions = std::move(resident); + plan->vae_tiling = vae_tiling; } else { time_share = true; } From 548ce0e6f88203c880a423f4ade13fec1aa771aa Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Wed, 26 Aug 2026 17:22:57 +0200 Subject: [PATCH 06/19] test: add fit params planner coverage --- tests/CMakeLists.txt | 5 ++ tests/test-fit-params.cpp | 123 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+) create mode 100644 tests/test-fit-params.cpp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index fb62a7840..77950a212 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -6,6 +6,11 @@ add_executable(test-vae-routing test-vae-routing.cpp) target_link_libraries(test-vae-routing PRIVATE stable-diffusion) add_test(NAME test-vae-routing COMMAND test-vae-routing) +add_executable(test-fit-params test-fit-params.cpp) +target_include_directories(test-fit-params PRIVATE "${PROJECT_SOURCE_DIR}/src") +target_link_libraries(test-fit-params PRIVATE stable-diffusion) +add_test(NAME test-fit-params COMMAND test-fit-params) + add_executable(test-ltx-vae-temporal test-ltx-vae-temporal.cpp) target_link_libraries(test-ltx-vae-temporal PRIVATE stable-diffusion) add_test(NAME test-ltx-vae-temporal COMMAND test-ltx-vae-temporal) diff --git a/tests/test-fit-params.cpp b/tests/test-fit-params.cpp new file mode 100644 index 000000000..e250eca77 --- /dev/null +++ b/tests/test-fit-params.cpp @@ -0,0 +1,123 @@ +#include +#include +#include +#include + +#include "core/fit_params.h" + +namespace { + +constexpr size_t GiB = 1024ull * 1024ull * 1024ull; + +bool expect(bool condition, const char* message) { + if (!condition) { + std::cerr << message << std::endl; + return false; + } + return true; +} + +sd::fit_params::ModuleMemory module(SDBackendModule module, + size_t params_gib, + size_t compute_gib, + bool splittable = false, + size_t tiled_compute_mib = 0) { + sd::fit_params::ModuleMemory memory; + memory.module = module; + memory.params_bytes = params_gib * GiB; + memory.compute_bytes = compute_gib * GiB; + memory.splittable = splittable; + memory.compute_bytes_tiled = tiled_compute_mib * 1024ull * 1024ull; + return memory; +} + +bool plan_with_devices(const char* devices, + float max_vram_gib, + const std::vector& modules, + sd::fit_params::FitPlan* plan) { + setenv("SD_FIT_DEBUG_DEVICES", devices, 1); + sd::ggml_graph_cut::MaxVramAssignment budgets; + budgets.reset(max_vram_gib); + return sd::fit_params::plan_placement(modules, budgets, plan); +} + +bool test_default_fits() { + sd::fit_params::FitPlan plan; + bool ok = plan_with_devices("GPU0:8", 8.f, + {module(SDBackendModule::DIFFUSION, 1, 1, true), + module(SDBackendModule::TE, 1, 1, true), + module(SDBackendModule::VAE, 1, 1)}, + &plan); + return expect(ok && plan.valid, "default fit plan should be valid") && + expect(!plan.changed, "default fit should not emit placement changes") && + expect(plan.runtime_spec.empty(), "default fit runtime spec should be empty") && + expect(plan.params_spec.empty(), "default fit params spec should be empty"); +} + +bool test_resident_spread() { + sd::fit_params::FitPlan plan; + bool ok = plan_with_devices("GPU0:8,GPU1:8", 8.f, + {module(SDBackendModule::DIFFUSION, 5, 2, true), + module(SDBackendModule::TE, 3, 1, true), + module(SDBackendModule::VAE, 2, 3)}, + &plan); + return expect(ok && plan.valid, "resident spread plan should be valid") && + expect(plan.changed, "resident spread should emit placement changes") && + expect(!plan.time_share, "resident spread should not time-share params") && + expect(plan.runtime_spec == "diffusion=GPU0,te=GPU1,vae=GPU1", + ("unexpected resident runtime spec: " + plan.runtime_spec).c_str()) && + expect(plan.params_spec.empty(), "resident spread should not place params on disk"); +} + +bool test_time_share_cpu_fallback() { + sd::fit_params::FitPlan plan; + bool ok = plan_with_devices("GPU0:6", 0.f, + {module(SDBackendModule::DIFFUSION, 5, 2, true), + module(SDBackendModule::TE, 1, 1, true), + module(SDBackendModule::VAE, 1, 2)}, + &plan); + return expect(ok && plan.valid, "time-share plan should be valid") && + expect(plan.time_share, "oversized resident plan should time-share") && + expect(plan.runtime_spec == "diffusion=cpu,te=GPU0,vae=GPU0", + ("unexpected time-share runtime spec: " + plan.runtime_spec).c_str()) && + expect(plan.params_spec == "te=disk,vae=disk", + ("unexpected time-share params spec: " + plan.params_spec).c_str()); +} + +bool test_split_and_tiling() { + sd::fit_params::FitPlan split_plan; + bool split_ok = plan_with_devices("GPU0:6,GPU1:6", 6.f, + {module(SDBackendModule::DIFFUSION, 8, 2, true)}, + &split_plan); + if (!expect(split_ok && split_plan.valid, "split plan should be valid") || + !expect(split_plan.runtime_spec == "diffusion=GPU0&GPU1", + ("unexpected split runtime spec: " + split_plan.runtime_spec).c_str()) || + !expect(split_plan.params_spec == "diffusion=disk", + ("unexpected split params spec: " + split_plan.params_spec).c_str())) { + return false; + } + + sd::fit_params::FitPlan tiling_plan; + bool tiling_ok = plan_with_devices("GPU0:4", 4.f, + {module(SDBackendModule::VAE, 1, 5, false, 512)}, + &tiling_plan); + return expect(tiling_ok && tiling_plan.valid, "tiling plan should be valid") && + expect(tiling_plan.vae_tiling, "tiling plan should request VAE tiling") && + expect(tiling_plan.runtime_spec == "vae=GPU0", + ("unexpected tiling runtime spec: " + tiling_plan.runtime_spec).c_str()) && + expect(tiling_plan.params_spec.empty(), + ("unexpected tiling params spec: " + tiling_plan.params_spec).c_str()); +} + +} // namespace + +int main() { + if (!test_default_fits() || + !test_resident_spread() || + !test_time_share_cpu_fallback() || + !test_split_and_tiling()) { + return 1; + } + unsetenv("SD_FIT_DEBUG_DEVICES"); + return 0; +} From 3942e51f02f129b4e745de08147d3f468b5877fc Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Wed, 26 Aug 2026 17:24:00 +0200 Subject: [PATCH 07/19] fit-params: recommend streaming diffusion params from cpu --- examples/fit-params/main.cpp | 12 ++++++++++++ include/stable-diffusion.h | 1 + src/core/fit_params.cpp | 27 ++++++++++++++++++++++++++- src/core/fit_params.h | 1 + src/stable-diffusion.cpp | 1 + tests/test-fit-params.cpp | 15 +++++++++++++++ 6 files changed, 56 insertions(+), 1 deletion(-) diff --git a/examples/fit-params/main.cpp b/examples/fit-params/main.cpp index eda0e13f0..f547ff406 100644 --- a/examples/fit-params/main.cpp +++ b/examples/fit-params/main.cpp @@ -114,6 +114,18 @@ int main(int argc, const char* argv[]) { } args += "--vae-tiling"; } + if (result.stream_layers && strlen(SAFE_STR(sd_ctx_params.max_vram)) > 0) { + if (!args.empty()) { + args += " "; + } + args += std::string("--max-vram \"") + sd_ctx_params.max_vram + "\""; + } + if (result.stream_layers && !sd_ctx_params.stream_layers) { + if (!args.empty()) { + args += " "; + } + args += "--stream-layers"; + } fprintf(stderr, "printing fitted CLI arguments to stdout...\n"); printf("%s\n", args.c_str()); } else { diff --git a/include/stable-diffusion.h b/include/stable-diffusion.h index 019ce2327..8644b0c2e 100644 --- a/include/stable-diffusion.h +++ b/include/stable-diffusion.h @@ -520,6 +520,7 @@ typedef struct { char* backend; // fitted runtime placement spec for --backend, owned by the result char* params_backend; // fitted params placement spec for --params-backend, owned by the result bool vae_tiling; // recommend enabling VAE tiling + bool stream_layers; // recommend enabling graph-cut streaming layers char* report; // human readable per-device / per-module memory table } sd_fit_result_t; diff --git a/src/core/fit_params.cpp b/src/core/fit_params.cpp index 6c07d98b5..43a04f225 100644 --- a/src/core/fit_params.cpp +++ b/src/core/fit_params.cpp @@ -21,13 +21,16 @@ namespace sd::fit_params { int64_t free_bytes = 0; int64_t total_bytes = 0; int64_t budget_bytes = 0; + bool graph_budget_enabled = false; }; struct Decision { bool placed = false; bool on_cpu = false; bool disk_params = false; + bool cpu_params = false; bool tiled = false; + bool stream_layers = false; std::vector device_idxs; }; @@ -43,10 +46,13 @@ namespace sd::fit_params { } if (gib > 0.f) { d.budget_bytes = std::min((int64_t)(gib * 1024.0 * 1024.0 * 1024.0), d.free_bytes); + d.graph_budget_enabled = true; } else if (gib < 0.f) { d.budget_bytes = d.free_bytes + (int64_t)(gib * 1024.0 * 1024.0 * 1024.0); + d.graph_budget_enabled = true; } else { d.budget_bytes = d.free_bytes - 512 * MiB; + d.graph_budget_enabled = false; } d.budget_bytes = std::max(d.budget_bytes, 0); } @@ -322,6 +328,22 @@ namespace sd::fit_params { continue; } } + if (m.module == SDBackendModule::DIFFUSION && m.splittable) { + for (size_t di = 0; di < devices.size(); di++) { + if (devices[di].graph_budget_enabled && devices[di].budget_bytes > 0 && + (best < 0 || devices[di].budget_bytes > devices[best].budget_bytes)) { + best = (int)di; + } + } + if (best >= 0) { + decision.placed = true; + decision.cpu_params = true; + decision.stream_layers = true; + plan->stream_layers = true; + decision.device_idxs.push_back((size_t)best); + continue; + } + } decision.placed = true; decision.on_cpu = true; } @@ -348,10 +370,11 @@ namespace sd::fit_params { target += " (split)"; } } - report_line(plan->report, " %-12s -> %s%s%s", + report_line(plan->report, " %-12s -> %s%s%s%s", module_spec_key(m.module).c_str(), target.c_str(), decision.disk_params ? ", params on disk" : "", + decision.cpu_params ? ", params on cpu, stream layers" : "", decision.tiled ? ", vae tiling" : ""); } @@ -379,6 +402,8 @@ namespace sd::fit_params { append_assignment(plan->runtime_spec, key, device_list); if (decision.disk_params) { append_assignment(plan->params_spec, key, "disk"); + } else if (decision.cpu_params) { + append_assignment(plan->params_spec, key, "cpu"); } } diff --git a/src/core/fit_params.h b/src/core/fit_params.h index b0ac83756..51a820c0a 100644 --- a/src/core/fit_params.h +++ b/src/core/fit_params.h @@ -23,6 +23,7 @@ namespace sd::fit_params { bool changed = false; // false = current/default placement already fits bool time_share = false; bool vae_tiling = false; + bool stream_layers = false; std::string runtime_spec; std::string params_spec; std::string report; // human readable per-device / per-module table diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index 8ee920050..aed12db9f 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -4170,6 +4170,7 @@ enum sd_fit_status_t sd_fit_params(const sd_ctx_params_t* sd_ctx_params, result->report = strdup(plan.report.c_str()); result->vae_tiling = plan.vae_tiling; + result->stream_layers = plan.stream_layers; if (plan.changed && user_set_placement) { LOG_WARN("fit-params: changes needed but --backend/--params-backend already set by user, abort"); return SD_FIT_FAILURE; diff --git a/tests/test-fit-params.cpp b/tests/test-fit-params.cpp index e250eca77..42eb2745b 100644 --- a/tests/test-fit-params.cpp +++ b/tests/test-fit-params.cpp @@ -84,6 +84,20 @@ bool test_time_share_cpu_fallback() { ("unexpected time-share params spec: " + plan.params_spec).c_str()); } +bool test_stream_layers_after_split_fails() { + sd::fit_params::FitPlan plan; + bool ok = plan_with_devices("GPU0:6,GPU1:6", 6.f, + {module(SDBackendModule::DIFFUSION, 20, 2, true)}, + &plan); + return expect(ok && plan.valid, "streaming plan should be valid") && + expect(plan.time_share, "streaming plan should be a time-share fallback") && + expect(plan.stream_layers, "streaming plan should request stream layers") && + expect(plan.runtime_spec == "diffusion=GPU0", + ("unexpected streaming runtime spec: " + plan.runtime_spec).c_str()) && + expect(plan.params_spec == "diffusion=cpu", + ("unexpected streaming params spec: " + plan.params_spec).c_str()); +} + bool test_split_and_tiling() { sd::fit_params::FitPlan split_plan; bool split_ok = plan_with_devices("GPU0:6,GPU1:6", 6.f, @@ -115,6 +129,7 @@ int main() { if (!test_default_fits() || !test_resident_spread() || !test_time_share_cpu_fallback() || + !test_stream_layers_after_split_fails() || !test_split_and_tiling()) { return 1; } From 9fe732db15706f95be9f1033ae16b76211d5399c Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Wed, 26 Aug 2026 17:46:54 +0200 Subject: [PATCH 08/19] docs: document fit params planner order --- examples/fit-params/README.md | 52 +++++++++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/examples/fit-params/README.md b/examples/fit-params/README.md index 7b0eceb3f..261de0b35 100644 --- a/examples/fit-params/README.md +++ b/examples/fit-params/README.md @@ -6,8 +6,8 @@ pipeline is executed once with graph building and memory measurement only, so no weight data is read and no buffers are allocated. The measured per-module memory is then packed against the free memory of every GPU device (minus a 512 MiB margin, or the `--max-vram` budgets) and the resulting placement is -printed to stdout as `--backend` / `--params-backend` / `--vae-tiling` -arguments. +printed to stdout as `--backend` / `--params-backend` / `--vae-tiling` / +`--stream-layers` arguments. Because compute memory depends on the generation parameters, pass the same width/height (and video frames) you intend to generate with. Example usage: @@ -38,12 +38,58 @@ Useful flags: - `-W` / `-H` / `--video-frames`: the workload the fit must accommodate - `--max-vram ` or `--max-vram cuda0=8,cuda1=14`: per-device budgets - (default: free memory minus 512 MiB per device) + (default: free memory minus 512 MiB per device). Positive values cap the + graph-splitting budget, negative values use auto budget detection, and `0` + disables graph splitting. - `--fit-print`: print the measured memory table to stdout instead of arguments - `-p`: representative prompt (token count affects text encoder memory) - model placement inputs such as `--type`, `--diffusion-fa`, `--vae-tiling` flow into the measurement exactly as they would into a real run +## Planner order + +The planner tries the fastest and most resident placements first, then falls +back to progressively lower-VRAM choices. Device budgets come from the current +free GPU memory minus a 512 MiB margin, unless `--max-vram` provides an explicit +budget. If no GPU device is available, the tool keeps the default backend. + +The checks run in this order: + +1. Default placement: put every module on the first GPU. This succeeds when + the sum of all module parameters plus the largest measured compute buffer + fits that device budget. If it succeeds, the tool prints an empty line + because no extra CLI arguments are needed. +2. Resident multi-device placement: sort modules by parameter size, largest + first, and place each module on one GPU while keeping all parameters resident. + For each GPU, resident parameters accumulate and only the largest compute + buffer assigned to that GPU is counted, because module compute phases do not + run at the same time. +3. Resident VAE tiling: while trying the resident plan, if a module has a + measured tiled compute size and full-resolution compute does not fit, retry + that module with tiled compute. This currently applies to VAE measurements + and emits `--vae-tiling`. +4. Time-share single-device placement: if resident placement fails, plan each + module as a separate phase. A module can run on a GPU with + `--params-backend =disk` when its parameters plus its compute buffer + fit one device budget. +5. Time-share VAE tiling: if the non-tiled time-share check fails and the module + has a tiled compute measurement, retry with the tiled compute size and emit + `--vae-tiling` if it fits. +6. Multi-GPU split: if the module is splittable and more than one GPU exists, + split its parameters across all GPUs when the sum of each device budget minus + that module's compute buffer can hold the module parameters. The emitted + backend uses `&`, for example `diffusion=CUDA0&CUDA1`, and parameters are + loaded per phase from disk. +7. Diffusion CPU params plus layer streaming: if split placement still does not + fit, and the module is a splittable diffusion module, choose the GPU with the + largest graph-splitting budget and keep diffusion parameters in CPU RAM while + streaming layers to the runtime GPU. This emits + `--params-backend diffusion=cpu`, preserves the original `--max-vram`, and + adds `--stream-layers`. This fallback is only considered when graph splitting + is enabled by a positive or negative `--max-vram`; `--max-vram 0` disables it. +8. CPU runtime fallback: if none of the GPU options above fit, put the module on + the CPU runtime backend. + If the current parameters already fit, the tool prints an empty line and reports that no changes are needed. If `--backend` / `--params-backend` are already set and changes would be needed, the tool fails instead of overriding From fa27c74909a013cea1491839d98d59d224492e0c Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Wed, 26 Aug 2026 18:14:07 +0200 Subject: [PATCH 09/19] fix: make fit params reflect generation workloads --- docs/backend.md | 7 +- examples/fit-params/README.md | 10 +- examples/fit-params/main.cpp | 174 ++++++++++++++++++++++++++++++++++ include/stable-diffusion.h | 6 ++ src/core/fit_params.cpp | 64 +++++++++++-- src/core/fit_params.h | 5 + src/core/ggml_extend.hpp | 34 ++++++- src/stable-diffusion.cpp | 169 +++++++++++++++++++++------------ src/upscaler.cpp | 1 + tests/test-fit-params.cpp | 45 ++++++++- 10 files changed, 438 insertions(+), 77 deletions(-) diff --git a/docs/backend.md b/docs/backend.md index ee9de9132..72a2c68d2 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -147,9 +147,10 @@ still runs out of memory, tiling is enabled and the decode retried once. `sd-fit-params` is a standalone tool that derives the same kind of placement, but from *measured* memory instead of auto-fit's fixed compute reserves. It runs the real generation pipeline in a metadata-only dry run: every module's -compute graphs are built for the requested width/height/frames and their -compute buffer sizes are measured without allocating anything or reading any -weight data, so a fit takes seconds even for very large models. Because +compute graphs are built for the requested generation request and their +compute buffer sizes are measured without allocating ggml weight/compute +buffers or reading weight data. Shaped host tensors are materialized between +graph builds, and allocation failures are returned as fit errors. Because compute memory depends on the generation parameters, they are inputs to the tool, and the printed arguments are valid for workloads up to that size. diff --git a/examples/fit-params/README.md b/examples/fit-params/README.md index 261de0b35..3755bcab4 100644 --- a/examples/fit-params/README.md +++ b/examples/fit-params/README.md @@ -3,8 +3,10 @@ `sd-fit-params` computes the CLI arguments that make a model fit into free device memory, using measured metadata-only dry runs: the real generation pipeline is executed once with graph building and memory measurement only, so -no weight data is read and no buffers are allocated. The measured per-module -memory is then packed against the free memory of every GPU device (minus a +no weight data is read and no ggml weight or compute buffers are allocated. +Shaped host tensors are still materialized to carry state between graph builds; +allocation failures are reported as fit errors. The measured per-module memory +is then packed against the free memory of every GPU device (minus a 512 MiB margin, or the `--max-vram` budgets) and the resulting placement is printed to stdout as `--backend` / `--params-backend` / `--vae-tiling` / `--stream-layers` arguments. @@ -43,8 +45,8 @@ Useful flags: disables graph splitting. - `--fit-print`: print the measured memory table to stdout instead of arguments - `-p`: representative prompt (token count affects text encoder memory) -- model placement inputs such as `--type`, `--diffusion-fa`, `--vae-tiling` - flow into the measurement exactly as they would into a real run +- generation inputs including init/control/reference images, LoRAs, hires, and + model placement options flow into the measurement as they would into a real run ## Planner order diff --git a/examples/fit-params/main.cpp b/examples/fit-params/main.cpp index f547ff406..716821974 100644 --- a/examples/fit-params/main.cpp +++ b/examples/fit-params/main.cpp @@ -1,10 +1,17 @@ #include + +#include +#include +#include #include #include #include "stable-diffusion.h" #include "common/common.h" +#include "common/media_io.h" + +namespace fs = std::filesystem; struct SDFitCliParams { bool verbose = false; @@ -49,6 +56,160 @@ static void fit_log_cb(enum sd_log_level_t level, const char* log, void* data) { fflush(stderr); } +static bool load_images_from_dir(const std::string& dir, + std::vector& images, + int expected_width, + int expected_height, + int max_image_num, + bool verbose) { + if (!fs::exists(dir) || !fs::is_directory(dir)) { + fprintf(stderr, "'%s' is not a valid directory\n", dir.c_str()); + return false; + } + + std::vector entries; + for (const auto& entry : fs::directory_iterator(dir)) { + if (entry.is_regular_file()) { + entries.push_back(entry); + } + } + std::sort(entries.begin(), entries.end(), [](const auto& a, const auto& b) { + return a.path().filename().string() < b.path().filename().string(); + }); + + for (const auto& entry : entries) { + std::string path = entry.path().string(); + std::string ext = entry.path().extension().string(); + std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c) { + return static_cast(std::tolower(c)); + }); + if (ext != ".jpg" && ext != ".jpeg" && ext != ".png" && ext != ".bmp" && ext != ".webp") { + continue; + } + if (verbose) { + fprintf(stderr, "load image %zu from '%s'\n", images.size(), path.c_str()); + } + int width = 0; + int height = 0; + uint8_t* image_buffer = load_image_from_file(path.c_str(), width, height, expected_width, expected_height); + if (image_buffer == nullptr) { + fprintf(stderr, "load image from '%s' failed\n", path.c_str()); + return false; + } + images.emplace_back(sd_image_t{static_cast(width), + static_cast(height), + 3, + image_buffer}); + if (max_image_num > 0 && static_cast(images.size()) >= max_image_num) { + break; + } + } + return true; +} + +static bool load_generation_inputs(SDGenerationParams& params, SDMode mode, bool verbose) { + auto load_image = [&](const std::string& path, + SDImageOwner& image, + bool resize_image = true, + int channels = 3) { + int width = resize_image && params.width_and_height_are_set() ? params.width : 0; + int height = resize_image && params.width_and_height_are_set() ? params.height : 0; + if (!load_sd_image_from_file(image.put(), path.c_str(), width, height, channels)) { + fprintf(stderr, "failed to load image from '%s'\n", path.c_str()); + return false; + } + params.set_width_and_height_if_unset(image.get().width, image.get().height); + return true; + }; + auto load_audio = [&](const std::string& path, SDAudioOwner& audio) { + std::vector samples; + uint32_t sample_rate = 0; + uint32_t channels = 0; + if (!load_wav_from_file(path, samples, sample_rate, channels)) { + fprintf(stderr, "failed to load WAV audio from '%s'\n", path.c_str()); + return false; + } + audio.reset(std::move(samples), sample_rate, channels); + return true; + }; + + if ((!params.init_image_path.empty() && !load_image(params.init_image_path, params.init_image)) || + (!params.end_image_path.empty() && !load_image(params.end_image_path, params.end_image))) { + return false; + } + params.ref_images.clear(); + for (const auto& path : params.ref_image_paths) { + SDImageOwner image({0, 0, 3, nullptr}); + if (!load_image(path, image, false)) { + return false; + } + params.ref_images.push_back(std::move(image)); + } + if (!params.validate(mode)) { + return false; + } + + params.ref_videos.clear(); + for (const auto& path : params.ref_video_paths) { + std::vector frames; + if (!load_images_from_dir(path, frames, 0, 0, 0, verbose) || frames.empty()) { + fprintf(stderr, "failed to load reference video frames from '%s'\n", path.c_str()); + return false; + } + params.ref_videos.push_back(std::move(frames)); + } + params.ref_video_audios.clear(); + params.ref_video_audios.resize(params.ref_videos.size()); + for (size_t i = 0; i < params.ref_video_audio_paths.size(); ++i) { + if (!load_audio(params.ref_video_audio_paths[i], params.ref_video_audios[i])) { + return false; + } + } + params.ref_audios.clear(); + params.ref_audios.resize(params.ref_audio_paths.size()); + for (size_t i = 0; i < params.ref_audio_paths.size(); ++i) { + if (!load_audio(params.ref_audio_paths[i], params.ref_audios[i])) { + return false; + } + } + + if (!params.mask_image_path.empty() && + !load_image(params.mask_image_path, params.mask_image, true, 1)) { + return false; + } + if (!params.control_image_path.empty() && + !load_image(params.control_image_path, params.control_image)) { + return false; + } + if (!params.ip_adapter_image_path.empty() && + !load_image(params.ip_adapter_image_path, params.ip_adapter_image, false)) { + return false; + } + if (!params.control_video_path.empty()) { + params.control_frames.clear(); + if (!load_images_from_dir(params.control_video_path, + params.control_frames, + params.get_resolved_width(), + params.get_resolved_height(), + params.video_frames, + verbose)) { + return false; + } + } + if (!params.pm_id_images_dir.empty()) { + params.pm_id_images.clear(); + if (!load_images_from_dir(params.pm_id_images_dir, + params.pm_id_images, + 0, + 0, + 0, + verbose)) { + return false; + } + } + return true; +} + int main(int argc, const char* argv[]) { if (argc > 1 && std::string(argv[1]) == "--version") { printf("%s\n", version_string().c_str()); @@ -73,6 +234,9 @@ int main(int argc, const char* argv[]) { print_usage(argc, argv, options_vec); return 1; } + if (!load_generation_inputs(gen_params, mode, fit_params.verbose)) { + return 1; + } sd_ctx_params_t sd_ctx_params = ctx_params.to_sd_ctx_params_t(false); @@ -84,6 +248,16 @@ int main(int argc, const char* argv[]) { workload.video_frames = gen_params.video_frames; workload.vae_tiling_params = gen_params.vae_tiling_params; + sd_img_gen_params_t image_request; + sd_vid_gen_params_t video_request; + if (mode == VID_GEN) { + video_request = gen_params.to_sd_vid_gen_params_t(); + workload.video_gen_params = &video_request; + } else { + image_request = gen_params.to_sd_img_gen_params_t(); + workload.image_gen_params = &image_request; + } + sd_fit_result_t result; enum sd_fit_status_t status = sd_fit_params(&sd_ctx_params, &workload, &result); if (status != SD_FIT_SUCCESS) { diff --git a/include/stable-diffusion.h b/include/stable-diffusion.h index 8644b0c2e..ab23b7fcd 100644 --- a/include/stable-diffusion.h +++ b/include/stable-diffusion.h @@ -507,6 +507,12 @@ typedef struct { int height; int video_frames; // <= 1 for image generation sd_tiling_params_t vae_tiling_params; + // Optional complete representative request. Set at most one. When present, + // it supplies conditioning, LoRAs, hires, cache, and other generation inputs; + // the scalar fields above remain the fallback for callers that only need a + // basic text-to-image/video workload. + const sd_img_gen_params_t* image_gen_params; + const sd_vid_gen_params_t* video_gen_params; } sd_fit_workload_t; enum sd_fit_status_t { diff --git a/src/core/fit_params.cpp b/src/core/fit_params.cpp index 43a04f225..7b7c04970 100644 --- a/src/core/fit_params.cpp +++ b/src/core/fit_params.cpp @@ -272,6 +272,53 @@ namespace sd::fit_params { // time-share plan: phases run sequentially, heavy modules load per phase and free after if (time_share) { + auto split_graphs_fit = [&](const ModuleMemory& m, + const std::vector& device_idxs, + int64_t compute) { + if (m.split_graph_segment_params.empty()) { + return false; + } + std::vector capacities; + capacities.reserve(device_idxs.size()); + for (size_t device_idx : device_idxs) { + capacities.push_back(std::max(devices[device_idx].budget_bytes - compute, 0)); + } + for (const auto& graph_segments : m.split_graph_segment_params) { + size_t device_pos = 0; + int64_t used = 0; + for (size_t segment_bytes : graph_segments) { + while (device_pos + 1 < capacities.size() && + used + (int64_t)segment_bytes > capacities[device_pos]) { + ++device_pos; + used = 0; + } + if (used + (int64_t)segment_bytes > capacities[device_pos]) { + return false; + } + used += (int64_t)segment_bytes; + } + } + return true; + }; + auto streamed_graphs_fit = [&](const ModuleMemory& m, int64_t budget) { + if (m.split_graph_segment_params.empty() || + m.split_graph_segment_params.size() != m.split_graph_segment_compute.size()) { + return false; + } + for (size_t graph_idx = 0; graph_idx < m.split_graph_segment_params.size(); ++graph_idx) { + const auto& params = m.split_graph_segment_params[graph_idx]; + const auto& compute = m.split_graph_segment_compute[graph_idx]; + if (params.size() != compute.size()) { + return false; + } + for (size_t segment_idx = 0; segment_idx < params.size(); ++segment_idx) { + if ((int64_t)params[segment_idx] + (int64_t)compute[segment_idx] > budget) { + return false; + } + } + } + return true; + }; for (size_t mi : order) { const ModuleMemory& m = modules[mi]; Decision& decision = decisions[mi]; @@ -311,19 +358,19 @@ namespace sd::fit_params { } if (m.splittable && devices.size() > 1) { int64_t capacity = 0; + std::vector idxs(devices.size()); + for (size_t i = 0; i < idxs.size(); i++) { + idxs[i] = i; + } + std::sort(idxs.begin(), idxs.end(), [&](size_t a, size_t b) { + return devices[a].budget_bytes > devices[b].budget_bytes; + }); for (const Device& d : devices) { capacity += std::max(d.budget_bytes - (int64_t)m.compute_bytes, 0); } - if ((int64_t)m.params_bytes <= capacity) { + if ((int64_t)m.params_bytes <= capacity && split_graphs_fit(m, idxs, (int64_t)m.compute_bytes)) { decision.placed = true; decision.disk_params = true; - std::vector idxs(devices.size()); - for (size_t i = 0; i < idxs.size(); i++) { - idxs[i] = i; - } - std::sort(idxs.begin(), idxs.end(), [&](size_t a, size_t b) { - return devices[a].budget_bytes > devices[b].budget_bytes; - }); decision.device_idxs = std::move(idxs); continue; } @@ -331,6 +378,7 @@ namespace sd::fit_params { if (m.module == SDBackendModule::DIFFUSION && m.splittable) { for (size_t di = 0; di < devices.size(); di++) { if (devices[di].graph_budget_enabled && devices[di].budget_bytes > 0 && + streamed_graphs_fit(m, devices[di].budget_bytes) && (best < 0 || devices[di].budget_bytes > devices[best].budget_bytes)) { best = (int)di; } diff --git a/src/core/fit_params.h b/src/core/fit_params.h index 51a820c0a..bf56ba25a 100644 --- a/src/core/fit_params.h +++ b/src/core/fit_params.h @@ -16,6 +16,11 @@ namespace sd::fit_params { size_t compute_bytes = 0; // largest measured compute buffer among the module's graphs size_t compute_bytes_tiled = 0; // VAE only: compute buffer with tiling enabled, 0 if not measured bool splittable = false; + // Ordered, de-duplicated parameter bytes per graph-cut segment for each + // measured graph. Multi-device placement must fit these indivisible + // segments, not just the aggregate module size. + std::vector> split_graph_segment_params; + std::vector> split_graph_segment_compute; }; struct FitPlan { diff --git a/src/core/ggml_extend.hpp b/src/core/ggml_extend.hpp index 9aef30575..e94d02372 100644 --- a/src/core/ggml_extend.hpp +++ b/src/core/ggml_extend.hpp @@ -1751,8 +1751,11 @@ struct GGMLRunner { std::string desc; const GGMLRunner* runner = nullptr; ggml_backend_t backend = nullptr; + SDBackendModule module = SDBackendModule::TE; size_t compute_bytes = 0; size_t params_bytes = 0; + std::vector split_segment_params_bytes; + std::vector split_segment_compute_bytes; bool valid = false; }; @@ -1794,9 +1797,10 @@ struct GGMLRunner { // static so nested runners (e.g. text encoders inside a conditioner) are also // intercepted; measurement is single-threaded like the rest of param fitting - static inline bool measure_mode_ = false; - static inline std::vector* measure_collector_ = nullptr; + static inline bool measure_mode_ = false; + static inline std::vector* measure_collector_ = nullptr; graph_memory_measurement last_measurement_; + SDBackendModule fit_module_ = SDBackendModule::TE; std::vector one_vec = {1.f}; ggml_tensor* one_tensor = nullptr; @@ -2334,6 +2338,28 @@ struct GGMLRunner { last_measurement_.desc = get_desc(); last_measurement_.runner = this; last_measurement_.backend = runtime_backend; + last_measurement_.module = fit_module_; + + const auto split_plan = sd::ggml_graph_cut::build_plan(runtime_backend, + gf, + params_tensor_set_, + get_desc().c_str()); + if (split_plan.valid && split_plan.has_cuts && split_plan.segments.size() > 1) { + std::unordered_set seen_split_params; + last_measurement_.split_segment_params_bytes.reserve(split_plan.segments.size()); + last_measurement_.split_segment_compute_bytes.reserve(split_plan.segments.size()); + for (const auto& segment : split_plan.segments) { + size_t segment_bytes = 0; + for (ggml_tensor* raw_param : sd::ggml_graph_cut::param_tensors(gf, segment)) { + ggml_tensor* param = canonical_param_tensor(raw_param); + if (param != nullptr && seen_split_params.insert(param).second) { + segment_bytes += ggml_nbytes(param); + } + } + last_measurement_.split_segment_params_bytes.push_back(segment_bytes); + last_measurement_.split_segment_compute_bytes.push_back(segment.compute_buffer_size); + } + } if (measure_collector_ != nullptr) { measure_collector_->push_back(last_measurement_); } @@ -3495,6 +3521,10 @@ struct GGMLRunner { return last_measurement_; } + void set_fit_module(SDBackendModule module) { + fit_module_ = module; + } + void set_stream_layers_enabled(bool enabled) { if (enabled && is_multi_device()) { LOG_WARN("%s: --stream-layers is not supported with multiple runtime backends; ignoring", diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index aed12db9f..1da73de90 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -345,6 +345,9 @@ class StableDiffusionGGML { if (model == nullptr) { return true; } + if constexpr (std::is_base_of_v) { + model->set_fit_module(module); + } std::map group_tensors; std::map tensor_ops; model->get_param_tensors(group_tensors); @@ -4000,10 +4003,17 @@ void sd_fit_workload_init(sd_fit_workload_t* workload) { enum sd_fit_status_t sd_fit_params(const sd_ctx_params_t* sd_ctx_params, const sd_fit_workload_t* workload, sd_fit_result_t* result) { - if (sd_ctx_params == nullptr || workload == nullptr || result == nullptr) { + if (result == nullptr) { return SD_FIT_ERROR; } *result = {}; + if (sd_ctx_params == nullptr || workload == nullptr) { + return SD_FIT_ERROR; + } + if (workload->image_gen_params != nullptr && workload->video_gen_params != nullptr) { + LOG_ERROR("fit-params: set at most one complete generation request"); + return SD_FIT_ERROR; + } int64_t t0 = ggml_time_ms(); @@ -4022,10 +4032,30 @@ enum sd_fit_status_t sd_fit_params(const sd_ctx_params_t* sd_ctx_params, return SD_FIT_ERROR; } - const char* prompt = workload->prompt != nullptr && workload->prompt[0] != '\0' - ? workload->prompt - : "a photo of an astronaut riding a horse on the moon"; - const bool video = sd_version_supports_video_generation(sd_ctx->sd->version); + const int workload_width = workload->image_gen_params != nullptr ? workload->image_gen_params->width + : workload->video_gen_params != nullptr ? workload->video_gen_params->width + : workload->width; + const int workload_height = workload->image_gen_params != nullptr ? workload->image_gen_params->height + : workload->video_gen_params != nullptr ? workload->video_gen_params->height + : workload->height; + const int workload_frames = workload->video_gen_params != nullptr ? workload->video_gen_params->video_frames + : workload->video_frames; + const char* prompt = workload->prompt != nullptr && workload->prompt[0] != '\0' + ? workload->prompt + : "a photo of an astronaut riding a horse on the moon"; + const bool animatediff_video = sd_ctx->sd->animatediff_loaded && + sd_version_supports_animatediff(sd_ctx->sd->version) && + workload_frames > 1; + const bool video = workload->video_gen_params != nullptr || + (workload->image_gen_params == nullptr && + (animatediff_video || sd_version_supports_video_generation(sd_ctx->sd->version))); + + sd_tiling_params_t requested_tiling = workload->vae_tiling_params; + if (workload->image_gen_params != nullptr) { + requested_tiling = workload->image_gen_params->vae_tiling_params; + } else if (workload->video_gen_params != nullptr) { + requested_tiling = workload->video_gen_params->vae_tiling_params; + } // silence step progress during measurement, it would pollute stdout sd_progress_cb_t saved_progress_cb = sd_get_progress_callback(); @@ -4038,44 +4068,79 @@ enum sd_fit_status_t sd_fit_params(const sd_ctx_params_t* sd_ctx_params, // the clip_vision and VAE encode graphs are built and measured too std::vector dummy_image_data; sd_image_t dummy_init_image = {0, 0, 3, nullptr}; - if (sd_ctx->sd->clip_vision != nullptr) { - dummy_image_data.assign((size_t)workload->width * workload->height * 3, 128); - dummy_init_image = {(uint32_t)workload->width, (uint32_t)workload->height, 3, dummy_image_data.data()}; + if (sd_ctx->sd->clip_vision != nullptr && workload_width > 0 && workload_height > 0) { + dummy_image_data.assign((size_t)workload_width * workload_height * 3, 128); + dummy_init_image = {(uint32_t)workload_width, (uint32_t)workload_height, 3, dummy_image_data.data()}; } auto measure = [&](const sd_tiling_params_t& tiling, std::vector& records) -> bool { records.clear(); - GGMLRunner::set_measure_mode(true, &records); + struct MeasureModeGuard { + explicit MeasureModeGuard(std::vector* records) { + GGMLRunner::set_measure_mode(true, records); + } + ~MeasureModeGuard() { + GGMLRunner::set_measure_mode(false); + } + } measure_mode_guard(&records); bool ok = false; sd_image_t* images = nullptr; int num_images = 0; - if (video) { - sd_vid_gen_params_t gen; - sd_vid_gen_params_init(&gen); - gen.prompt = prompt; - gen.width = workload->width; - gen.height = workload->height; - gen.video_frames = std::max(workload->video_frames, 1); - gen.sample_params.sample_steps = 1; - gen.vae_tiling_params = tiling; - gen.init_image = dummy_init_image; - sd_audio_t* audio = nullptr; - ok = generate_video(sd_ctx, &gen, &images, &num_images, &audio); - free_sd_audio(audio); - } else { - sd_img_gen_params_t gen; - sd_img_gen_params_init(&gen); - gen.prompt = prompt; - gen.width = workload->width; - gen.height = workload->height; - gen.sample_params.sample_steps = 1; - gen.batch_count = 1; - gen.vae_tiling_params = tiling; - gen.ip_adapter_image = dummy_init_image; // image models have clip_vision only for ip-adapter - ok = generate_image(sd_ctx, &gen, &images, &num_images); - } - GGMLRunner::set_measure_mode(false); + sd_audio_t* audio = nullptr; + try { + if (video) { + sd_vid_gen_params_t gen; + if (workload->video_gen_params != nullptr) { + gen = *workload->video_gen_params; + } else { + sd_vid_gen_params_init(&gen); + gen.prompt = prompt; + gen.width = workload_width; + gen.height = workload_height; + gen.video_frames = std::max(workload_frames, 1); + } + gen.sample_params.sample_steps = 1; + gen.sample_params.custom_sigmas = nullptr; + gen.sample_params.custom_sigmas_count = 0; + if (gen.high_noise_sample_params.sample_steps > 0) { + gen.high_noise_sample_params.sample_steps = 1; + gen.high_noise_sample_params.custom_sigmas = nullptr; + gen.high_noise_sample_params.custom_sigmas_count = 0; + } + gen.vae_tiling_params = tiling; + if (gen.init_image.data == nullptr && dummy_init_image.data != nullptr) { + gen.init_image = dummy_init_image; + } + ok = generate_video(sd_ctx, &gen, &images, &num_images, &audio); + } else { + sd_img_gen_params_t gen; + if (workload->image_gen_params != nullptr) { + gen = *workload->image_gen_params; + } else { + sd_img_gen_params_init(&gen); + gen.prompt = prompt; + gen.width = workload_width; + gen.height = workload_height; + } + gen.sample_params.sample_steps = 1; + gen.sample_params.custom_sigmas = nullptr; + gen.sample_params.custom_sigmas_count = 0; + gen.batch_count = 1; + gen.vae_tiling_params = tiling; + if (gen.ip_adapter_image.data == nullptr && dummy_init_image.data != nullptr) { + gen.ip_adapter_image = dummy_init_image; // image models have clip_vision only for ip-adapter + } + ok = generate_image(sd_ctx, &gen, &images, &num_images); + } + } catch (const std::bad_alloc&) { + LOG_ERROR("fit-params: host memory exhausted while materializing dry-run pipeline tensors"); + ok = false; + } catch (const std::exception& error) { + LOG_ERROR("fit-params: dry-run pipeline failed: %s", error.what()); + ok = false; + } + free_sd_audio(audio); if (images != nullptr) { for (int i = 0; i < num_images; i++) { free(images[i].data); @@ -4086,7 +4151,7 @@ enum sd_fit_status_t sd_fit_params(const sd_ctx_params_t* sd_ctx_params, }; std::vector records; - if (!measure(workload->vae_tiling_params, records) || records.empty()) { + if (!measure(requested_tiling, records) || records.empty()) { LOG_ERROR("fit-params: measurement dry run failed"); sd_set_progress_callback(saved_progress_cb, saved_progress_data); delete sd_ctx->sd; @@ -4094,25 +4159,6 @@ enum sd_fit_status_t sd_fit_params(const sd_ctx_params_t* sd_ctx_params, return SD_FIT_ERROR; } - auto module_for_runner = [&](const GGMLRunner* runner) -> SDBackendModule { - StableDiffusionGGML* sd = sd_ctx->sd; - if (runner == static_cast(sd->diffusion_model.get()) || - runner == static_cast(sd->high_noise_diffusion_model.get())) { - return SDBackendModule::DIFFUSION; - } - if (runner == static_cast(sd->first_stage_model.get()) || - runner == static_cast(sd->preview_vae.get())) { - return SDBackendModule::VAE; - } - if (runner == static_cast(sd->control_net.get())) { - return SDBackendModule::CONTROL_NET; - } - if (runner == static_cast(sd->clip_vision.get())) { - return SDBackendModule::CLIP_VISION; - } - return SDBackendModule::TE; - }; - std::map module_map; for (const auto& kv : sd_ctx->sd->fit_module_params_bytes) { auto& m = module_map[kv.first]; @@ -4120,10 +4166,15 @@ enum sd_fit_status_t sd_fit_params(const sd_ctx_params_t* sd_ctx_params, m.params_bytes = kv.second; } for (const auto& record : records) { - SDBackendModule module = module_for_runner(record.runner); + SDBackendModule module = record.module; auto& m = module_map[module]; m.module = module; + m.params_bytes = std::max(m.params_bytes, record.params_bytes); m.compute_bytes = std::max(m.compute_bytes, record.compute_bytes); + if (!record.split_segment_params_bytes.empty()) { + m.split_graph_segment_params.push_back(record.split_segment_params_bytes); + m.split_graph_segment_compute.push_back(record.split_segment_compute_bytes); + } } for (auto module : {SDBackendModule::DIFFUSION, SDBackendModule::TE}) { auto it = module_map.find(module); @@ -4135,13 +4186,13 @@ enum sd_fit_status_t sd_fit_params(const sd_ctx_params_t* sd_ctx_params, // price VAE tiling so the planner can fall back to it when full-resolution decode does not fit { auto it = module_map.find(SDBackendModule::VAE); - if (it != module_map.end() && !workload->vae_tiling_params.enabled) { - sd_tiling_params_t tiled = workload->vae_tiling_params; + if (it != module_map.end() && !requested_tiling.enabled) { + sd_tiling_params_t tiled = requested_tiling; tiled.enabled = true; std::vector tiled_records; if (measure(tiled, tiled_records)) { for (const auto& record : tiled_records) { - if (module_for_runner(record.runner) == SDBackendModule::VAE) { + if (record.module == SDBackendModule::VAE) { it->second.compute_bytes_tiled = std::max(it->second.compute_bytes_tiled, record.compute_bytes); } } diff --git a/src/upscaler.cpp b/src/upscaler.cpp index 007396245..7cc69f22c 100644 --- a/src/upscaler.cpp +++ b/src/upscaler.cpp @@ -96,6 +96,7 @@ bool UpscalerGGML::load_from_file(const std::string& esrgan_path, LOG_ERROR("init esrgan model from metadata failed: '%s'", esrgan_path.c_str()); return false; } + esrgan_upscaler->set_fit_module(SDBackendModule::UPSCALER); esrgan_upscaler->set_max_graph_vram_bytes(max_graph_vram_bytes); esrgan_upscaler->set_stream_layers_enabled(stream_layers_enabled); if (direct) { diff --git a/tests/test-fit-params.cpp b/tests/test-fit-params.cpp index 42eb2745b..ea8d08f0f 100644 --- a/tests/test-fit-params.cpp +++ b/tests/test-fit-params.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -28,6 +29,12 @@ sd::fit_params::ModuleMemory module(SDBackendModule module, memory.compute_bytes = compute_gib * GiB; memory.splittable = splittable; memory.compute_bytes_tiled = tiled_compute_mib * 1024ull * 1024ull; + if (splittable && params_gib > 0) { + std::vector segment_params(params_gib, GiB); + std::vector segment_compute(params_gib, std::min(compute_gib, 1) * GiB); + memory.split_graph_segment_params.push_back(std::move(segment_params)); + memory.split_graph_segment_compute.push_back(std::move(segment_compute)); + } return memory; } @@ -123,6 +130,40 @@ bool test_split_and_tiling() { ("unexpected tiling params spec: " + tiling_plan.params_spec).c_str()); } +bool test_split_rejects_indivisible_segment() { + auto memory = module(SDBackendModule::DIFFUSION, 8, 2, true); + memory.split_graph_segment_params = {{5 * GiB, 3 * GiB}}; + memory.split_graph_segment_compute = {{1 * GiB, 1 * GiB}}; + + sd::fit_params::FitPlan plan; + bool ok = plan_with_devices("GPU0:6,GPU1:6", 6.f, {memory}, &plan); + return expect(ok && plan.valid, "indivisible split fallback should remain valid") && + expect(plan.stream_layers, "indivisible split should fall back to streaming") && + expect(plan.runtime_spec == "diffusion=GPU0", + ("unexpected indivisible fallback runtime spec: " + plan.runtime_spec).c_str()) && + expect(plan.params_spec == "diffusion=cpu", + ("unexpected indivisible fallback params spec: " + plan.params_spec).c_str()); +} + +bool test_public_result_is_initialized_on_error() { + sd_fit_workload_t workload; + sd_fit_workload_init(&workload); + if (!expect(workload.image_gen_params == nullptr && workload.video_gen_params == nullptr, + "fit workload request pointers should default to null")) { + return false; + } + + sd_fit_result_t result{}; + result.changed = true; + result.backend = reinterpret_cast(1); + result.params_backend = reinterpret_cast(1); + result.report = reinterpret_cast(1); + const auto status = sd_fit_params(nullptr, &workload, &result); + return expect(status == SD_FIT_ERROR, "invalid fit arguments should return SD_FIT_ERROR") && + expect(!result.changed && result.backend == nullptr && result.params_backend == nullptr && result.report == nullptr, + "fit result should be initialized before argument validation"); +} + } // namespace int main() { @@ -130,7 +171,9 @@ int main() { !test_resident_spread() || !test_time_share_cpu_fallback() || !test_stream_layers_after_split_fails() || - !test_split_and_tiling()) { + !test_split_and_tiling() || + !test_split_rejects_indivisible_segment() || + !test_public_result_is_initialized_on_error()) { return 1; } unsetenv("SD_FIT_DEBUG_DEVICES"); From 65a08b44ae5b0275d1789bc4d2dc3ffa21955624 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Wed, 26 Aug 2026 20:21:10 +0200 Subject: [PATCH 10/19] docs: document fit params api usage --- docs/backend.md | 8 +++- examples/fit-params/README.md | 71 +++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/docs/backend.md b/docs/backend.md index 72a2c68d2..a09cd44a5 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -169,7 +169,13 @@ changes would be needed, the tool fails instead of overriding them. The same measurement is available to library users through `sd_fit_params()` in `stable-diffusion.h`, which takes the context params plus an -`sd_fit_workload_t` and returns the derived specs and a report. +`sd_fit_workload_t` and returns the derived specs and a report. The workload can +be the simple scalar fields (`prompt`, `width`, `height`, `video_frames`) or one +complete representative request through `image_gen_params` or +`video_gen_params`. Use the full request form when conditioning, LoRAs, hires, +cache settings, image/video/audio inputs, or VAE tiling settings materially +affect the graph being measured. Result strings are owned by `sd_fit_result_t` +and must be released with `sd_fit_result_free()`. ## Modules diff --git a/examples/fit-params/README.md b/examples/fit-params/README.md index 3755bcab4..15f972a12 100644 --- a/examples/fit-params/README.md +++ b/examples/fit-params/README.md @@ -100,6 +100,77 @@ them. See `docs/backend.md` for the placement spec syntax and the heuristic `--auto-fit` alternative built into `sd-cli`. +## Library API + +Library callers can use the same measured fitting through `sd_fit_params()` in +`stable-diffusion.h`. Start from initialized context params and workload params, +then free the result when done: + +```c +sd_ctx_params_t ctx; +sd_ctx_params_init(&ctx); +ctx.diffusion_model_path = "/models/model.gguf"; +ctx.max_vram = "8"; + +sd_fit_workload_t workload; +sd_fit_workload_init(&workload); +workload.prompt = "a cat"; +workload.width = 1024; +workload.height = 1024; +workload.video_frames = 1; + +sd_fit_result_t result; +sd_fit_status_t status = sd_fit_params(&ctx, &workload, &result); +if (status == SD_FIT_SUCCESS && result.changed) { + printf("backend=%s\n", result.backend ? result.backend : ""); + printf("params_backend=%s\n", result.params_backend ? result.params_backend : ""); + printf("vae_tiling=%d\n", result.vae_tiling); + printf("stream_layers=%d\n", result.stream_layers); +} +sd_fit_result_free(&result); +``` + +For simple text-to-image or text-to-video fitting, the scalar workload fields +are enough. `sd_fit_workload_init()` defaults to a 512x512 image workload +(`video_frames = 1`) and default VAE tiling params. + +For the most accurate plan, pass a full representative request: + +```c +sd_img_gen_params_t image_request; +sd_img_gen_params_init(&image_request); +image_request.prompt = "a cat"; +image_request.width = 1024; +image_request.height = 1024; +image_request.batch_count = 1; +image_request.vae_tiling_params = workload.vae_tiling_params; + +workload.image_gen_params = &image_request; +``` + +Use `workload.video_gen_params` with `sd_vid_gen_params_t` for video. Set at +most one of `image_gen_params` and `video_gen_params`; setting both returns +`SD_FIT_ERROR`. When a full request is present, it supplies conditioning, +LoRAs, hires/cache options, image/video/audio inputs, VAE tiling settings, and +other generation fields. The scalar workload fields remain as a fallback for +callers that only need a basic request. + +`sd_fit_params()` returns: + +- `SD_FIT_SUCCESS`: a placement was found, or the current/default placement + already fits. +- `SD_FIT_FAILURE`: no placement was projected to fit, or placement was needed + but `ctx.backend` / `ctx.params_backend` was already set by the caller. +- `SD_FIT_ERROR`: invalid inputs or a hard measurement error such as an + unreadable model. + +`sd_fit_result_t` owns `backend`, `params_backend`, and `report`; always call +`sd_fit_result_free()`. If `result.changed` is false, the current/default +placement already fits and the placement strings are null. If +`result.stream_layers` is true, preserve the caller's nonzero `ctx.max_vram` +when applying the result and also enable `--stream-layers`; the max-VRAM value +is not duplicated in the result. + ## Debugging the planner Set `SD_FIT_DEBUG_DEVICES` to plan against simulated devices instead of the From 4c85946172a46411211a1708fb19b53392927028 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Thu, 27 Aug 2026 19:22:46 +0200 Subject: [PATCH 11/19] fix: make fit params memory planning reliable --- docs/backend.md | 8 +- examples/fit-params/README.md | 44 ++++--- src/core/fit_params.cpp | 204 ++++++++++++++++++++++++++++--- src/core/ggml_extend.hpp | 83 +++++++++++-- src/core/ggml_extend_backend.cpp | 2 + src/core/ggml_extend_backend.h | 1 + src/model/adapter/lora.hpp | 10 +- src/model/diffusion/control.hpp | 5 +- src/stable-diffusion.cpp | 64 ++++++++-- tests/test-fit-params.cpp | 116 +++++++++++++++++- 10 files changed, 472 insertions(+), 65 deletions(-) diff --git a/docs/backend.md b/docs/backend.md index a09cd44a5..ee6ae1217 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -162,7 +162,8 @@ cat args.txt | xargs sd-cli -m model.gguf -p "a cat" -W 1024 -H 1024 ``` `--fit-print` prints the measured per-device / per-module memory table instead -of arguments. Budgets reuse `--max-vram` with the same semantics as auto-fit. +of arguments. Budgets reuse `--max-vram`, with measured fitting retaining a +512 MiB safety margin after applying automatic or explicit per-device limits. If the current parameters already fit, nothing needs to change and the tool prints an empty line. If `--backend` / `--params-backend` are already set and changes would be needed, the tool fails instead of overriding them. @@ -175,7 +176,10 @@ complete representative request through `image_gen_params` or `video_gen_params`. Use the full request form when conditioning, LoRAs, hires, cache settings, image/video/audio inputs, or VAE tiling settings materially affect the graph being measured. Result strings are owned by `sd_fit_result_t` -and must be released with `sd_fit_result_free()`. +and must be released with `sd_fit_result_free()`. Persistent cache buffers and +the concurrent diffusion/ControlNet compute phase are included in the measured +peak. CPU fallbacks are accepted only when the projected CPU parameters and +compute buffers fit currently available host memory. ## Modules diff --git a/examples/fit-params/README.md b/examples/fit-params/README.md index 15f972a12..72f711fe0 100644 --- a/examples/fit-params/README.md +++ b/examples/fit-params/README.md @@ -2,12 +2,13 @@ `sd-fit-params` computes the CLI arguments that make a model fit into free device memory, using measured metadata-only dry runs: the real generation -pipeline is executed once with graph building and memory measurement only, so +pipeline is executed with graph building and memory measurement only, so no weight data is read and no ggml weight or compute buffers are allocated. Shaped host tensors are still materialized to carry state between graph builds; allocation failures are reported as fit errors. The measured per-module memory -is then packed against the free memory of every GPU device (minus a -512 MiB margin, or the `--max-vram` budgets) and the resulting placement is +includes projected persistent cache buffers and is packed against the free +memory of every GPU device. A 512 MiB safety margin is retained after applying +either detected or explicit `--max-vram` limits, and the resulting placement is printed to stdout as `--backend` / `--params-backend` / `--vae-tiling` / `--stream-layers` arguments. @@ -19,7 +20,7 @@ width/height (and video frames) you intend to generate with. Example usage: > ./build/bin/sd-fit-params -m sd_v1-5.gguf -W 1024 -H 1024 --max-vram 4 | tee args.txt [INFO ] fit_params.cpp:93 - fit-params: measured memory plan [INFO ] fit_params.cpp:93 - devices: -[INFO ] fit_params.cpp:93 - MTL0 Apple M4 free 12123 MiB, budget 4096 MiB +[INFO ] fit_params.cpp:93 - MTL0 Apple M4 free 12123 MiB, budget 3584 MiB [INFO ] fit_params.cpp:93 - modules (measured for this workload): [INFO ] fit_params.cpp:93 - diffusion params 1398 MiB, compute 8360 MiB [INFO ] fit_params.cpp:93 - te params 125 MiB, compute 1 MiB @@ -39,10 +40,10 @@ printing fitted CLI arguments to stdout... Useful flags: - `-W` / `-H` / `--video-frames`: the workload the fit must accommodate -- `--max-vram ` or `--max-vram cuda0=8,cuda1=14`: per-device budgets - (default: free memory minus 512 MiB per device). Positive values cap the - graph-splitting budget, negative values use auto budget detection, and `0` - disables graph splitting. +- `--max-vram ` or `--max-vram cuda0=8,cuda1=14`: per-device limits. + The planner retains 512 MiB of headroom after applying the limit. Positive + values cap the graph-splitting budget, negative values use auto budget + detection, and `0` disables graph splitting. - `--fit-print`: print the measured memory table to stdout instead of arguments - `-p`: representative prompt (token count affects text encoder memory) - generation inputs including init/control/reference images, LoRAs, hires, and @@ -51,21 +52,24 @@ Useful flags: ## Planner order The planner tries the fastest and most resident placements first, then falls -back to progressively lower-VRAM choices. Device budgets come from the current -free GPU memory minus a 512 MiB margin, unless `--max-vram` provides an explicit -budget. If no GPU device is available, the tool keeps the default backend. +back to progressively lower-VRAM choices. Device budgets retain a 512 MiB +margin after both automatic and explicit `--max-vram` limits. If no GPU device +is available, the tool verifies that the workload fits available host memory +before keeping the default CPU backend. The checks run in this order: 1. Default placement: put every module on the first GPU. This succeeds when - the sum of all module parameters plus the largest measured compute buffer - fits that device budget. If it succeeds, the tool prints an empty line + the sum of all module parameters plus the peak measured compute phase fits + that device budget. Diffusion and ControlNet buffers are added because both + remain live during denoising; other sequential module buffers use their + maximum. If it succeeds, the tool prints an empty line because no extra CLI arguments are needed. 2. Resident multi-device placement: sort modules by parameter size, largest first, and place each module on one GPU while keeping all parameters resident. - For each GPU, resident parameters accumulate and only the largest compute - buffer assigned to that GPU is counted, because module compute phases do not - run at the same time. + For each GPU, resident parameters accumulate and sequential compute buffers + use their maximum. Diffusion and ControlNet compute buffers are summed when + assigned to the same GPU. 3. Resident VAE tiling: while trying the resident plan, if a module has a measured tiled compute size and full-resolution compute does not fit, retry that module with tiled compute. This currently applies to VAE measurements @@ -90,7 +94,8 @@ The checks run in this order: adds `--stream-layers`. This fallback is only considered when graph splitting is enabled by a positive or negative `--max-vram`; `--max-vram 0` disables it. 8. CPU runtime fallback: if none of the GPU options above fit, put the module on - the CPU runtime backend. + the CPU runtime backend. The planner returns `SD_FIT_FAILURE` if the CPU + parameters and compute phases exceed currently available host memory. If the current parameters already fit, the tool prints an empty line and reports that no changes are needed. If `--backend` / `--params-backend` are @@ -153,7 +158,10 @@ most one of `image_gen_params` and `video_gen_params`; setting both returns `SD_FIT_ERROR`. When a full request is present, it supplies conditioning, LoRAs, hires/cache options, image/video/audio inputs, VAE tiling settings, and other generation fields. The scalar workload fields remain as a fallback for -callers that only need a basic request. +callers that only need a basic request. A video-only model selects the video +measurement pipeline even if an image request was supplied; shared request +fields are promoted to a video request. Supplying a video request for an +image-only model returns `SD_FIT_ERROR`. `sd_fit_params()` returns: diff --git a/src/core/fit_params.cpp b/src/core/fit_params.cpp index 7b7c04970..abfa1d0cd 100644 --- a/src/core/fit_params.cpp +++ b/src/core/fit_params.cpp @@ -5,6 +5,18 @@ #include #include #include +#include + +#if defined(_WIN32) +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#elif defined(__APPLE__) +#include +#elif defined(__linux__) +#include +#endif #include "core/util.h" #include "ggml-backend.h" @@ -13,6 +25,8 @@ namespace sd::fit_params { namespace { constexpr int64_t MiB = 1024ll * 1024; + constexpr int64_t GiB = 1024ll * MiB; + constexpr int64_t MEMORY_RESERVE = 512 * MiB; struct Device { ggml_backend_dev_t dev = nullptr; @@ -34,6 +48,73 @@ namespace sd::fit_params { std::vector device_idxs; }; + struct ComputePhases { + int64_t serial = 0; + int64_t diffusion = 0; + int64_t control_net = 0; + + void add(SDBackendModule module, int64_t bytes) { + if (module == SDBackendModule::DIFFUSION) { + diffusion = std::max(diffusion, bytes); + } else if (module == SDBackendModule::CONTROL_NET) { + control_net = std::max(control_net, bytes); + } else { + serial = std::max(serial, bytes); + } + } + + int64_t peak() const { + return std::max(serial, diffusion + control_net); + } + }; + + int64_t available_host_memory() { + const char* debug_gib = getenv("SD_FIT_DEBUG_HOST_MEMORY_GIB"); + if (debug_gib != nullptr && debug_gib[0] != '\0') { + return std::max((int64_t)(std::strtod(debug_gib, nullptr) * GiB), 0); + } +#if defined(_WIN32) + MEMORYSTATUSEX status{}; + status.dwLength = sizeof(status); + return GlobalMemoryStatusEx(&status) ? (int64_t)status.ullAvailPhys : -1; +#elif defined(__APPLE__) + vm_statistics64_data_t stats{}; + mach_msg_type_number_t count = HOST_VM_INFO64_COUNT; + vm_size_t page_size = 0; + if (host_page_size(mach_host_self(), &page_size) != KERN_SUCCESS || + host_statistics64(mach_host_self(), HOST_VM_INFO64, + reinterpret_cast(&stats), &count) != KERN_SUCCESS) { + return -1; + } + return (int64_t)(stats.free_count + stats.inactive_count + stats.speculative_count) * + (int64_t)page_size; +#elif defined(__linux__) + std::ifstream meminfo("/proc/meminfo"); + std::string key; + int64_t value = 0; + std::string unit; + while (meminfo >> key >> value >> unit) { + if (key == "MemAvailable:") { + return value * 1024; + } + } + struct sysinfo info {}; + if (sysinfo(&info) == 0) { + return (int64_t)(info.freeram + info.bufferram) * (int64_t)info.mem_unit; + } + return -1; +#else + ggml_backend_dev_t cpu = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU); + if (cpu == nullptr) { + return -1; + } + size_t free_bytes = 0; + size_t total_bytes = 0; + ggml_backend_dev_memory(cpu, &free_bytes, &total_bytes); + return free_bytes > 0 ? (int64_t)free_bytes : -1; +#endif + } + void apply_device_budget(Device& d, sd::ggml_graph_cut::MaxVramAssignment& budgets) { float gib = budgets.default_gib; { @@ -45,13 +126,13 @@ namespace sd::fit_params { } } if (gib > 0.f) { - d.budget_bytes = std::min((int64_t)(gib * 1024.0 * 1024.0 * 1024.0), d.free_bytes); + d.budget_bytes = std::min((int64_t)(gib * GiB), d.free_bytes) - MEMORY_RESERVE; d.graph_budget_enabled = true; } else if (gib < 0.f) { - d.budget_bytes = d.free_bytes + (int64_t)(gib * 1024.0 * 1024.0 * 1024.0); + d.budget_bytes = d.free_bytes + (int64_t)(gib * GiB) - MEMORY_RESERVE; d.graph_budget_enabled = true; } else { - d.budget_bytes = d.free_bytes - 512 * MiB; + d.budget_bytes = d.free_bytes - MEMORY_RESERVE; d.graph_budget_enabled = false; } d.budget_bytes = std::max(d.budget_bytes, 0); @@ -134,6 +215,45 @@ namespace sd::fit_params { report += "\n"; } + int64_t host_memory_requirement(const std::vector& modules, + const std::vector* decisions) { + int64_t params = 0; + ComputePhases compute; + for (size_t i = 0; i < modules.size(); ++i) { + const bool on_cpu = decisions == nullptr || + (*decisions)[i].on_cpu || + (*decisions)[i].cpu_params; + if (!on_cpu) { + continue; + } + params += (int64_t)modules[i].params_bytes; + if (decisions == nullptr || (*decisions)[i].on_cpu) { + compute.add(modules[i].module, (int64_t)modules[i].compute_bytes); + } + } + return params + compute.peak(); + } + + bool host_memory_fits(const std::vector& modules, + const std::vector* decisions, + std::string& report) { + const int64_t required = host_memory_requirement(modules, decisions); + if (required == 0) { + return true; + } + const int64_t available = available_host_memory(); + const int64_t budget = available < 0 ? -1 : std::max(available - MEMORY_RESERVE, 0); + if (budget < 0) { + report_line(report, " host memory availability could not be determined; refusing an unverified CPU placement"); + return false; + } + report_line(report, " host memory: available %lld MiB, budget %lld MiB, projected use %lld MiB", + (long long)(available / MiB), + (long long)(budget / MiB), + (long long)(required / MiB)); + return required <= budget; + } + } // namespace bool plan_placement(const std::vector& modules, @@ -181,23 +301,23 @@ namespace sd::fit_params { } if (devices.empty()) { - report_line(plan->report, " no usable GPU devices; keeping the default backend"); - plan->valid = true; + report_line(plan->report, " no usable GPU devices; checking the default CPU backend"); + plan->valid = host_memory_fits(modules, nullptr, plan->report); plan->changed = false; return true; } // check-first: the default placement puts every module on the default (first GPU) device { - int64_t params_sum = 0; - int64_t compute_max = 0; + int64_t params_sum = 0; + ComputePhases compute; for (const ModuleMemory& m : modules) { params_sum += (int64_t)m.params_bytes; - compute_max = std::max(compute_max, (int64_t)m.compute_bytes); + compute.add(m.module, (int64_t)m.compute_bytes); } - if (params_sum + compute_max <= devices[0].budget_bytes) { + if (params_sum + compute.peak() <= devices[0].budget_bytes) { report_line(plan->report, " projected use %lld MiB <= budget %lld MiB on %s, no changes needed", - (long long)((params_sum + compute_max) / MiB), + (long long)((params_sum + compute.peak()) / MiB), (long long)(devices[0].budget_bytes / MiB), devices[0].name.c_str()); plan->valid = true; @@ -220,15 +340,17 @@ namespace sd::fit_params { // resident plan: every module keeps its params loaded, compute buffers coexist per device { std::vector params_sum(devices.size(), 0); - std::vector max_compute(devices.size(), 0); + std::vector compute_phases(devices.size()); bool ok = true; bool vae_tiling = false; std::vector resident(modules.size()); auto find_device = [&](const ModuleMemory& m, int64_t compute) -> int { int best = -1; for (size_t di = 0; di < devices.size(); di++) { + ComputePhases candidate = compute_phases[di]; + candidate.add(m.module, compute); int64_t need = params_sum[di] + (int64_t)m.params_bytes + - std::max(max_compute[di], compute); + candidate.peak(); if (need <= devices[di].budget_bytes && (best < 0 || devices[di].budget_bytes - params_sum[di] > devices[best].budget_bytes - params_sum[best])) { best = (int)di; @@ -258,7 +380,7 @@ namespace sd::fit_params { break; } params_sum[best] += (int64_t)m.params_bytes; - max_compute[best] = std::max(max_compute[best], compute); + compute_phases[best].add(m.module, compute); resident[mi].placed = true; resident[mi].device_idxs.push_back((size_t)best); } @@ -272,6 +394,28 @@ namespace sd::fit_params { // time-share plan: phases run sequentially, heavy modules load per phase and free after if (time_share) { + std::vector compute_phases(devices.size()); + auto compute_with_concurrent_phase = [&](size_t device_idx, + SDBackendModule module, + int64_t compute) { + if (module == SDBackendModule::DIFFUSION) { + return compute + compute_phases[device_idx].control_net; + } + if (module == SDBackendModule::CONTROL_NET) { + return compute + compute_phases[device_idx].diffusion; + } + return compute; + }; + auto concurrent_compute_extra = [&](size_t device_idx, + SDBackendModule module) { + if (module == SDBackendModule::DIFFUSION) { + return compute_phases[device_idx].control_net; + } + if (module == SDBackendModule::CONTROL_NET) { + return compute_phases[device_idx].diffusion; + } + return int64_t{0}; + }; auto split_graphs_fit = [&](const ModuleMemory& m, const std::vector& device_idxs, int64_t compute) { @@ -329,7 +473,8 @@ namespace sd::fit_params { } int best = -1; for (size_t di = 0; di < devices.size(); di++) { - if ((int64_t)m.params_bytes + (int64_t)m.compute_bytes <= devices[di].budget_bytes && + const int64_t compute = compute_with_concurrent_phase(di, m.module, (int64_t)m.compute_bytes); + if ((int64_t)m.params_bytes + compute <= devices[di].budget_bytes && (best < 0 || devices[di].budget_bytes > devices[best].budget_bytes)) { best = (int)di; } @@ -338,11 +483,13 @@ namespace sd::fit_params { decision.placed = true; decision.disk_params = true; decision.device_idxs.push_back((size_t)best); + compute_phases[best].add(m.module, (int64_t)m.compute_bytes); continue; } if (m.compute_bytes_tiled > 0) { for (size_t di = 0; di < devices.size(); di++) { - if ((int64_t)m.params_bytes + (int64_t)m.compute_bytes_tiled <= devices[di].budget_bytes && + const int64_t compute = compute_with_concurrent_phase(di, m.module, (int64_t)m.compute_bytes_tiled); + if ((int64_t)m.params_bytes + compute <= devices[di].budget_bytes && (best < 0 || devices[di].budget_bytes > devices[best].budget_bytes)) { best = (int)di; } @@ -353,6 +500,7 @@ namespace sd::fit_params { decision.tiled = true; plan->vae_tiling = true; decision.device_idxs.push_back((size_t)best); + compute_phases[best].add(m.module, (int64_t)m.compute_bytes_tiled); continue; } } @@ -365,20 +513,31 @@ namespace sd::fit_params { std::sort(idxs.begin(), idxs.end(), [&](size_t a, size_t b) { return devices[a].budget_bytes > devices[b].budget_bytes; }); - for (const Device& d : devices) { - capacity += std::max(d.budget_bytes - (int64_t)m.compute_bytes, 0); + for (size_t di = 0; di < devices.size(); ++di) { + const int64_t compute = compute_with_concurrent_phase(di, m.module, (int64_t)m.compute_bytes); + capacity += std::max(devices[di].budget_bytes - compute, 0); + } + int64_t split_compute = 0; + for (size_t di : idxs) { + split_compute = std::max(split_compute, + compute_with_concurrent_phase(di, m.module, (int64_t)m.compute_bytes)); } - if ((int64_t)m.params_bytes <= capacity && split_graphs_fit(m, idxs, (int64_t)m.compute_bytes)) { + if ((int64_t)m.params_bytes <= capacity && split_graphs_fit(m, idxs, split_compute)) { decision.placed = true; decision.disk_params = true; decision.device_idxs = std::move(idxs); + for (size_t di : decision.device_idxs) { + compute_phases[di].add(m.module, (int64_t)m.compute_bytes); + } continue; } } if (m.module == SDBackendModule::DIFFUSION && m.splittable) { for (size_t di = 0; di < devices.size(); di++) { if (devices[di].graph_budget_enabled && devices[di].budget_bytes > 0 && - streamed_graphs_fit(m, devices[di].budget_bytes) && + streamed_graphs_fit(m, + devices[di].budget_bytes - + concurrent_compute_extra(di, m.module)) && (best < 0 || devices[di].budget_bytes > devices[best].budget_bytes)) { best = (int)di; } @@ -389,6 +548,7 @@ namespace sd::fit_params { decision.stream_layers = true; plan->stream_layers = true; decision.device_idxs.push_back((size_t)best); + compute_phases[best].add(m.module, (int64_t)m.compute_bytes); continue; } } @@ -397,6 +557,12 @@ namespace sd::fit_params { } } + if (!host_memory_fits(modules, &decisions, plan->report)) { + report_line(plan->report, " no placement fits available host memory"); + plan->valid = false; + return true; + } + report_line(plan->report, " placement%s:", time_share ? " (time-share: params load per phase and free after)" : ""); for (size_t mi = 0; mi < modules.size(); mi++) { const ModuleMemory& m = modules[mi]; diff --git a/src/core/ggml_extend.hpp b/src/core/ggml_extend.hpp index e94d02372..cc4cc0af9 100644 --- a/src/core/ggml_extend.hpp +++ b/src/core/ggml_extend.hpp @@ -1751,8 +1751,9 @@ struct GGMLRunner { std::string desc; const GGMLRunner* runner = nullptr; ggml_backend_t backend = nullptr; - SDBackendModule module = SDBackendModule::TE; + SDBackendModule module = SDBackendModule::UNSET; size_t compute_bytes = 0; + size_t cache_bytes = 0; size_t params_bytes = 0; std::vector split_segment_params_bytes; std::vector split_segment_compute_bytes; @@ -1799,8 +1800,17 @@ struct GGMLRunner { // intercepted; measurement is single-threaded like the rest of param fitting static inline bool measure_mode_ = false; static inline std::vector* measure_collector_ = nullptr; + static inline size_t measure_generation_ = 0; graph_memory_measurement last_measurement_; - SDBackendModule fit_module_ = SDBackendModule::TE; + SDBackendModule fit_module_ = SDBackendModule::UNSET; + size_t measure_generation_seen_ = 0; + + struct measured_cache_tensor { + ggml_type type = GGML_TYPE_F32; + std::vector shape; + size_t alloc_bytes = 0; + }; + std::map measured_cache_tensors_; std::vector one_vec = {1.f}; ggml_tensor* one_tensor = nullptr; @@ -2329,6 +2339,35 @@ struct GGMLRunner { last_measurement_.compute_bytes = sizes[0]; ggml_gallocr_free(allocr); + ggml_backend_buffer_type_t cache_buft = + ggml_backend_get_default_buffer_type(runtime_backend); + const size_t cache_alignment = ggml_backend_buft_get_alignment(cache_buft); + size_t previous_cache_bytes = 0; + for (const auto& entry : measured_cache_tensors_) { + previous_cache_bytes += entry.second.alloc_bytes; + } + const bool replaces_cache_buffer = !cache_tensor_map.empty(); + for (const auto& entry : cache_tensor_map) { + ggml_tensor* tensor = sd::ggml_graph_cut::cache_source_tensor(entry.second); + if (tensor == nullptr) { + continue; + } + measured_cache_tensor measured; + measured.type = tensor->type; + measured.shape.assign(tensor->ne, tensor->ne + ggml_n_dims(tensor)); + measured.alloc_bytes = ggml_backend_buft_get_alloc_size(cache_buft, tensor); + if (cache_alignment > 0) { + measured.alloc_bytes = GGML_PAD(measured.alloc_bytes, cache_alignment); + } + measured_cache_tensors_[entry.first] = std::move(measured); + } + for (const auto& entry : measured_cache_tensors_) { + last_measurement_.cache_bytes += entry.second.alloc_bytes; + } + const size_t live_cache_bytes = last_measurement_.cache_bytes + + (replaces_cache_buffer ? previous_cache_bytes : 0); + last_measurement_.compute_bytes += live_cache_bytes; + for (const auto& kv : saved_bindings) { kv.first->buffer = kv.second.buffer; kv.first->data = kv.second.data; @@ -2357,7 +2396,8 @@ struct GGMLRunner { } } last_measurement_.split_segment_params_bytes.push_back(segment_bytes); - last_measurement_.split_segment_compute_bytes.push_back(segment.compute_buffer_size); + last_measurement_.split_segment_compute_bytes.push_back( + segment.compute_buffer_size + live_cache_bytes); } } if (measure_collector_ != nullptr) { @@ -3206,6 +3246,9 @@ struct GGMLRunner { } void reset_compute_ctx() { + if (measure_mode_) { + cache_tensor_map.clear(); + } free_compute_ctx(); alloc_compute_ctx(); } @@ -3283,10 +3326,23 @@ struct GGMLRunner { } ggml_tensor* get_cache_tensor_by_name(const std::string& name) { - if (cache_ctx == nullptr) { + if (cache_ctx != nullptr) { + return ggml_get_tensor(cache_ctx, name.c_str()); + } + if (!measure_mode_) { return nullptr; } - return ggml_get_tensor(cache_ctx, name.c_str()); + auto it = measured_cache_tensors_.find(name); + if (it == measured_cache_tensors_.end() || compute_ctx == nullptr || it->second.shape.empty()) { + return nullptr; + } + ggml_tensor* tensor = ggml_new_tensor(compute_ctx, + it->second.type, + static_cast(it->second.shape.size()), + it->second.shape.data()); + ggml_set_name(tensor, name.c_str()); + tensor->data = reinterpret_cast(static_cast(1)); + return tensor; } template @@ -3315,6 +3371,12 @@ struct GGMLRunner { }; RunnerDoneGuard runner_done_guard(this, auto_free); + if (measure_mode_ && measure_generation_seen_ != measure_generation_) { + cache_tensor_map.clear(); + measured_cache_tensors_.clear(); + measure_generation_seen_ = measure_generation_; + } + ggml_cgraph* gf = nullptr; if (!prepare_compute_graph(get_graph, &gf)) { return std::nullopt; @@ -3328,10 +3390,12 @@ struct GGMLRunner { // (condition assembly, samplers) keeps working without weight data std::optional> result = sd::Tensor(); if (!no_return && ggml_graph_n_nodes(gf) > 0) { - ggml_tensor* out = ggml_graph_node(gf, -1); - result = sd::zeros({out->ne[0], out->ne[1], out->ne[2], out->ne[3]}); + ggml_tensor* out = ggml_get_tensor(compute_ctx, final_result_name.c_str()); + if (out == nullptr) { + return std::nullopt; + } + result = sd::zeros(sd::shape_from_ggml(out)); } - free_compute_ctx(); return result; } @@ -3509,6 +3573,9 @@ struct GGMLRunner { // in measure mode compute() builds the graph, records memory requirements and // returns a shaped zero tensor without loading weights or allocating buffers static void set_measure_mode(bool enabled, std::vector* collector = nullptr) { + if (enabled && !measure_mode_) { + ++measure_generation_; + } measure_mode_ = enabled; measure_collector_ = enabled ? collector : nullptr; } diff --git a/src/core/ggml_extend_backend.cpp b/src/core/ggml_extend_backend.cpp index a11506ffc..74d11d5ec 100644 --- a/src/core/ggml_extend_backend.cpp +++ b/src/core/ggml_extend_backend.cpp @@ -993,6 +993,8 @@ const char* sd_backend_module_name(SDBackendModule module) { return "upscaler"; case SDBackendModule::DETECTOR: return "detector"; + case SDBackendModule::UNSET: + return "unset"; } return "unknown"; } diff --git a/src/core/ggml_extend_backend.h b/src/core/ggml_extend_backend.h index b8085c7b6..f81c3b98e 100644 --- a/src/core/ggml_extend_backend.h +++ b/src/core/ggml_extend_backend.h @@ -21,6 +21,7 @@ enum class SDBackendModule { PHOTOMAKER, UPSCALER, DETECTOR, + UNSET, }; struct SDBackendAssignment { diff --git a/src/model/adapter/lora.hpp b/src/model/adapter/lora.hpp index 26edbc07c..6c0f41a0a 100644 --- a/src/model/adapter/lora.hpp +++ b/src/model/adapter/lora.hpp @@ -1179,11 +1179,13 @@ struct LoraModel : public GGMLRunner { }; GGMLRunner::compute(get_graph, n_threads, false, false, false, true); stat(!warn_unused); - for (auto item : original_tensor_to_final_tensor) { - ggml_tensor* original_tensor = item.first; - ggml_tensor* final_tensor = item.second; + if (!GGMLRunner::measure_mode_enabled()) { + for (auto item : original_tensor_to_final_tensor) { + ggml_tensor* original_tensor = item.first; + ggml_tensor* final_tensor = item.second; - ggml_backend_tensor_copy(final_tensor, original_tensor); + ggml_backend_tensor_copy(final_tensor, original_tensor); + } } original_tensor_to_final_tensor.clear(); GGMLRunner::free_compute_buffer(); diff --git a/src/model/diffusion/control.hpp b/src/model/diffusion/control.hpp index bf3c7e435..9cbeed581 100644 --- a/src/model/diffusion/control.hpp +++ b/src/model/diffusion/control.hpp @@ -432,7 +432,10 @@ struct ControlNet : public GGMLRunner { controls.clear(); controls.reserve(control_outputs_ggml.size()); for (ggml_tensor* control : control_outputs_ggml) { - auto control_host = restore_trailing_singleton_dims(sd::make_sd_tensor_from_ggml(control), 4); + auto control_host = GGMLRunner::measure_mode_enabled() + ? sd::zeros(sd::shape_from_ggml(control)) + : sd::make_sd_tensor_from_ggml(control); + control_host = restore_trailing_singleton_dims(std::move(control_host), 4); GGML_ASSERT(!control_host.empty()); controls.push_back(std::move(control_host)); } diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index 1da73de90..674103481 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -2002,6 +2002,7 @@ class StableDiffusionGGML { lora_spec.path, lora_spec.is_high_noise ? "model.high_noise_" : "", version); + lora->set_fit_module(module); LoraModel::filter_t lora_tensor_filter = module_filter; if (!lora_spec.tensor_name_prefix_filter.empty()) { lora_tensor_filter = [module_filter, prefix = lora_spec.tensor_name_prefix_filter](const std::string& tensor_name) { @@ -3951,6 +3952,35 @@ static bool sd_version_supports_image_generation(SDVersion version) { return version != VERSION_ABOT_WORLD && !sd_version_supports_video_generation(version); } +static void sd_vid_gen_params_from_image_request(sd_vid_gen_params_t* video, + const sd_img_gen_params_t& image, + int video_frames) { + sd_vid_gen_params_init(video); + video->loras = image.loras; + video->lora_count = image.lora_count; + video->prompt = image.prompt; + video->negative_prompt = image.negative_prompt; + video->clip_skip = image.clip_skip; + video->init_image = image.init_image; + video->ref_images = image.ref_images; + video->ref_images_count = image.ref_images_count; + video->width = image.width; + video->height = image.height; + video->sample_params = image.sample_params; + video->strength = image.strength; + video->seed = image.seed; + video->video_frames = std::max(video_frames, 1); + video->vae_tiling_params = image.vae_tiling_params; + video->cache = image.cache; + video->hires = image.hires; + video->circular_x = image.circular_x; + video->circular_y = image.circular_y; + if (image.control_image.data != nullptr) { + video->control_frames = const_cast(&image.control_image); + video->control_frames_size = 1; + } +} + sd_ctx_t* new_sd_ctx(const sd_ctx_params_t* sd_ctx_params) { sd_ctx_t* sd_ctx = (sd_ctx_t*)malloc(sizeof(sd_ctx_t)); if (sd_ctx == nullptr) { @@ -4043,12 +4073,18 @@ enum sd_fit_status_t sd_fit_params(const sd_ctx_params_t* sd_ctx_params, const char* prompt = workload->prompt != nullptr && workload->prompt[0] != '\0' ? workload->prompt : "a photo of an astronaut riding a horse on the moon"; - const bool animatediff_video = sd_ctx->sd->animatediff_loaded && - sd_version_supports_animatediff(sd_ctx->sd->version) && - workload_frames > 1; - const bool video = workload->video_gen_params != nullptr || - (workload->image_gen_params == nullptr && - (animatediff_video || sd_version_supports_video_generation(sd_ctx->sd->version))); + const bool model_video_only = sd_version_supports_video_generation(sd_ctx->sd->version); + const bool animatediff_capable = sd_ctx->sd->animatediff_loaded && + sd_version_supports_animatediff(sd_ctx->sd->version); + const bool animatediff_video = animatediff_capable && workload_frames > 1; + if ((workload->video_gen_params != nullptr || workload_frames > 1) && + !model_video_only && !animatediff_capable) { + LOG_ERROR("fit-params: a video workload was supplied for an image-only model"); + delete sd_ctx->sd; + sd_ctx->sd = nullptr; + return SD_FIT_ERROR; + } + const bool video = workload->video_gen_params != nullptr || model_video_only || animatediff_video; sd_tiling_params_t requested_tiling = workload->vae_tiling_params; if (workload->image_gen_params != nullptr) { @@ -4093,6 +4129,11 @@ enum sd_fit_status_t sd_fit_params(const sd_ctx_params_t* sd_ctx_params, sd_vid_gen_params_t gen; if (workload->video_gen_params != nullptr) { gen = *workload->video_gen_params; + } else if (workload->image_gen_params != nullptr) { + LOG_WARN("fit-params: promoting the image request to the video pipeline required by this model"); + sd_vid_gen_params_from_image_request(&gen, + *workload->image_gen_params, + workload_frames); } else { sd_vid_gen_params_init(&gen); gen.prompt = prompt; @@ -4100,11 +4141,11 @@ enum sd_fit_status_t sd_fit_params(const sd_ctx_params_t* sd_ctx_params, gen.height = workload_height; gen.video_frames = std::max(workload_frames, 1); } - gen.sample_params.sample_steps = 1; + gen.sample_params.sample_steps = 2; gen.sample_params.custom_sigmas = nullptr; gen.sample_params.custom_sigmas_count = 0; if (gen.high_noise_sample_params.sample_steps > 0) { - gen.high_noise_sample_params.sample_steps = 1; + gen.high_noise_sample_params.sample_steps = 2; gen.high_noise_sample_params.custom_sigmas = nullptr; gen.high_noise_sample_params.custom_sigmas_count = 0; } @@ -4123,7 +4164,7 @@ enum sd_fit_status_t sd_fit_params(const sd_ctx_params_t* sd_ctx_params, gen.width = workload_width; gen.height = workload_height; } - gen.sample_params.sample_steps = 1; + gen.sample_params.sample_steps = 2; gen.sample_params.custom_sigmas = nullptr; gen.sample_params.custom_sigmas_count = 0; gen.batch_count = 1; @@ -4167,6 +4208,11 @@ enum sd_fit_status_t sd_fit_params(const sd_ctx_params_t* sd_ctx_params, } for (const auto& record : records) { SDBackendModule module = record.module; + if (module == SDBackendModule::UNSET) { + LOG_WARN("fit-params: ignoring unattributed graph measurement from %s", + record.desc.c_str()); + continue; + } auto& m = module_map[module]; m.module = module; m.params_bytes = std::max(m.params_bytes, record.params_bytes); diff --git a/tests/test-fit-params.cpp b/tests/test-fit-params.cpp index ea8d08f0f..f01a25c21 100644 --- a/tests/test-fit-params.cpp +++ b/tests/test-fit-params.cpp @@ -5,6 +5,7 @@ #include #include "core/fit_params.h" +#include "core/ggml_extend.hpp" namespace { @@ -18,6 +19,40 @@ bool expect(bool condition, const char* message) { return true; } +struct MeasureRunner : public GGMLRunner { + bool warm_cache_seen = false; + ggml_tensor* output = nullptr; + + explicit MeasureRunner(ggml_backend_t backend) + : GGMLRunner(backend) { + set_fit_module(SDBackendModule::VAE); + } + + std::string get_desc() override { + return "fit measurement test"; + } + + std::optional> run(bool no_return = false) { + const sd::Tensor input = sd::zeros({2, 3}); + const sd::Tensor initial_cache = sd::zeros({5}); + auto get_graph = [&]() { + ggml_cgraph* graph = new_graph_custom(32); + ggml_tensor* x = make_input(input); + output = ggml_scale(compute_ctx, x, 2.f); + ggml_build_forward_expand(graph, output); + + ggml_tensor* previous_cache = get_cache_tensor_by_name("state"); + warm_cache_seen = previous_cache != nullptr; + ggml_tensor* cache_input = previous_cache != nullptr + ? previous_cache + : make_input(initial_cache); + cache("state", ggml_scale(compute_ctx, cache_input, 2.f)); + return graph; + }; + return compute(get_graph, 1, false, true, true, no_return); + } +}; + sd::fit_params::ModuleMemory module(SDBackendModule module, size_t params_gib, size_t compute_gib, @@ -41,8 +76,10 @@ sd::fit_params::ModuleMemory module(SDBackendModule module, bool plan_with_devices(const char* devices, float max_vram_gib, const std::vector& modules, - sd::fit_params::FitPlan* plan) { + sd::fit_params::FitPlan* plan, + float host_memory_gib = 64.f) { setenv("SD_FIT_DEBUG_DEVICES", devices, 1); + setenv("SD_FIT_DEBUG_HOST_MEMORY_GIB", std::to_string(host_memory_gib).c_str(), 1); sd::ggml_graph_cut::MaxVramAssignment budgets; budgets.reset(max_vram_gib); return sd::fit_params::plan_placement(modules, budgets, plan); @@ -63,7 +100,7 @@ bool test_default_fits() { bool test_resident_spread() { sd::fit_params::FitPlan plan; - bool ok = plan_with_devices("GPU0:8,GPU1:8", 8.f, + bool ok = plan_with_devices("GPU0:9,GPU1:9", 9.f, {module(SDBackendModule::DIFFUSION, 5, 2, true), module(SDBackendModule::TE, 3, 1, true), module(SDBackendModule::VAE, 2, 3)}, @@ -107,7 +144,7 @@ bool test_stream_layers_after_split_fails() { bool test_split_and_tiling() { sd::fit_params::FitPlan split_plan; - bool split_ok = plan_with_devices("GPU0:6,GPU1:6", 6.f, + bool split_ok = plan_with_devices("GPU0:7,GPU1:7", 7.f, {module(SDBackendModule::DIFFUSION, 8, 2, true)}, &split_plan); if (!expect(split_ok && split_plan.valid, "split plan should be valid") || @@ -136,7 +173,7 @@ bool test_split_rejects_indivisible_segment() { memory.split_graph_segment_compute = {{1 * GiB, 1 * GiB}}; sd::fit_params::FitPlan plan; - bool ok = plan_with_devices("GPU0:6,GPU1:6", 6.f, {memory}, &plan); + bool ok = plan_with_devices("GPU0:7,GPU1:7", 7.f, {memory}, &plan); return expect(ok && plan.valid, "indivisible split fallback should remain valid") && expect(plan.stream_layers, "indivisible split should fall back to streaming") && expect(plan.runtime_spec == "diffusion=GPU0", @@ -145,6 +182,72 @@ bool test_split_rejects_indivisible_segment() { ("unexpected indivisible fallback params spec: " + plan.params_spec).c_str()); } +bool test_explicit_budget_keeps_headroom() { + sd::fit_params::FitPlan plan; + bool ok = plan_with_devices("GPU0:8", 8.f, + {module(SDBackendModule::VAE, 7, 1)}, + &plan); + return expect(ok && plan.valid, "headroom fallback plan should be valid") && + expect(plan.changed, "explicit max-vram must retain safety headroom") && + expect(plan.runtime_spec == "vae=cpu", + ("unexpected headroom fallback runtime spec: " + plan.runtime_spec).c_str()); +} + +bool test_controlnet_compute_is_concurrent() { + sd::fit_params::FitPlan plan; + bool ok = plan_with_devices("GPU0:8", 8.f, + {module(SDBackendModule::DIFFUSION, 2, 2, true), + module(SDBackendModule::CONTROL_NET, 2, 2)}, + &plan); + return expect(ok && plan.valid, "ControlNet plan should remain valid") && + expect(plan.changed, "concurrent ControlNet and diffusion buffers must not use the default placement") && + expect(plan.time_share, "concurrent ControlNet pressure should require the time-share tier"); +} + +bool test_cpu_fallback_checks_host_memory() { + sd::fit_params::FitPlan plan; + bool ok = plan_with_devices("GPU0:4", 4.f, + {module(SDBackendModule::DIFFUSION, 8, 2, true)}, + &plan, + 8.f); + return expect(ok, "host-capacity failure should be a completed planning attempt") && + expect(!plan.valid, "CPU fallback must fail when projected use exceeds host memory") && + expect(plan.report.find("no placement fits available host memory") != std::string::npos, + "host-capacity failure should be explained in the report"); +} + +bool test_measure_mode_preserves_outputs_and_projects_cache() { + ggml_backend_t backend = sd_backend_cpu_init(); + if (!expect(backend != nullptr, "CPU backend should initialize for measurement test")) { + return false; + } + + bool passed = true; + { + MeasureRunner runner(backend); + std::vector records; + GGMLRunner::set_measure_mode(true, &records); + + auto first = runner.run(); + passed &= expect(first.has_value() && first->dim() == 2, + "measure output should preserve the named result rank"); + passed &= expect(first.has_value() && first->shape()[0] == 2 && first->shape()[1] == 3, + "measure output should preserve the named result shape"); + passed &= expect(!records.empty() && records.back().cache_bytes > 0, + "measurements should include projected persistent cache bytes"); + + auto second = runner.run(true); + passed &= expect(second.has_value() && runner.warm_cache_seen, + "the next measured graph should observe projected warm cache state"); + passed &= expect(runner.output != nullptr && ggml_n_dims(runner.output) == 2, + "no-return graph tensors should remain alive for the caller"); + + GGMLRunner::set_measure_mode(false); + } + ggml_backend_free(backend); + return passed; +} + bool test_public_result_is_initialized_on_error() { sd_fit_workload_t workload; sd_fit_workload_init(&workload); @@ -173,9 +276,14 @@ int main() { !test_stream_layers_after_split_fails() || !test_split_and_tiling() || !test_split_rejects_indivisible_segment() || + !test_explicit_budget_keeps_headroom() || + !test_controlnet_compute_is_concurrent() || + !test_cpu_fallback_checks_host_memory() || + !test_measure_mode_preserves_outputs_and_projects_cache() || !test_public_result_is_initialized_on_error()) { return 1; } unsetenv("SD_FIT_DEBUG_DEVICES"); + unsetenv("SD_FIT_DEBUG_HOST_MEMORY_GIB"); return 0; } From 78736f8a0d0a8571476fdd4be6787154840da54e Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Thu, 27 Aug 2026 23:41:06 +0200 Subject: [PATCH 12/19] refactor: organize fit input loading handlers --- examples/fit-params/main.cpp | 213 ++++++++++++++++++++--------------- 1 file changed, 124 insertions(+), 89 deletions(-) diff --git a/examples/fit-params/main.cpp b/examples/fit-params/main.cpp index 716821974..11df52533 100644 --- a/examples/fit-params/main.cpp +++ b/examples/fit-params/main.cpp @@ -107,103 +107,138 @@ static bool load_images_from_dir(const std::string& dir, return true; } -static bool load_generation_inputs(SDGenerationParams& params, SDMode mode, bool verbose) { - auto load_image = [&](const std::string& path, - SDImageOwner& image, - bool resize_image = true, - int channels = 3) { - int width = resize_image && params.width_and_height_are_set() ? params.width : 0; - int height = resize_image && params.width_and_height_are_set() ? params.height : 0; - if (!load_sd_image_from_file(image.put(), path.c_str(), width, height, channels)) { - fprintf(stderr, "failed to load image from '%s'\n", path.c_str()); - return false; - } - params.set_width_and_height_if_unset(image.get().width, image.get().height); - return true; - }; - auto load_audio = [&](const std::string& path, SDAudioOwner& audio) { - std::vector samples; - uint32_t sample_rate = 0; - uint32_t channels = 0; - if (!load_wav_from_file(path, samples, sample_rate, channels)) { - fprintf(stderr, "failed to load WAV audio from '%s'\n", path.c_str()); - return false; - } - audio.reset(std::move(samples), sample_rate, channels); +static bool load_image_file(SDGenerationParams& params, + const std::string& path, + SDImageOwner& image, + bool resize_image = true, + int channels = 3) { + if (path.empty()) { return true; - }; + } - if ((!params.init_image_path.empty() && !load_image(params.init_image_path, params.init_image)) || - (!params.end_image_path.empty() && !load_image(params.end_image_path, params.end_image))) { + const bool use_size = resize_image && params.width_and_height_are_set(); + if (!load_sd_image_from_file(image.put(), + path.c_str(), + use_size ? params.width : 0, + use_size ? params.height : 0, + channels)) { + fprintf(stderr, "failed to load image from '%s'\n", path.c_str()); return false; } - params.ref_images.clear(); - for (const auto& path : params.ref_image_paths) { - SDImageOwner image({0, 0, 3, nullptr}); - if (!load_image(path, image, false)) { - return false; - } - params.ref_images.push_back(std::move(image)); - } - if (!params.validate(mode)) { + params.set_width_and_height_if_unset(image.get().width, image.get().height); + return true; +} + +static bool load_audio_file(const std::string& path, SDAudioOwner& audio) { + std::vector samples; + uint32_t sample_rate = 0; + uint32_t channels = 0; + if (!load_wav_from_file(path, samples, sample_rate, channels)) { + fprintf(stderr, "failed to load WAV audio from '%s'\n", path.c_str()); return false; } + audio.reset(std::move(samples), sample_rate, channels); + return true; +} - params.ref_videos.clear(); - for (const auto& path : params.ref_video_paths) { - std::vector frames; - if (!load_images_from_dir(path, frames, 0, 0, 0, verbose) || frames.empty()) { - fprintf(stderr, "failed to load reference video frames from '%s'\n", path.c_str()); - return false; - } - params.ref_videos.push_back(std::move(frames)); - } - params.ref_video_audios.clear(); - params.ref_video_audios.resize(params.ref_videos.size()); - for (size_t i = 0; i < params.ref_video_audio_paths.size(); ++i) { - if (!load_audio(params.ref_video_audio_paths[i], params.ref_video_audios[i])) { - return false; - } - } - params.ref_audios.clear(); - params.ref_audios.resize(params.ref_audio_paths.size()); - for (size_t i = 0; i < params.ref_audio_paths.size(); ++i) { - if (!load_audio(params.ref_audio_paths[i], params.ref_audios[i])) { - return false; - } - } +using LoadInput = bool (*)(SDGenerationParams&, SDMode, bool); - if (!params.mask_image_path.empty() && - !load_image(params.mask_image_path, params.mask_image, true, 1)) { - return false; - } - if (!params.control_image_path.empty() && - !load_image(params.control_image_path, params.control_image)) { - return false; - } - if (!params.ip_adapter_image_path.empty() && - !load_image(params.ip_adapter_image_path, params.ip_adapter_image, false)) { - return false; - } - if (!params.control_video_path.empty()) { - params.control_frames.clear(); - if (!load_images_from_dir(params.control_video_path, - params.control_frames, - params.get_resolved_width(), - params.get_resolved_height(), - params.video_frames, - verbose)) { - return false; - } - } - if (!params.pm_id_images_dir.empty()) { - params.pm_id_images.clear(); - if (!load_images_from_dir(params.pm_id_images_dir, - params.pm_id_images, - 0, - 0, - 0, - verbose)) { +struct LoadInputHandler { + const char* name; + LoadInput load; +}; + +static const LoadInputHandler LOAD_INPUT_HANDLERS[] = { + {"init_image", [](SDGenerationParams& params, SDMode, bool) { + return load_image_file(params, params.init_image_path, params.init_image); + }}, + {"end_image", [](SDGenerationParams& params, SDMode, bool) { + return load_image_file(params, params.end_image_path, params.end_image); + }}, + {"ref_images", [](SDGenerationParams& params, SDMode, bool) { + params.ref_images.clear(); + for (const auto& path : params.ref_image_paths) { + SDImageOwner image({0, 0, 3, nullptr}); + if (!load_image_file(params, path, image, false)) { + return false; + } + params.ref_images.push_back(std::move(image)); + } + return true; + }}, + {"validate", [](SDGenerationParams& params, SDMode mode, bool) { + return params.validate(mode); + }}, + {"ref_videos", [](SDGenerationParams& params, SDMode, bool verbose) { + params.ref_videos.clear(); + for (const auto& path : params.ref_video_paths) { + std::vector frames; + if (!load_images_from_dir(path, frames, 0, 0, 0, verbose) || frames.empty()) { + fprintf(stderr, "failed to load reference video frames from '%s'\n", path.c_str()); + return false; + } + params.ref_videos.push_back(std::move(frames)); + } + return true; + }}, + {"ref_video_audios", [](SDGenerationParams& params, SDMode, bool) { + params.ref_video_audios.clear(); + params.ref_video_audios.resize(params.ref_videos.size()); + for (size_t i = 0; i < params.ref_video_audio_paths.size(); ++i) { + if (!load_audio_file(params.ref_video_audio_paths[i], params.ref_video_audios[i])) { + return false; + } + } + return true; + }}, + {"ref_audios", [](SDGenerationParams& params, SDMode, bool) { + params.ref_audios.clear(); + params.ref_audios.resize(params.ref_audio_paths.size()); + for (size_t i = 0; i < params.ref_audio_paths.size(); ++i) { + if (!load_audio_file(params.ref_audio_paths[i], params.ref_audios[i])) { + return false; + } + } + return true; + }}, + {"mask_image", [](SDGenerationParams& params, SDMode, bool) { + return load_image_file(params, params.mask_image_path, params.mask_image, true, 1); + }}, + {"control_image", [](SDGenerationParams& params, SDMode, bool) { + return load_image_file(params, params.control_image_path, params.control_image); + }}, + {"ip_adapter_image", [](SDGenerationParams& params, SDMode, bool) { + return load_image_file(params, params.ip_adapter_image_path, params.ip_adapter_image, false); + }}, + {"control_video", [](SDGenerationParams& params, SDMode, bool verbose) { + if (params.control_video_path.empty()) { + return true; + } + params.control_frames.clear(); + return load_images_from_dir(params.control_video_path, + params.control_frames, + params.get_resolved_width(), + params.get_resolved_height(), + params.video_frames, + verbose); + }}, + {"pm_id_images", [](SDGenerationParams& params, SDMode, bool verbose) { + if (params.pm_id_images_dir.empty()) { + return true; + } + params.pm_id_images.clear(); + return load_images_from_dir(params.pm_id_images_dir, + params.pm_id_images, + 0, + 0, + 0, + verbose); + }}, +}; + +static bool load_generation_inputs(SDGenerationParams& params, SDMode mode, bool verbose) { + for (const auto& handler : LOAD_INPUT_HANDLERS) { + if (!handler.load(params, mode, verbose)) { return false; } } From 96bef628bea265d0626b6551e87dc906fb23182a Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Fri, 28 Aug 2026 00:06:00 +0200 Subject: [PATCH 13/19] fix: isolate fit measurement state per thread --- src/core/ggml_extend.hpp | 10 +++++----- src/core/util.cpp | 15 +++++++++++++++ src/core/util.h | 2 ++ src/stable-diffusion.cpp | 19 ++++++++++++------- tests/CMakeLists.txt | 2 +- tests/test-fit-params.cpp | 17 +++++++++++++++++ 6 files changed, 52 insertions(+), 13 deletions(-) diff --git a/src/core/ggml_extend.hpp b/src/core/ggml_extend.hpp index cc4cc0af9..67f182ad5 100644 --- a/src/core/ggml_extend.hpp +++ b/src/core/ggml_extend.hpp @@ -1796,11 +1796,11 @@ struct GGMLRunner { std::unordered_set runner_param_tensor_set; bool params_tensor_set_dirty_ = true; - // static so nested runners (e.g. text encoders inside a conditioner) are also - // intercepted; measurement is single-threaded like the rest of param fitting - static inline bool measure_mode_ = false; - static inline std::vector* measure_collector_ = nullptr; - static inline size_t measure_generation_ = 0; + // Thread-local so nested runners in one fitting call are intercepted without + // affecting generation or fitting calls running on other threads. + static inline thread_local bool measure_mode_ = false; + static inline thread_local std::vector* measure_collector_ = nullptr; + static inline thread_local size_t measure_generation_ = 0; graph_memory_measurement last_measurement_; SDBackendModule fit_module_ = SDBackendModule::UNSET; size_t measure_generation_seen_ = 0; diff --git a/src/core/util.cpp b/src/core/util.cpp index 23af71e73..4106cff52 100644 --- a/src/core/util.cpp +++ b/src/core/util.cpp @@ -341,6 +341,7 @@ int32_t sd_get_num_physical_cores() { static sd_progress_cb_t sd_progress_cb = nullptr; void* sd_progress_cb_data = nullptr; +static thread_local bool sd_progress_suppressed = false; static sd_abort_cb_t sd_abort_cb = nullptr; static void* sd_abort_cb_data = nullptr; @@ -537,6 +538,9 @@ static void print_progress_line(int step, int steps, const std::string& speed_te } void pretty_progress(int step, int steps, float time) { + if (sd_progress_suppressed) { + return; + } if (sd_progress_cb) { sd_progress_cb(step, steps, time, sd_progress_cb_data); return; @@ -554,6 +558,9 @@ void pretty_progress(int step, int steps, float time) { } void pretty_bytes_progress(int step, int steps, uint64_t bytes_processed, float elapsed_seconds) { + if (sd_progress_suppressed) { + return; + } if (sd_progress_cb) { float time = elapsed_seconds / (step + 1e-6f); sd_progress_cb(step, steps, time, sd_progress_cb_data); @@ -687,6 +694,14 @@ void* sd_get_progress_callback_data() { return sd_progress_cb_data; } +bool sd_get_progress_suppressed() { + return sd_progress_suppressed; +} + +void sd_set_progress_suppressed(bool suppressed) { + sd_progress_suppressed = suppressed; +} + sd_image_t tensor_to_sd_image(const sd::Tensor& tensor, int frame_index) { const auto& shape = tensor.shape(); GGML_ASSERT(shape.size() == 4 || shape.size() == 5); diff --git a/src/core/util.h b/src/core/util.h index e882ece23..e0308d330 100644 --- a/src/core/util.h +++ b/src/core/util.h @@ -91,6 +91,8 @@ std::vector> split_quotation_attention( sd_progress_cb_t sd_get_progress_callback(); void* sd_get_progress_callback_data(); +bool sd_get_progress_suppressed(); +void sd_set_progress_suppressed(bool suppressed); sd_preview_cb_t sd_get_preview_callback(); void* sd_get_preview_callback_data(); diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index 674103481..fbbf669fc 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -4093,10 +4093,18 @@ enum sd_fit_status_t sd_fit_params(const sd_ctx_params_t* sd_ctx_params, requested_tiling = workload->video_gen_params->vae_tiling_params; } - // silence step progress during measurement, it would pollute stdout - sd_progress_cb_t saved_progress_cb = sd_get_progress_callback(); - void* saved_progress_data = sd_get_progress_callback_data(); - sd_set_progress_callback([](int, int, float, void*) {}, nullptr); + // Silence progress from this dry run without replacing the process-wide + // callback used by concurrent generation calls. + struct ProgressSuppressionGuard { + ProgressSuppressionGuard() + : previous(sd_get_progress_suppressed()) { + sd_set_progress_suppressed(true); + } + ~ProgressSuppressionGuard() { + sd_set_progress_suppressed(previous); + } + bool previous; + } progress_suppression_guard; // run the real generation pipeline in measure mode: every runner builds its graphs, // records memory requirements and returns shaped zero tensors, no weights are read @@ -4194,7 +4202,6 @@ enum sd_fit_status_t sd_fit_params(const sd_ctx_params_t* sd_ctx_params, std::vector records; if (!measure(requested_tiling, records) || records.empty()) { LOG_ERROR("fit-params: measurement dry run failed"); - sd_set_progress_callback(saved_progress_cb, saved_progress_data); delete sd_ctx->sd; sd_ctx->sd = nullptr; return SD_FIT_ERROR; @@ -4246,8 +4253,6 @@ enum sd_fit_status_t sd_fit_params(const sd_ctx_params_t* sd_ctx_params, } } - sd_set_progress_callback(saved_progress_cb, saved_progress_data); - const bool user_set_placement = strlen(SAFE_STR(sd_ctx_params->backend)) > 0 || strlen(SAFE_STR(sd_ctx_params->params_backend)) > 0; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 77950a212..74c60e8d0 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -8,7 +8,7 @@ add_test(NAME test-vae-routing COMMAND test-vae-routing) add_executable(test-fit-params test-fit-params.cpp) target_include_directories(test-fit-params PRIVATE "${PROJECT_SOURCE_DIR}/src") -target_link_libraries(test-fit-params PRIVATE stable-diffusion) +target_link_libraries(test-fit-params PRIVATE stable-diffusion ${CMAKE_THREAD_LIBS_INIT}) add_test(NAME test-fit-params COMMAND test-fit-params) add_executable(test-ltx-vae-temporal test-ltx-vae-temporal.cpp) diff --git a/tests/test-fit-params.cpp b/tests/test-fit-params.cpp index f01a25c21..a9ae21aca 100644 --- a/tests/test-fit-params.cpp +++ b/tests/test-fit-params.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include "core/fit_params.h" @@ -248,6 +249,21 @@ bool test_measure_mode_preserves_outputs_and_projects_cache() { return passed; } +bool test_measure_mode_is_thread_local() { + std::vector records; + GGMLRunner::set_measure_mode(true, &records); + const bool enabled_on_calling_thread = GGMLRunner::measure_mode_enabled(); + bool enabled_on_other_thread = true; + std::thread other_thread([&]() { + enabled_on_other_thread = GGMLRunner::measure_mode_enabled(); + }); + other_thread.join(); + GGMLRunner::set_measure_mode(false); + + return expect(enabled_on_calling_thread, "measurement should be enabled on the fitting thread") && + expect(!enabled_on_other_thread, "measurement must not affect another thread"); +} + bool test_public_result_is_initialized_on_error() { sd_fit_workload_t workload; sd_fit_workload_init(&workload); @@ -280,6 +296,7 @@ int main() { !test_controlnet_compute_is_concurrent() || !test_cpu_fallback_checks_host_memory() || !test_measure_mode_preserves_outputs_and_projects_cache() || + !test_measure_mode_is_thread_local() || !test_public_result_is_initialized_on_error()) { return 1; } From 3e36b3fe5a7b9b50890d5168a8840c996b52c8aa Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Fri, 28 Aug 2026 07:42:27 +0200 Subject: [PATCH 14/19] fix: reject unvalidated fit placement inputs --- docs/backend.md | 10 ++++++---- examples/fit-params/README.md | 17 +++++++++-------- include/stable-diffusion.h | 2 +- src/stable-diffusion.cpp | 13 ++++++------- tests/test-fit-params.cpp | 17 +++++++++++++++++ 5 files changed, 39 insertions(+), 20 deletions(-) diff --git a/docs/backend.md b/docs/backend.md index ee6ae1217..eff576d1e 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -164,9 +164,10 @@ cat args.txt | xargs sd-cli -m model.gguf -p "a cat" -W 1024 -H 1024 `--fit-print` prints the measured per-device / per-module memory table instead of arguments. Budgets reuse `--max-vram`, with measured fitting retaining a 512 MiB safety margin after applying automatic or explicit per-device limits. -If the current parameters already fit, nothing needs to change and the tool -prints an empty line. If `--backend` / `--params-backend` are already set and -changes would be needed, the tool fails instead of overriding them. +If the default placement already fits, nothing needs to change and the tool +prints an empty line. Explicit `--backend` / `--params-backend` assignments are +rejected because measured fitting derives a new placement rather than validating +an existing one. The same measurement is available to library users through `sd_fit_params()` in `stable-diffusion.h`, which takes the context params plus an @@ -179,7 +180,8 @@ affect the graph being measured. Result strings are owned by `sd_fit_result_t` and must be released with `sd_fit_result_free()`. Persistent cache buffers and the concurrent diffusion/ControlNet compute phase are included in the measured peak. CPU fallbacks are accepted only when the projected CPU parameters and -compute buffers fit currently available host memory. +compute buffers fit currently available host memory. Callers must leave +`backend` and `params_backend` unset while requesting a fit. ## Modules diff --git a/examples/fit-params/README.md b/examples/fit-params/README.md index 72f711fe0..2bca8fdc0 100644 --- a/examples/fit-params/README.md +++ b/examples/fit-params/README.md @@ -46,8 +46,9 @@ Useful flags: detection, and `0` disables graph splitting. - `--fit-print`: print the measured memory table to stdout instead of arguments - `-p`: representative prompt (token count affects text encoder memory) -- generation inputs including init/control/reference images, LoRAs, hires, and - model placement options flow into the measurement as they would into a real run +- generation inputs including init/control/reference images, LoRAs, and hires + options flow into the measurement as they would into a real run; explicit + `--backend` / `--params-backend` placement is rejected ## Planner order @@ -97,10 +98,10 @@ The checks run in this order: the CPU runtime backend. The planner returns `SD_FIT_FAILURE` if the CPU parameters and compute phases exceed currently available host memory. -If the current parameters already fit, the tool prints an empty line and -reports that no changes are needed. If `--backend` / `--params-backend` are -already set and changes would be needed, the tool fails instead of overriding -them. +If the default placement already fits, the tool prints an empty line and +reports that no changes are needed. Explicit `--backend` / `--params-backend` +assignments are rejected because the tool derives placement rather than +validating an existing assignment. See `docs/backend.md` for the placement spec syntax and the heuristic `--auto-fit` alternative built into `sd-cli`. @@ -167,8 +168,8 @@ image-only model returns `SD_FIT_ERROR`. - `SD_FIT_SUCCESS`: a placement was found, or the current/default placement already fits. -- `SD_FIT_FAILURE`: no placement was projected to fit, or placement was needed - but `ctx.backend` / `ctx.params_backend` was already set by the caller. +- `SD_FIT_FAILURE`: no placement was projected to fit, or `ctx.backend` / + `ctx.params_backend` was already set by the caller. - `SD_FIT_ERROR`: invalid inputs or a hard measurement error such as an unreadable model. diff --git a/include/stable-diffusion.h b/include/stable-diffusion.h index ab23b7fcd..856641410 100644 --- a/include/stable-diffusion.h +++ b/include/stable-diffusion.h @@ -533,7 +533,7 @@ typedef struct { SD_API void sd_fit_workload_init(sd_fit_workload_t* workload); // fit model/context placement to free device memory using measured no-alloc dry runs; -// reads only model metadata, never weight data +// reads only model metadata, never weight data; backend and params_backend must be unset SD_API enum sd_fit_status_t sd_fit_params(const sd_ctx_params_t* sd_ctx_params, const sd_fit_workload_t* workload, sd_fit_result_t* result); diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index fbbf669fc..4042354f5 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -4044,6 +4044,11 @@ enum sd_fit_status_t sd_fit_params(const sd_ctx_params_t* sd_ctx_params, LOG_ERROR("fit-params: set at most one complete generation request"); return SD_FIT_ERROR; } + if (strlen(SAFE_STR(sd_ctx_params->backend)) > 0 || + strlen(SAFE_STR(sd_ctx_params->params_backend)) > 0) { + LOG_WARN("fit-params: explicit backend placement cannot be validated; clear backend and params_backend before fitting"); + return SD_FIT_FAILURE; + } int64_t t0 = ggml_time_ms(); @@ -4253,9 +4258,6 @@ enum sd_fit_status_t sd_fit_params(const sd_ctx_params_t* sd_ctx_params, } } - const bool user_set_placement = strlen(SAFE_STR(sd_ctx_params->backend)) > 0 || - strlen(SAFE_STR(sd_ctx_params->params_backend)) > 0; - std::vector modules; for (const auto& kv : module_map) { modules.push_back(kv.second); @@ -4273,10 +4275,7 @@ enum sd_fit_status_t sd_fit_params(const sd_ctx_params_t* sd_ctx_params, result->report = strdup(plan.report.c_str()); result->vae_tiling = plan.vae_tiling; result->stream_layers = plan.stream_layers; - if (plan.changed && user_set_placement) { - LOG_WARN("fit-params: changes needed but --backend/--params-backend already set by user, abort"); - return SD_FIT_FAILURE; - } else if (plan.changed) { + if (plan.changed) { result->changed = true; if (!plan.runtime_spec.empty()) { result->backend = strdup(plan.runtime_spec.c_str()); diff --git a/tests/test-fit-params.cpp b/tests/test-fit-params.cpp index a9ae21aca..bb449f962 100644 --- a/tests/test-fit-params.cpp +++ b/tests/test-fit-params.cpp @@ -283,6 +283,22 @@ bool test_public_result_is_initialized_on_error() { "fit result should be initialized before argument validation"); } +bool test_public_rejects_explicit_placement() { + sd_ctx_params_t params; + sd_ctx_params_init(¶ms); + params.backend = "cpu"; + sd_fit_workload_t workload; + sd_fit_workload_init(&workload); + sd_fit_result_t result; + + const enum sd_fit_status_t status = sd_fit_params(¶ms, &workload, &result); + const bool passed = expect(status == SD_FIT_FAILURE, "explicit placement should be rejected") && + expect(!result.changed && result.backend == nullptr && result.params_backend == nullptr && result.report == nullptr, + "placement rejection should leave an initialized result"); + sd_fit_result_free(&result); + return passed; +} + } // namespace int main() { @@ -297,6 +313,7 @@ int main() { !test_cpu_fallback_checks_host_memory() || !test_measure_mode_preserves_outputs_and_projects_cache() || !test_measure_mode_is_thread_local() || + !test_public_rejects_explicit_placement() || !test_public_result_is_initialized_on_error()) { return 1; } From 8d2a90f05ae2e1c92cb6368c0c9a5f73be409c75 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Fri, 28 Aug 2026 07:43:30 +0200 Subject: [PATCH 15/19] fix: return fit report when planning fails --- src/stable-diffusion.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index 4042354f5..b85b35e06 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -4268,13 +4268,13 @@ enum sd_fit_status_t sd_fit_params(const sd_ctx_params_t* sd_ctx_params, delete sd_ctx->sd; sd_ctx->sd = nullptr; + result->report = strdup(plan.report.c_str()); + result->vae_tiling = plan.vae_tiling; + result->stream_layers = plan.stream_layers; if (!planned || !plan.valid) { return SD_FIT_FAILURE; } - result->report = strdup(plan.report.c_str()); - result->vae_tiling = plan.vae_tiling; - result->stream_layers = plan.stream_layers; if (plan.changed) { result->changed = true; if (!plan.runtime_spec.empty()) { From f37b185f8edb59fb9814bef67e50bb9cce33b1ae Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Fri, 28 Aug 2026 07:45:04 +0200 Subject: [PATCH 16/19] fix: serialize gguf probe log suppression --- src/model_io/gguf_io.cpp | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/model_io/gguf_io.cpp b/src/model_io/gguf_io.cpp index 9c4cd28d3..c58b08d6a 100644 --- a/src/model_io/gguf_io.cpp +++ b/src/model_io/gguf_io.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -18,6 +19,8 @@ static void set_error(std::string* error, const std::string& message) { } } +static std::mutex gguf_probe_log_mutex; + bool is_gguf_file(const std::string& file_path) { std::ifstream file(file_path, std::ios::binary); if (!file.is_open()) { @@ -50,12 +53,15 @@ bool read_gguf_file(const std::string& file_path, // ggml's reader rejects tensors with more than GGML_MAX_DIMS dimensions (e.g. // the 5-D Wan patch_embedding.weight) that the GGUFReader fallback handles, // so silence its log during the probe to avoid spurious error messages - ggml_log_callback saved_log_cb = nullptr; - void* saved_log_data = nullptr; - ggml_log_get(&saved_log_cb, &saved_log_data); - ggml_log_set([](ggml_log_level, const char*, void*) {}, nullptr); - ctx_gguf_ = gguf_init_from_file(file_path.c_str(), {true, &ctx_meta_}); - ggml_log_set(saved_log_cb, saved_log_data); + { + std::lock_guard lock(gguf_probe_log_mutex); + ggml_log_callback saved_log_cb = nullptr; + void* saved_log_data = nullptr; + ggml_log_get(&saved_log_cb, &saved_log_data); + ggml_log_set([](ggml_log_level, const char*, void*) {}, nullptr); + ctx_gguf_ = gguf_init_from_file(file_path.c_str(), {true, &ctx_meta_}); + ggml_log_set(saved_log_cb, saved_log_data); + } if (!ctx_gguf_) { GGUFReader gguf_reader; From b02025668ec45780d1a9b4ca2f3e895b237a1cf5 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Fri, 28 Aug 2026 07:45:50 +0200 Subject: [PATCH 17/19] test: make fit environment setup portable --- tests/test-fit-params.cpp | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/tests/test-fit-params.cpp b/tests/test-fit-params.cpp index bb449f962..0206cf98b 100644 --- a/tests/test-fit-params.cpp +++ b/tests/test-fit-params.cpp @@ -20,6 +20,22 @@ bool expect(bool condition, const char* message) { return true; } +bool set_test_env(const char* name, const std::string& value) { +#if defined(_WIN32) + return _putenv_s(name, value.c_str()) == 0; +#else + return setenv(name, value.c_str(), 1) == 0; +#endif +} + +void unset_test_env(const char* name) { +#if defined(_WIN32) + _putenv_s(name, ""); +#else + unsetenv(name); +#endif +} + struct MeasureRunner : public GGMLRunner { bool warm_cache_seen = false; ggml_tensor* output = nullptr; @@ -79,8 +95,10 @@ bool plan_with_devices(const char* devices, const std::vector& modules, sd::fit_params::FitPlan* plan, float host_memory_gib = 64.f) { - setenv("SD_FIT_DEBUG_DEVICES", devices, 1); - setenv("SD_FIT_DEBUG_HOST_MEMORY_GIB", std::to_string(host_memory_gib).c_str(), 1); + if (!set_test_env("SD_FIT_DEBUG_DEVICES", devices) || + !set_test_env("SD_FIT_DEBUG_HOST_MEMORY_GIB", std::to_string(host_memory_gib))) { + return false; + } sd::ggml_graph_cut::MaxVramAssignment budgets; budgets.reset(max_vram_gib); return sd::fit_params::plan_placement(modules, budgets, plan); @@ -317,7 +335,7 @@ int main() { !test_public_result_is_initialized_on_error()) { return 1; } - unsetenv("SD_FIT_DEBUG_DEVICES"); - unsetenv("SD_FIT_DEBUG_HOST_MEMORY_GIB"); + unset_test_env("SD_FIT_DEBUG_DEVICES"); + unset_test_env("SD_FIT_DEBUG_HOST_MEMORY_GIB"); return 0; } From e0da992eb9a59a8302f8158fa2a120132cefd4fd Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Fri, 28 Aug 2026 07:54:06 +0200 Subject: [PATCH 18/19] docs: fix fit status type in C example --- examples/fit-params/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/fit-params/README.md b/examples/fit-params/README.md index 2bca8fdc0..2a16be97d 100644 --- a/examples/fit-params/README.md +++ b/examples/fit-params/README.md @@ -126,7 +126,7 @@ workload.height = 1024; workload.video_frames = 1; sd_fit_result_t result; -sd_fit_status_t status = sd_fit_params(&ctx, &workload, &result); +enum sd_fit_status_t status = sd_fit_params(&ctx, &workload, &result); if (status == SD_FIT_SUCCESS && result.changed) { printf("backend=%s\n", result.backend ? result.backend : ""); printf("params_backend=%s\n", result.params_backend ? result.params_backend : ""); From a6095334b3806dc5d05bf3083175d3543ef56eb4 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Fri, 28 Aug 2026 07:56:51 +0200 Subject: [PATCH 19/19] fix: keep gguf probe guard merge-friendly --- src/model_io/gguf_io.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/model_io/gguf_io.cpp b/src/model_io/gguf_io.cpp index c58b08d6a..decea8996 100644 --- a/src/model_io/gguf_io.cpp +++ b/src/model_io/gguf_io.cpp @@ -1,10 +1,11 @@ #include "gguf_io.h" +#include + #include #include #include #include -#include #include #include #include @@ -19,8 +20,6 @@ static void set_error(std::string* error, const std::string& message) { } } -static std::mutex gguf_probe_log_mutex; - bool is_gguf_file(const std::string& file_path) { std::ifstream file(file_path, std::ios::binary); if (!file.is_open()) { @@ -49,6 +48,7 @@ bool read_gguf_file(const std::string& file_path, gguf_context* ctx_gguf_ = nullptr; ggml_context* ctx_meta_ = nullptr; + static std::mutex gguf_probe_log_mutex; // ggml's reader rejects tensors with more than GGML_MAX_DIMS dimensions (e.g. // the 5-D Wan patch_embedding.weight) that the GGUFReader fallback handles,