From f1c8b80662ce43985f3f876fa3c5abf45453fd5d Mon Sep 17 00:00:00 2001 From: Dmitry Malishev Date: Tue, 25 Aug 2026 15:11:07 +0300 Subject: [PATCH 1/2] vae: re-point graph params at the fallback backend during CPU fallback The VAE auto-CPU-fallback switches the runner's runtime backend to CPU and recomputes the graph, but prepare_params stages weights to each tensor state's registration-time compute backend - and staging swaps the live tensor's buffer/data pointers to the staged copy. The CPU graph then dereferences device-staged memory: SIGSEGV in ggml_vec_dot_f16 on a Vulkan-staged weight, killing the process right after the "routing only this graph to CPU" log line. Fix inside the fallback branch: collect the graph's param tensors (before switching backends - switch_runtime_backend frees the compute ctx that owns the graph), quiesce the runner's prepared params (runner_done) and re-point them at the fallback backend via assign_compute_backend. With compute and params on the same backend, stage_tensors_to_compute_backend skips staging entirely and the CPU graph reads the params in place. Restore the assignment on every exit path so later graphs stage back to the real runtime backend. Surfaced by the qvac diffusion 2026-08-11 pair bump (QVAC-23767): the Wan 2.1 img2vid integration test - the only leg that exercises the fallback (win32, whose Vulkan driver caps logical buffers at 4GB) - died with a silent access violation on every run, while the same test on the 2026-07-03 engine passed with the same reroutes. Reproduced locally by forcing the budget down (--vae-auto-cpu-fallback-memory- ratio 0.05) on a Vulkan build; root-caused with gdb. After the fix the same forced run completes 25 reroute cycles across encode and decode tiles, with GPU sampling in between, and writes a valid video. --- src/core/ggml_extend.hpp | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/core/ggml_extend.hpp b/src/core/ggml_extend.hpp index 017d3d8f3..7f1409fbd 100644 --- a/src/core/ggml_extend.hpp +++ b/src/core/ggml_extend.hpp @@ -3315,7 +3315,37 @@ struct GGMLRunner { previous_backend_name.c_str(), cpu_backend_name.c_str()); + // prepare_params stages weights to each tensor state's + // registration-time compute backend, which does not follow + // the switched runtime: the CPU graph would read + // device-staged memory (SIGSEGV in ggml_vec_dot on a + // Vulkan-staged weight). Quiesce this runner's prepared + // params and re-point the graph's params at the fallback + // backend — with compute and params on the same backend, + // staging is skipped and the CPU graph reads the params in + // place. Restore on every exit path so later graphs stage + // back to the real runtime backend. Collect the params + // BEFORE switching: switch_runtime_backend frees the compute + // ctx that owns gf (the param pointers themselves are stable + // model tensors). + std::vector fallback_params = collect_used_param_tensors(gf); switch_runtime_backend(vae_fallback_backend); + auto repoint_params = [&](ggml_backend_t target) -> bool { + runner_done(); + auto manager = weight_manager.lock(); + if (manager == nullptr || fallback_params.empty()) { + return true; + } + return manager->assign_compute_backend(fallback_params, target); + }; + if (!repoint_params(vae_fallback_backend)) { + LOG_ERROR("%s VAE CPU fallback failed to re-point graph params to %s", + get_desc().c_str(), + cpu_backend_name.c_str()); + switch_runtime_backend(previous_backend); + free_compute_ctx(); + return std::nullopt; + } try { auto output = compute(get_graph, n_threads, @@ -3324,12 +3354,18 @@ struct GGMLRunner { free_compute_params, no_return); switch_runtime_backend(previous_backend); + if (!repoint_params(previous_backend)) { + LOG_WARN("%s VAE CPU fallback could not restore graph params to %s", + get_desc().c_str(), + previous_backend_name.c_str()); + } LOG_INFO("%s VAE CPU fallback complete; restored runtime backend %s", get_desc().c_str(), previous_backend_name.c_str()); return output; } catch (...) { switch_runtime_backend(previous_backend); + repoint_params(previous_backend); throw; } } From 97b90330a6c297186e4b064031e15bfd5ce42759 Mon Sep 17 00:00:00 2001 From: Dmitry Malishev Date: Wed, 26 Aug 2026 13:41:43 +0300 Subject: [PATCH 2/2] vae: harden the CPU-fallback reroute (review follow-ups) - Lift the collect/switch/re-point/restore sequence into one shared helper (compute_on_vae_fallback_backend) and call it from BOTH fallback entry points: the preflight route and the reactive retry_stateless_on_cpu. The retry previously performed the raw backend switch with no param re-pointing -- the same device-staged-weights SIGSEGV through the other door -- and a shared helper stops the two paths drifting apart again. - Make ModelManager::assign_compute_backend two-phase (validate every state, then commit) so a mid-loop refusal can no longer leave a prefix of the tensors re-pointed at the new backend with no rollback. - Treat a failed restore as fatal to the feature: log at error level, disable vae_auto_cpu_fallback_enabled so later graphs cannot re-enter a path whose restore is known broken, and fail the call instead of returning output computed against inconsistent bindings; the exception path now reports it too instead of silently discarding the result. Validated locally (Vulkan, forced with --vae-auto-cpu-fallback-memory-ratio 0.05): 3 preflight reroute cycles AND one reactive-retry cycle ("alloc compute buffer failed" -> "retrying stateless graph on CPU") all complete and restore the runtime backend; rc=0, structurally valid AVI. --- src/core/ggml_extend.hpp | 173 ++++++++++++++++++++++----------------- src/model_manager.cpp | 35 ++++++-- 2 files changed, 128 insertions(+), 80 deletions(-) diff --git a/src/core/ggml_extend.hpp b/src/core/ggml_extend.hpp index 7f1409fbd..79e7197b7 100644 --- a/src/core/ggml_extend.hpp +++ b/src/core/ggml_extend.hpp @@ -3194,6 +3194,90 @@ struct GGMLRunner { return ggml_get_tensor(cache_ctx, name.c_str()); } + // Shared VAE-CPU-fallback reroute used by both entry points (the + // preflight route and the reactive retry after a failed + // execute_graph). prepare_params stages weights to each tensor state's + // registration-time compute backend, which does not follow a switched + // runtime: a CPU graph would read device-staged memory (SIGSEGV in + // ggml_vec_dot on a Vulkan-staged weight). So: collect the graph's + // params BEFORE switching (switch_runtime_backend frees the compute + // ctx that owns gf; the param pointers themselves are stable model + // tensors), quiesce this runner's prepared params, re-point the params + // at the fallback backend — with compute and params on the same + // backend staging is skipped and the CPU graph reads them in place — + // run the graph, then restore the runtime backend and the params on + // every exit path. A failed restore leaves the runner's weight + // bindings inconsistent with its runtime backend (the next graph would + // hand a host pointer to a GPU kernel), so it is fatal to the feature: + // disable the fallback and fail the call instead of returning output. + template + std::optional> compute_on_vae_fallback_backend( + ggml_cgraph* gf, + get_graph_cb_t get_graph, + int n_threads, + bool free_compute_buffer, + bool free_compute_params, + bool no_return) { + std::vector fallback_params = collect_used_param_tensors(gf); + ggml_backend_t previous_backend = runtime_backend; + const std::string previous_backend_name = ggml_backend_name(previous_backend); + const std::string cpu_backend_name = ggml_backend_name(vae_fallback_backend); + auto repoint_params = [&](ggml_backend_t target) -> bool { + runner_done(); + auto manager = weight_manager.lock(); + if (manager == nullptr || fallback_params.empty()) { + return true; + } + return manager->assign_compute_backend(fallback_params, target); + }; + switch_runtime_backend(vae_fallback_backend); + if (!repoint_params(vae_fallback_backend)) { + // assign_compute_backend validates before it commits, so a + // refusal here means nothing moved; switching back restores + // the exact pre-call state. + LOG_ERROR("%s VAE CPU fallback failed to re-point graph params to %s", + get_desc().c_str(), + cpu_backend_name.c_str()); + switch_runtime_backend(previous_backend); + free_compute_ctx(); + return std::nullopt; + } + auto restore = [&]() -> bool { + switch_runtime_backend(previous_backend); + return repoint_params(previous_backend); + }; + auto on_failed_restore = [&]() { + LOG_ERROR( + "%s VAE CPU fallback could not restore graph params to %s; " + "weight bindings no longer match the runtime backend — " + "disabling VAE auto CPU fallback and failing this call", + get_desc().c_str(), + previous_backend_name.c_str()); + vae_auto_cpu_fallback_enabled = false; + }; + try { + auto output = compute(get_graph, + n_threads, + false, + free_compute_buffer, + free_compute_params, + no_return); + if (!restore()) { + on_failed_restore(); + return std::nullopt; + } + LOG_INFO("%s VAE CPU fallback complete; restored runtime backend %s", + get_desc().c_str(), + previous_backend_name.c_str()); + return output; + } catch (...) { + if (!restore()) { + on_failed_restore(); + } + throw; + } + } + template std::optional> compute(get_graph_cb_t get_graph, int n_threads, @@ -3242,27 +3326,18 @@ struct GGMLRunner { has_stateful_cache) { return std::nullopt; } - ggml_backend_t previous_backend = runtime_backend; - const std::string previous_backend_name = - ggml_backend_name(previous_backend); LOG_WARN("%s VAE %s on %s; retrying stateless graph on CPU", get_desc().c_str(), failure, - previous_backend_name.c_str()); - switch_runtime_backend(vae_fallback_backend); - try { - auto output = compute(get_graph, - n_threads, - false, - free_compute_buffer, - free_compute_params, - no_return); - switch_runtime_backend(previous_backend); - return output; - } catch (...) { - switch_runtime_backend(previous_backend); - throw; - } + ggml_backend_name(runtime_backend)); + // gf is still owned by the (unfreed) compute ctx here; the + // shared reroute collects its params before switching. + return compute_on_vae_fallback_backend(gf, + get_graph, + n_threads, + free_compute_buffer, + free_compute_params, + no_return); }; if (vae_auto_cpu_fallback_enabled && @@ -3295,8 +3370,7 @@ struct GGMLRunner { capacity.free_memory_ratio); if (decision.use_cpu_fallback()) { - ggml_backend_t previous_backend = runtime_backend; - const std::string previous_backend_name = ggml_backend_name(previous_backend); + const std::string previous_backend_name = ggml_backend_name(runtime_backend); const std::string cpu_backend_name = ggml_backend_name(vae_fallback_backend); LOG_WARN( @@ -3315,59 +3389,12 @@ struct GGMLRunner { previous_backend_name.c_str(), cpu_backend_name.c_str()); - // prepare_params stages weights to each tensor state's - // registration-time compute backend, which does not follow - // the switched runtime: the CPU graph would read - // device-staged memory (SIGSEGV in ggml_vec_dot on a - // Vulkan-staged weight). Quiesce this runner's prepared - // params and re-point the graph's params at the fallback - // backend — with compute and params on the same backend, - // staging is skipped and the CPU graph reads the params in - // place. Restore on every exit path so later graphs stage - // back to the real runtime backend. Collect the params - // BEFORE switching: switch_runtime_backend frees the compute - // ctx that owns gf (the param pointers themselves are stable - // model tensors). - std::vector fallback_params = collect_used_param_tensors(gf); - switch_runtime_backend(vae_fallback_backend); - auto repoint_params = [&](ggml_backend_t target) -> bool { - runner_done(); - auto manager = weight_manager.lock(); - if (manager == nullptr || fallback_params.empty()) { - return true; - } - return manager->assign_compute_backend(fallback_params, target); - }; - if (!repoint_params(vae_fallback_backend)) { - LOG_ERROR("%s VAE CPU fallback failed to re-point graph params to %s", - get_desc().c_str(), - cpu_backend_name.c_str()); - switch_runtime_backend(previous_backend); - free_compute_ctx(); - return std::nullopt; - } - try { - auto output = compute(get_graph, - n_threads, - false, - free_compute_buffer, - free_compute_params, - no_return); - switch_runtime_backend(previous_backend); - if (!repoint_params(previous_backend)) { - LOG_WARN("%s VAE CPU fallback could not restore graph params to %s", - get_desc().c_str(), - previous_backend_name.c_str()); - } - LOG_INFO("%s VAE CPU fallback complete; restored runtime backend %s", - get_desc().c_str(), - previous_backend_name.c_str()); - return output; - } catch (...) { - switch_runtime_backend(previous_backend); - repoint_params(previous_backend); - throw; - } + return compute_on_vae_fallback_backend(gf, + get_graph, + n_threads, + free_compute_buffer, + free_compute_params, + no_return); } if (decision.reason == sd::VaeGraphRouteReason::STATEFUL_GRAPH) { diff --git a/src/model_manager.cpp b/src/model_manager.cpp index 802f2f79e..825af29b9 100644 --- a/src/model_manager.cpp +++ b/src/model_manager.cpp @@ -1142,30 +1142,51 @@ bool ModelManager::assign_compute_backend(const std::vector& tenso return false; } - for (TensorState* state : required_states) { - if (state == nullptr || state->tensor == nullptr) { - continue; - } - + // Two-phase: validate every state first, then commit. A mid-loop + // refusal must not leave a prefix of the tensors re-pointed at the new + // backend — a partially-moved set silently skips staging on the next + // graph (compute == params backend) and hands host pointers to device + // kernels, and the caller has no way to roll it back. + auto needs_move = [&](const TensorState* state, bool* params_follow_out) { const bool params_follow_compute = state->params_follow_compute_backend || state->residency_mode == ResidencyMode::Disk; + if (params_follow_out != nullptr) { + *params_follow_out = params_follow_compute; + } const bool compute_changes = state->compute_backend != compute_backend; const bool params_changes = params_follow_compute && state->params_backend != compute_backend; - if (!compute_changes && !params_changes) { + return compute_changes || params_changes; + }; + + for (const TensorState* state : required_states) { + if (state == nullptr || state->tensor == nullptr) { + continue; + } + bool params_follow_compute = false; + if (!needs_move(state, ¶ms_follow_compute)) { continue; } - if (state->active_prepare_count > 0 || state->staged_to_compute_backend) { LOG_ERROR("model manager cannot move active tensor '%s' to another compute backend", state->name.c_str()); return false; } + const bool params_changes = params_follow_compute && state->params_backend != compute_backend; if (params_changes && state->loaded_to_params_backend) { LOG_ERROR("model manager cannot move loaded tensor '%s' to another params backend", state->name.c_str()); return false; } + } + for (TensorState* state : required_states) { + if (state == nullptr || state->tensor == nullptr) { + continue; + } + bool params_follow_compute = false; + if (!needs_move(state, ¶ms_follow_compute)) { + continue; + } state->compute_backend = compute_backend; if (params_follow_compute) { state->params_backend = compute_backend;