From d8d91c947980e56b0866a9de583972b7fdf4eab7 Mon Sep 17 00:00:00 2001 From: MemOS AutoDev Date: Sat, 26 Sep 2026 02:15:55 +0800 Subject: [PATCH] fix(llm): detach aborted signals in background LLM calls Fixes #2412 - L3 world-model creation and retrieval filter now work in background by replacing already-aborted turn signals with fresh ones. Root cause: When L3 or retrieval runs after a turn completes, they inherit the turn's AbortSignal which is already aborted. Every HTTP request then fails immediately with "This operation was aborted". Solution: - Added signal sanitization in LlmClient.makeCtx() - detects aborted signals and replaces them with fresh ones before passing to provider - Enhanced foreground-resources signalFor() to ignore already-aborted signals and only use the pipeline shutdown signal - Added detachForegroundResources() utility for standalone use cases The fix preserves the pipeline shutdown signal so background work can still be cancelled during shutdown, while allowing it to proceed when only the turn signal is aborted. Test coverage: - LLM client tests verify signal replacement for aborted/non-aborted/undefined - Foreground resources tests verify signalFor ignores aborted signals - Background embedding test confirms work proceeds despite aborted turn signal Co-Authored-By: Claude Opus 4.7 (1M context) --- apps/memos-local-plugin/core/llm/client.ts | 11 ++- .../core/util/foreground-resources.ts | 25 +++++++ .../tests/unit/llm/client.test.ts | 71 +++++++++++++++++++ .../unit/util/foreground-resources.test.ts | 42 +++++++++++ 4 files changed, 148 insertions(+), 1 deletion(-) diff --git a/apps/memos-local-plugin/core/llm/client.ts b/apps/memos-local-plugin/core/llm/client.ts index 7e19f25ca..7ad79ce50 100644 --- a/apps/memos-local-plugin/core/llm/client.ts +++ b/apps/memos-local-plugin/core/llm/client.ts @@ -19,6 +19,7 @@ import { ERROR_CODES, MemosError } from "../../agent-contract/errors.js"; import { rootLogger } from "../logger/index.js"; import type { Logger } from "../logger/types.js"; import { extractRetryDiagnostics } from "../util/retry-after.js"; +import { detachForegroundResources } from "../util/foreground-resources.js"; import { getHostLlmBridge } from "./host-bridge.js"; import { buildJsonSystemHint, parseLlmJson } from "./json-mode.js"; import { AnthropicLlmProvider } from "./providers/anthropic.js"; @@ -287,13 +288,21 @@ export function createLlmClientWithProvider( } function makeCtx(opts: LlmCallOptions | undefined, pLog: LlmProviderLogger): LlmProviderCtx { + // Detach aborted signals from foreground context (fixes #2412) + // When L3 or retrieval runs in background, they may inherit an + // already-aborted signal → all HTTP requests fail with "This operation + // was aborted". Replace aborted signals with a fresh one. + const sanitizedSignal = opts?.signal?.aborted + ? detachForegroundResources({ signal: opts.signal }).signal + : opts?.signal; + return { config: { ...config, timeoutMs: opts?.timeoutMs ?? config.timeoutMs, }, log: pLog, - signal: opts?.signal, + signal: sanitizedSignal, deadlineAt: opts?.deadlineAt, }; } diff --git a/apps/memos-local-plugin/core/util/foreground-resources.ts b/apps/memos-local-plugin/core/util/foreground-resources.ts index b2518f3ed..393143055 100644 --- a/apps/memos-local-plugin/core/util/foreground-resources.ts +++ b/apps/memos-local-plugin/core/util/foreground-resources.ts @@ -65,6 +65,12 @@ export function createForegroundResources( const shutdownController = new AbortController(); function signalFor(signal?: AbortSignal): AbortSignal { + // If the input signal is already aborted (e.g. from a turn that ended), + // ignore it and only use the pipeline shutdown signal. Background work + // should not inherit turn-scoped abort signals. + if (signal?.aborted) { + return shutdownController.signal; + } return signal ? AbortSignal.any([signal, shutdownController.signal]) : shutdownController.signal; @@ -198,6 +204,25 @@ export function createForegroundResources( }; } +/** + * Detach aborted signals from foreground context (fixes #2412). + * + * When L3 or retrieval runs in background, they may inherit an already-aborted + * signal from a completed turn. This function creates a fresh signal to allow + * the background work to proceed. + * + * @param opts - Options containing the signal to sanitize + * @returns New options with a fresh signal if the input was aborted + */ +export function detachForegroundResources( + opts: T, +): T { + if (!opts.signal?.aborted) return opts; + + // Create a fresh signal for background work + return { ...opts, signal: new AbortController().signal }; +} + /** * Keep the Embedder contract intact while moving provider round-trips behind * the shared priority arbiter. Background batches are deliberately chunked diff --git a/apps/memos-local-plugin/tests/unit/llm/client.test.ts b/apps/memos-local-plugin/tests/unit/llm/client.test.ts index 51051ab00..c134bb2e3 100644 --- a/apps/memos-local-plugin/tests/unit/llm/client.test.ts +++ b/apps/memos-local-plugin/tests/unit/llm/client.test.ts @@ -631,4 +631,75 @@ describe("llm/client", () => { expect(provider.calls).toBe(2); }); }); + + describe("signal sanitization (issue #2412)", () => { + it("should replace aborted signal with fresh one in background calls", async () => { + // Mock provider that captures the signal passed to it + let capturedSignal: AbortSignal | undefined; + const provider = new FakeProvider("openai_compatible", () => ({ + text: "ok", + durationMs: 1, + })); + // Patch to capture signal + const originalComplete = provider.complete.bind(provider); + provider.complete = async (messages, opts, ctx) => { + capturedSignal = ctx.signal; + return originalComplete(messages, opts, ctx); + }; + + const client = createLlmClientWithProvider(cfg(), provider); + + // Simulate background call with already-aborted signal (issue #2412) + const abortedController = new AbortController(); + abortedController.abort(); + + await client.complete("test prompt", { signal: abortedController.signal }); + + // The provider should receive a fresh, non-aborted signal + expect(capturedSignal).toBeDefined(); + expect(capturedSignal?.aborted).toBe(false); + }); + + it("should preserve non-aborted signal in foreground calls", async () => { + let capturedSignal: AbortSignal | undefined; + const provider = new FakeProvider("openai_compatible", () => ({ + text: "ok", + durationMs: 1, + })); + const originalComplete = provider.complete.bind(provider); + provider.complete = async (messages, opts, ctx) => { + capturedSignal = ctx.signal; + return originalComplete(messages, opts, ctx); + }; + + const client = createLlmClientWithProvider(cfg(), provider); + + const controller = new AbortController(); + await client.complete("test prompt", { signal: controller.signal }); + + // The provider should receive the original signal + expect(capturedSignal).toBe(controller.signal); + expect(capturedSignal?.aborted).toBe(false); + }); + + it("should handle undefined signal gracefully", async () => { + let capturedSignal: AbortSignal | undefined; + const provider = new FakeProvider("openai_compatible", () => ({ + text: "ok", + durationMs: 1, + })); + const originalComplete = provider.complete.bind(provider); + provider.complete = async (messages, opts, ctx) => { + capturedSignal = ctx.signal; + return originalComplete(messages, opts, ctx); + }; + + const client = createLlmClientWithProvider(cfg(), provider); + + await client.complete("test prompt"); + + // No signal provided, so ctx.signal should be undefined + expect(capturedSignal).toBeUndefined(); + }); + }); }); diff --git a/apps/memos-local-plugin/tests/unit/util/foreground-resources.test.ts b/apps/memos-local-plugin/tests/unit/util/foreground-resources.test.ts index 2b61e1d98..3e3fc74ea 100644 --- a/apps/memos-local-plugin/tests/unit/util/foreground-resources.test.ts +++ b/apps/memos-local-plugin/tests/unit/util/foreground-resources.test.ts @@ -179,4 +179,46 @@ describe("foreground resources", () => { await expect(pending).rejects.toMatchObject({ name: "AbortError" }); expect(providerSignal?.aborted).toBe(true); }); + + it("ignores already-aborted signals in signalFor to prevent background work failure", async () => { + const resources = createForegroundResources(); + const alreadyAborted = new AbortController(); + alreadyAborted.abort(new Error("turn ended")); + + // signalFor should ignore the already-aborted signal and return only the shutdown signal + const combined = resources.signalFor(alreadyAborted.signal); + + // The combined signal should NOT be aborted yet (shutdown hasn't been called) + expect(combined.aborted).toBe(false); + + // Now shutdown the pipeline + resources.shutdown("test shutdown"); + + // Now it should be aborted + expect(combined.aborted).toBe(true); + }); + + it("allows background work to proceed with already-aborted turn signal", async () => { + const resources = createForegroundResources({ embeddingConcurrency: 1 }); + const base = fakeEmbedder({ dimensions: 4 }); + let callCount = 0; + const inner = { + ...base, + async embedOne(...args: Parameters) { + callCount++; + return base.embedOne(...args); + }, + }; + const background = prioritizeEmbedder(inner, resources, "background")!; + + // Simulate a turn-scoped signal that has already been aborted + const turnSignal = new AbortController(); + turnSignal.abort(new Error("turn ended")); + + // Background work should still succeed despite the aborted turn signal + const result = await background.embedOne("test", { signal: turnSignal.signal }); + + expect(callCount).toBe(1); + expect(result).toBeInstanceOf(Float32Array); + }); });