diff --git a/bun.lock b/bun.lock index ac2e31b..a678512 100644 --- a/bun.lock +++ b/bun.lock @@ -20,7 +20,7 @@ }, "packages/plugins/xiaoyunque-generation": { "name": "@microvoid/convax-plugin-xiaoyunque-generation", - "version": "0.3.4", + "version": "0.3.5", "dependencies": { "@microvoid/convax-xiaoyunque-mcp": "workspace:*" } @@ -83,7 +83,7 @@ }, "packages/tools/xiaoyunque-mcp": { "name": "@microvoid/convax-xiaoyunque-mcp", - "version": "0.3.2", + "version": "0.3.3", "devDependencies": { "@types/bun": "1.3.14", "typescript": "5.9.3" diff --git a/packages/tools/xiaoyunque-mcp/scripts/live-image-smoke.ts b/packages/tools/xiaoyunque-mcp/scripts/live-image-smoke.ts index 9e4fc10..c46292e 100644 --- a/packages/tools/xiaoyunque-mcp/scripts/live-image-smoke.ts +++ b/packages/tools/xiaoyunque-mcp/scripts/live-image-smoke.ts @@ -55,7 +55,7 @@ function notify(method: string, params: unknown) { try { const initialized = await request("initialize", { capabilities: {}, - clientInfo: { name: "convax-xiaoyunque-live-smoke", version: "0.3.2" }, + clientInfo: { name: "convax-xiaoyunque-live-smoke", version: "0.3.3" }, protocolVersion: "2025-03-26", }) if ((initialized.result as Record | undefined)?.protocolVersion !== "2025-03-26") { diff --git a/packages/tools/xiaoyunque-mcp/scripts/live-video-smoke.ts b/packages/tools/xiaoyunque-mcp/scripts/live-video-smoke.ts index 6723535..eadb676 100644 --- a/packages/tools/xiaoyunque-mcp/scripts/live-video-smoke.ts +++ b/packages/tools/xiaoyunque-mcp/scripts/live-video-smoke.ts @@ -117,7 +117,7 @@ try { progress("mcp.initialize") const initialized = await request("initialize", { capabilities: {}, - clientInfo: { name: "convax-xiaoyunque-live-video-smoke", version: "0.3.2" }, + clientInfo: { name: "convax-xiaoyunque-live-video-smoke", version: "0.3.3" }, protocolVersion: "2025-03-26", }) if ((initialized.result as Record | undefined)?.protocolVersion !== "2025-03-26") { diff --git a/packages/tools/xiaoyunque-mcp/src/mcp-server.ts b/packages/tools/xiaoyunque-mcp/src/mcp-server.ts index 1666a9f..a7fdb45 100644 --- a/packages/tools/xiaoyunque-mcp/src/mcp-server.ts +++ b/packages/tools/xiaoyunque-mcp/src/mcp-server.ts @@ -153,7 +153,9 @@ export function publicGenerationErrorMessage(error: unknown) { return "The selected XiaoYunque image model is no longer available. Choose another image model and try again." } if (error instanceof XiaoYunqueReferenceAssetRegistrationError) { - return "XiaoYunque could not prepare the reference image for generation. No generation was submitted; try again." + return error.referenceType === "video" + ? "XiaoYunque could not prepare the reference video for generation. No generation was submitted; try again." + : "XiaoYunque could not prepare the reference image for generation. No generation was submitted; try again." } if (error instanceof XiaoYunqueRequestRejectedError) { return "XiaoYunque did not accept this generation request. Refresh Services and try a model listed for this capability." @@ -169,6 +171,7 @@ export const safeGenerationDiagnosticCodes = [ "status-check-timeout", "unsupported-image-model", "reference-image-registration-failed", + "reference-video-registration-failed", "upstream-envelope-rejected", "upstream-http-rejected", "upstream-request-rejected", @@ -185,7 +188,9 @@ export function safeGenerationDiagnosticCode(error: unknown): SafeGenerationDiag if (error instanceof XiaoYunqueQueryTimeoutError) return "status-check-timeout" if (error instanceof XiaoYunqueUnsupportedImageModelError) return "unsupported-image-model" if (error instanceof XiaoYunqueReferenceAssetRegistrationError) { - return "reference-image-registration-failed" + return error.referenceType === "video" + ? "reference-video-registration-failed" + : "reference-image-registration-failed" } if (error instanceof XiaoYunqueRequestRejectedError) return error.diagnosticCode return "unclassified-failure" diff --git a/packages/tools/xiaoyunque-mcp/src/xiaoyunque-api.ts b/packages/tools/xiaoyunque-mcp/src/xiaoyunque-api.ts index 04544b5..095c775 100644 --- a/packages/tools/xiaoyunque-mcp/src/xiaoyunque-api.ts +++ b/packages/tools/xiaoyunque-mcp/src/xiaoyunque-api.ts @@ -37,9 +37,11 @@ export class XiaoYunqueQueryTimeoutError extends Error { */ export class XiaoYunqueReferenceAssetRegistrationError extends Error { override name = "XiaoYunqueReferenceAssetRegistrationError" + readonly referenceType: "image" | "video" - constructor() { - super("XiaoYunque reference image asset registration failed") + constructor(referenceType: "image" | "video" = "image") { + super(`XiaoYunque reference ${referenceType} asset registration failed`) + this.referenceType = referenceType } } @@ -136,6 +138,8 @@ interface XiaoYunqueUserInfo { } type FetchLike = typeof fetch +type UploadAssetType = 1 | 2 | 4 +type RegisteredAssetType = 1 | 2 function record(value: unknown) { return value && typeof value === "object" && !Array.isArray(value) @@ -316,7 +320,7 @@ async function boundedJson(response: Response, label: string) { } } -function uploadAssetType(reference: FileGenerationReference) { +function uploadAssetType(reference: FileGenerationReference): UploadAssetType { if (reference.role === "reference_video" || reference.mime_type.startsWith("video/")) return 1 if (reference.role === "audio" || reference.mime_type.startsWith("audio/")) return 4 if ( @@ -328,10 +332,21 @@ function uploadAssetType(reference: FileGenerationReference) { throw new Error("XiaoYunque does not support this reference type") } +function registeredAssetTypeForUpload(assetType: UploadAssetType): RegisteredAssetType | undefined { + switch (assetType) { + case 1: // video + return 2 + case 2: // image + return 1 + case 4: // audio does not require AssetCreateV2 registration + return undefined + } +} + function uploadedAsset( value: unknown, reference: FileGenerationReference, - assetType: 1 | 2 | 4, + assetType: UploadAssetType, allowLoopbackHttp: boolean, ): UploadedAsset { const data = record(value) @@ -631,27 +646,36 @@ export class XiaoYunqueApi { const response = await this.#request(uploadPath, session, signal, { body, method: "POST" }) const payload = requireSuccess(response.value, response.status, "XiaoYunque reference upload") const asset = uploadedAsset(payload.data, reference, assetType, this.#allowLoopbackTest) - if (assetType !== 2 || asset.pippitAssetId !== undefined) return asset + const registeredAssetType = registeredAssetTypeForUpload(assetType) + if (registeredAssetType === undefined || asset.pippitAssetId !== undefined) return asset return { ...asset, - pippitAssetId: await this.#registerUploadedImage(asset.assetId, session, signal), + pippitAssetId: await this.#registerUploadedAsset( + asset.assetId, + registeredAssetType, + session, + signal, + ), } } - async #registerUploadedImage( + async #registerUploadedAsset( assetId: string, + assetType: RegisteredAssetType, session: StoredWebSession, signal: AbortSignal, ) { try { // `upload_file.asset_id` is an EverPhoto source id. The first-party Web // product converts it into the identity consumed by image and video generation via - // AssetCreateV2; the two ids are deliberately never treated as aliases. + // AssetCreateV2; the two ids are deliberately never treated as aliases. The + // upload_file and AssetCreateV2 asset-type enums differ: upload 2/1 maps to + // registered image/video 1/2 respectively. const response = await this.#jsonRequest(assetCreateV2Path, { asset_source_type: 3, asset_source_id: assetId, - asset_type: 1, + asset_type: assetType, Base: { Client: "web" }, }, session, signal) const payload = requireAssetRegistrationSuccess(response.value, response.status) @@ -664,7 +688,7 @@ export class XiaoYunqueApi { } catch (error) { if (signal.aborted) throw cancellationError(signal) if (error instanceof XiaoYunqueAuthenticationError) throw error - throw new XiaoYunqueReferenceAssetRegistrationError() + throw new XiaoYunqueReferenceAssetRegistrationError(assetType === 1 ? "image" : "video") } } diff --git a/packages/tools/xiaoyunque-mcp/test/generator.test.ts b/packages/tools/xiaoyunque-mcp/test/generator.test.ts index 425d8a8..4abe54b 100644 --- a/packages/tools/xiaoyunque-mcp/test/generator.test.ts +++ b/packages/tools/xiaoyunque-mcp/test/generator.test.ts @@ -482,6 +482,110 @@ describe("XiaoYunque generation engine", () => { } }) + test("registers and submits one Canvas reference video for video generation", async () => { + const directory = await mkdtemp(path.join(os.tmpdir(), "xiaoyunque-reference-video-")) + directories.push(directory) + const output = path.join(directory, "output") + const referenceVideo = path.join(directory, "reference.mp4") + await Promise.all([mkdir(output), writeFile(referenceVideo, mp4)]) + let uploadCount = 0 + let registrationCount = 0 + let submittedBody: Record | undefined + let submittedTask: ReturnType | undefined + const server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch: async (request): Promise => { + const url = new URL(request.url) + if (url.pathname === "/api/web/v1/common/upload_file") { + uploadCount += 1 + const form = await request.formData() + expect(path.basename((form.get("file") as File | null)?.name ?? "")).toBe("reference.mp4") + expect(form.get("asset_type")).toBe("1") + return Response.json({ + ret: 0, + data: { + asset_id: "everphoto-reference-video", + download_url: `${url.origin}/uploaded-reference.mp4`, + duration_ms: 5_000, + }, + }) + } + if (url.pathname === "/api/biz/v1/asset/create_v2") { + registrationCount += 1 + expect(await request.json()).toEqual({ + asset_source_type: 3, + asset_source_id: "everphoto-reference-video", + asset_type: 2, + Base: { Client: "web" }, + }) + return Response.json({ ret: 0, data: { PippitAssetID: "pippit-reference-video" } }) + } + const accountPreflight = await accountPreflightResponse(request) + if (accountPreflight) return accountPreflight + if (url.pathname === "/api/biz/v1/agent/submit_run") { + submittedBody = await request.json() as Record + submittedTask = taskFromSubmitBody(submittedBody) + return Response.json({ ret: 0, data: { accepted: true } }) + } + if (url.pathname === "/api/biz/v1/agent/get_thread") { + return Response.json({ + ret: 0, + data: { + thread: { + thread_id: submittedTask?.threadId, + run_list: [{ + entry_list: [artifactEntry("biz/x_data_video", `${url.origin}/result.mp4`)], + run_id: submittedTask?.runId, + state: 3, + thread_id: submittedTask?.threadId, + }], + }, + }, + }) + } + if (url.pathname === "/result.mp4") return new Response(mp4, { headers: { "Content-Type": "video/mp4" } }) + return new Response("not found", { status: 404 }) + }, + }) + const store = new OperationStore(path.join(directory, "state", "operations.json")) + const engine = new GenerationEngine({ + api: new XiaoYunqueApi(`http://127.0.0.1:${server.port}`), + authorizer: { session: async () => testSession }, + operationStore: store, + pollIntervalMs: 1, + }) + try { + const artifacts = await engine.generate(call({ + operation_id: "reference-video-generation", + output: "video", + output_directory: output, + references: [{ + kind: "file", + mime_type: "video/mp4", + name: "reference.mp4", + node_id: "reference-video-node", + path: referenceVideo, + role: "reference_video", + }], + }), videoModel, new AbortController().signal) + + expect(artifacts[0]?.mimeType).toBe("video/mp4") + expect(uploadCount).toBe(1) + expect(registrationCount).toBe(1) + expect(directVideoParameters(submittedBody!)).toMatchObject({ + model: "Seedance_2.0_mini_lite", + videos: [{ + asset_id: "everphoto-reference-video", + pippit_asset_id: "pippit-reference-video", + }], + }) + expect(await store.find("reference-video-generation")).toMatchObject({ status: "submitted" }) + } finally { + server.stop(true) + } + }) + test("rejects a last frame without a first frame before any API request", async () => { const directory = await mkdtemp(path.join(os.tmpdir(), "xiaoyunque-last-frame-only-")) directories.push(directory) diff --git a/packages/tools/xiaoyunque-mcp/test/mcp-server-errors.test.ts b/packages/tools/xiaoyunque-mcp/test/mcp-server-errors.test.ts index 76ae714..efb2257 100644 --- a/packages/tools/xiaoyunque-mcp/test/mcp-server-errors.test.ts +++ b/packages/tools/xiaoyunque-mcp/test/mcp-server-errors.test.ts @@ -38,6 +38,10 @@ describe("safe public generation errors", () => { .toBe( "XiaoYunque could not prepare the reference image for generation. No generation was submitted; try again.", ) + expect(publicGenerationErrorMessage(new XiaoYunqueReferenceAssetRegistrationError("video"))) + .toBe( + "XiaoYunque could not prepare the reference video for generation. No generation was submitted; try again.", + ) expect(publicGenerationErrorMessage(new XiaoYunqueRequestRejectedError("private rejection detail"))) .toBe( "XiaoYunque did not accept this generation request. Refresh Services and try a model listed for this capability.", @@ -51,6 +55,7 @@ describe("safe public generation errors", () => { publicGenerationErrorMessage(new XiaoYunqueObservationRejectedError("upstream-envelope-rejected")), publicGenerationErrorMessage(new XiaoYunqueQueryTimeoutError()), publicGenerationErrorMessage(new XiaoYunqueReferenceAssetRegistrationError()), + publicGenerationErrorMessage(new XiaoYunqueReferenceAssetRegistrationError("video")), publicGenerationErrorMessage(new XiaoYunqueRequestRejectedError("private rejection detail")), publicGenerationErrorMessage(new XiaoYunqueUnsupportedImageModelError()), ] @@ -74,6 +79,7 @@ describe("safe public generation errors", () => { safeGenerationDiagnosticCode(new XiaoYunqueObservationRejectedError("upstream-http-rejected")), safeGenerationDiagnosticCode(new XiaoYunqueQueryTimeoutError(privateDetail)), safeGenerationDiagnosticCode(new XiaoYunqueReferenceAssetRegistrationError()), + safeGenerationDiagnosticCode(new XiaoYunqueReferenceAssetRegistrationError("video")), safeGenerationDiagnosticCode(new XiaoYunqueUnsupportedImageModelError()), safeGenerationDiagnosticCode(new XiaoYunqueRequestRejectedError( privateDetail, @@ -87,6 +93,7 @@ describe("safe public generation errors", () => { "status-check-rejected", "status-check-timeout", "reference-image-registration-failed", + "reference-video-registration-failed", "unsupported-image-model", "upstream-envelope-rejected", "unclassified-failure", diff --git a/packages/tools/xiaoyunque-mcp/test/xiaoyunque-api.test.ts b/packages/tools/xiaoyunque-mcp/test/xiaoyunque-api.test.ts index b5b5f7c..308c69d 100644 --- a/packages/tools/xiaoyunque-mcp/test/xiaoyunque-api.test.ts +++ b/packages/tools/xiaoyunque-mcp/test/xiaoyunque-api.test.ts @@ -270,6 +270,7 @@ describe("XiaoYunque first-party Web generation API", () => { for (const api of [rejected, incomplete]) { const error = await api.upload(reference, session, signal).catch((reason: unknown) => reason) expect(error).toBeInstanceOf(XiaoYunqueReferenceAssetRegistrationError) + expect((error as XiaoYunqueReferenceAssetRegistrationError).referenceType).toBe("image") expect(String(error)).toBe( "XiaoYunqueReferenceAssetRegistrationError: XiaoYunque reference image asset registration failed", ) @@ -294,6 +295,7 @@ describe("XiaoYunque first-party Web generation API", () => { data: { asset_id: `asset-${seenTypes.length}`, download_url: `https://cdn.example.test/reference-${seenTypes.length}`, + pippit_asset_id: `pippit-${seenTypes.length}`, }, }) }) @@ -318,6 +320,95 @@ describe("XiaoYunque first-party Web generation API", () => { expect(seenTypes).toEqual(["1", "4"]) }) + test("registers an uploaded EverPhoto video with the video AssetCreateV2 type", async () => { + const directory = await mkdtemp(path.join(os.tmpdir(), "xiaoyunque-web-register-video-")) + directories.push(directory) + const referencePath = path.join(directory, "reference.mp4") + await writeFile(referencePath, Uint8Array.from([0, 0, 0, 16, 102, 116, 121, 112])) + const requests: string[] = [] + const api = apiFor((url, init) => { + requests.push(url.pathname) + expectWebHeaders(init?.headers) + if (url.pathname === "/api/web/v1/common/upload_file") { + const body = init?.body as FormData + expect(body.get("asset_type")).toBe("1") + return Response.json({ + ret: 0, + data: { + asset_id: "asset-video", + download_url: "https://cdn.example.test/reference.mp4", + duration_ms: 5_000, + pippit_asset_id: " ", + }, + }) + } + expect(url.pathname).toBe("/api/biz/v1/asset/create_v2") + expect(init?.method).toBe("POST") + expect(JSON.parse(String(init?.body))).toEqual({ + asset_source_type: 3, + asset_source_id: "asset-video", + asset_type: 2, + Base: { Client: "web" }, + }) + return Response.json({ + ret: "0", + data: { PippitAssetID: "pippit-video" }, + }) + }) + + expect(await api.upload({ + kind: "file", + mime_type: "video/mp4", + name: "reference.mp4", + node_id: "video-node", + path: referencePath, + role: "reference_video", + }, session, signal)).toEqual({ + assetId: "asset-video", + metadata: { durationMilliseconds: 5_000 }, + name: "reference.mp4", + pippitAssetId: "pippit-video", + url: "https://cdn.example.test/reference.mp4", + }) + expect(requests).toEqual([ + "/api/web/v1/common/upload_file", + "/api/biz/v1/asset/create_v2", + ]) + }) + + test("classifies failed reference video registration without leaking upstream details", async () => { + const directory = await mkdtemp(path.join(os.tmpdir(), "xiaoyunque-web-register-video-failure-")) + directories.push(directory) + const referencePath = path.join(directory, "reference.mp4") + await writeFile(referencePath, Uint8Array.from([0, 0, 0, 16, 102, 116, 121, 112])) + const api = apiFor((url) => url.pathname === "/api/web/v1/common/upload_file" + ? Response.json({ + ret: 0, + data: { + asset_id: "asset-video", + download_url: "https://cdn.example.test/reference.mp4", + }, + }) + : Response.json({ ret: 4001, errmsg: "private upstream video detail" })) + + const error = await api.upload({ + kind: "file", + mime_type: "video/mp4", + name: "reference.mp4", + node_id: "video-node", + path: referencePath, + role: "reference_video", + }, session, signal).catch((reason: unknown) => reason) + + expect(error).toBeInstanceOf(XiaoYunqueReferenceAssetRegistrationError) + expect((error as XiaoYunqueReferenceAssetRegistrationError).referenceType).toBe("video") + expect(String(error)).toBe( + "XiaoYunqueReferenceAssetRegistrationError: XiaoYunque reference video asset registration failed", + ) + expect(String(error)).not.toContain("private upstream video detail") + expect(String(error)).not.toContain("asset-video") + }) + test("submits one image through the first-party Canvas image flow", async () => { let submitBabiParam: Record | undefined let submitBody: Record | undefined