From 997247103c52ab3f5cabd30925971bb6df358cab Mon Sep 17 00:00:00 2001 From: Anton Guljajev Date: Wed, 30 Sep 2026 17:56:54 +0900 Subject: [PATCH 1/4] feat: capture open Unity Editor windows through MCP --- .../Helpers/EditorWindowScreenshotUtility.cs | 34 +- .../Editor/Tools/ManageEditorWindows.cs | 251 ++++++++++ .../Editor/Tools/ManageEditorWindows.cs.meta | 2 + Server/src/cli/CLI_USAGE_GUIDE.md | 7 + Server/src/cli/commands/editor.py | 36 ++ .../services/tools/manage_editor_windows.py | 64 +++ Server/src/services/tools/utils.py | 17 +- Server/tests/test_cli_editor_windows.py | 49 ++ Server/tests/test_manage_editor_windows.py | 123 +++++ .../Tools/ManageEditorWindowsTests.cs | 450 ++++++++++++++++++ .../Tools/ManageEditorWindowsTests.cs.meta | 2 + manifest.json | 4 + .../docs/guides/editor-window-screenshots.md | 95 ++++ website/docs/reference/cli.md | 15 + website/docs/reference/tools/core/index.md | 1 + .../tools/core/manage_editor_windows.md | 40 ++ website/docs/reference/tools/index.md | 3 +- website/sidebars.js | 1 + 18 files changed, 1183 insertions(+), 11 deletions(-) create mode 100644 MCPForUnity/Editor/Tools/ManageEditorWindows.cs create mode 100644 MCPForUnity/Editor/Tools/ManageEditorWindows.cs.meta create mode 100644 Server/src/services/tools/manage_editor_windows.py create mode 100644 Server/tests/test_cli_editor_windows.py create mode 100644 Server/tests/test_manage_editor_windows.py create mode 100644 TestProjects/UnityMCPTests/Assets/Tests/EditMode/Tools/ManageEditorWindowsTests.cs create mode 100644 TestProjects/UnityMCPTests/Assets/Tests/EditMode/Tools/ManageEditorWindowsTests.cs.meta create mode 100644 website/docs/guides/editor-window-screenshots.md create mode 100644 website/docs/reference/tools/core/manage_editor_windows.md diff --git a/MCPForUnity/Editor/Helpers/EditorWindowScreenshotUtility.cs b/MCPForUnity/Editor/Helpers/EditorWindowScreenshotUtility.cs index ee688718b..4913d511e 100644 --- a/MCPForUnity/Editor/Helpers/EditorWindowScreenshotUtility.cs +++ b/MCPForUnity/Editor/Helpers/EditorWindowScreenshotUtility.cs @@ -177,11 +177,31 @@ private static Rect GetViewportLocalRectPoints(SceneView sceneView, float pixels Mathf.Min(windowRect.height, viewportHeight)); } - private static Texture2D CaptureViewRect(SceneView sceneView, Rect viewportRectPixels) + internal static float GetWindowPixelsPerPoint(EditorWindow window) { - object hostView = GetHostView(sceneView); + // IPanel.scaledPixelsPerPoint is not public in the oldest supported Editors. + var panel = window.rootVisualElement.panel; + var property = panel?.GetType().GetProperty("scaledPixelsPerPoint", + BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); + if (property?.GetValue(panel, null) is float scale && scale > 0 && !float.IsInfinity(scale)) + return scale; + return EditorGUIUtility.pixelsPerPoint; + } + + internal static Texture2D CaptureWindowPixels(EditorWindow window, int width, int height) + { + if (window == null) throw new ArgumentNullException(nameof(window)); + if (width <= 0 || height <= 0 || (long)width * height > 16777216) + throw new ArgumentOutOfRangeException(nameof(width), "Empty or excessive capture area."); + InvokeMethodIfExists(GetHostView(window), "RepaintImmediately"); + return CaptureViewRect(window, new Rect(0, 0, width, height)); + } + + private static Texture2D CaptureViewRect(EditorWindow window, Rect viewportRectPixels) + { + object hostView = GetHostView(window); if (hostView == null) - throw new InvalidOperationException("Failed to resolve Scene view host view."); + throw new InvalidOperationException("Failed to resolve Editor window host view."); // GrabPixels is an internal extern on GUIView (parent of HostView), present since at least Unity 2021.1. // See: UnityCsReference/Editor/Mono/GUIView.bindings.cs — `internal extern void GrabPixels(RenderTexture, Rect)` @@ -200,6 +220,7 @@ private static Texture2D CaptureViewRect(SceneView sceneView, Rect viewportRectP int height = Mathf.RoundToInt(viewportRectPixels.height); RenderTexture rt = null; + Texture2D texture = null; RenderTexture previousActive = RenderTexture.active; try { @@ -214,11 +235,13 @@ private static Texture2D CaptureViewRect(SceneView sceneView, Rect viewportRectP grabPixels.Invoke(hostView, new object[] { rt, viewportRectPixels }); RenderTexture.active = rt; - var texture = new Texture2D(width, height, TextureFormat.RGBA32, false); + texture = new Texture2D(width, height, TextureFormat.RGBA32, false); texture.ReadPixels(new Rect(0, 0, width, height), 0, 0); texture.Apply(); FlipTextureVertically(texture); - return texture; + var result = texture; + texture = null; + return result; } catch (TargetInvocationException ex) { @@ -228,6 +251,7 @@ private static Texture2D CaptureViewRect(SceneView sceneView, Rect viewportRectP finally { RenderTexture.active = previousActive; + if (texture != null) UnityEngine.Object.DestroyImmediate(texture); if (rt != null) { rt.Release(); diff --git a/MCPForUnity/Editor/Tools/ManageEditorWindows.cs b/MCPForUnity/Editor/Tools/ManageEditorWindows.cs new file mode 100644 index 000000000..aa5f4f993 --- /dev/null +++ b/MCPForUnity/Editor/Tools/ManageEditorWindows.cs @@ -0,0 +1,251 @@ +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using MCPForUnity.Editor.Helpers; +using MCPForUnity.Runtime.Helpers; +using Newtonsoft.Json.Linq; +using UnityEditor; +using UnityEngine; +using Object = UnityEngine.Object; + +namespace MCPForUnity.Editor.Tools +{ + [McpForUnityTool("manage_editor_windows", AutoRegister = false)] + public static class ManageEditorWindows + { + private static Action cancelPendingCapture; + + internal sealed class Parameters + { + public string action = "list"; + public int? window_id; + public string window_title; + public string window_type; + public bool focus = true; + public bool restore_focus = true; + public bool include_image = true; + public bool save_file = false; + public int max_resolution = 1600; + } + + public static Task HandleCommand(JObject args) + { + try + { + var p = args?.ToObject() ?? new Parameters(); + var windows = UnityEngine.Resources.FindObjectsOfTypeAll() + .Where(w => w != null && (w.docked || w.hasFocus)) + .OrderBy(w => w.titleContent.text, StringComparer.OrdinalIgnoreCase) + .ThenBy(w => w.GetInstanceID()).ToArray(); + if (string.Equals(p.action, "list", StringComparison.OrdinalIgnoreCase)) + return Task.FromResult(new SuccessResponse("Open Editor windows.", new + { + windows = windows.Select(Describe).ToArray(), + focused_window_id = EditorWindow.focusedWindow != null + ? (int?)EditorWindow.focusedWindow.GetInstanceID() : null, + unity_version = Application.unityVersion + })); + if (!string.Equals(p.action, "screenshot", StringComparison.OrdinalIgnoreCase)) + return Failure("action must be list or screenshot."); + if (Application.isBatchMode) + return Failure("Window capture requires a visible Editor. Batch mode is not supported."); + if (EditorApplication.isCompiling || EditorApplication.isUpdating) + return Failure("The Editor is busy. Wait for compilation and asset import to finish."); + if (cancelPendingCapture != null) + return Failure("Another Editor window capture is in progress."); + if (p.max_resolution < 64 || p.max_resolution > 4096) + return Failure("max_resolution must be from 64 to 4096."); + if (!p.include_image && !p.save_file) + return Failure("Set include_image or save_file to true."); + var target = Resolve(windows, p, out string error); + if (target == null) + return Task.FromResult(new ErrorResponse(error)); + if (!p.focus && !target.hasFocus) + return Failure("With focus=false, the target must be a selected tab."); + return CaptureAfterRepaint(target, p); + } + catch (Exception ex) + { + return Failure("Window capture failed: " + ex.Message); + } + } + + internal static EditorWindow Resolve(EditorWindow[] windows, Parameters p, out string error) + { + error = null; + int selectorCount = (p.window_id.HasValue ? 1 : 0) + + (!string.IsNullOrWhiteSpace(p.window_title) ? 1 : 0) + + (!string.IsNullOrWhiteSpace(p.window_type) ? 1 : 0); + if (selectorCount > 1) + { + error = "Use only one of window_id, window_title, or window_type."; + return null; + } + var matches = windows.Where(w => w != null && (p.window_id.HasValue + ? w.GetInstanceID() == p.window_id.Value + : !string.IsNullOrWhiteSpace(p.window_title) + ? string.Equals(w.titleContent.text, p.window_title.Trim(), StringComparison.OrdinalIgnoreCase) + : !string.IsNullOrWhiteSpace(p.window_type) + ? string.Equals(w.GetType().Name, p.window_type.Trim(), StringComparison.Ordinal) + || string.Equals(w.GetType().FullName, p.window_type.Trim(), StringComparison.Ordinal) + : w == EditorWindow.focusedWindow)).ToArray(); + if (matches.Length == 1) + return matches[0]; + error = matches.Length == 0 ? "No matching open window. Call action=list to get current window IDs." + : "More than one window matches. Call action=list, then use window_id."; + return null; + } + + internal static void CancelPendingCapture() + { + cancelPendingCapture?.Invoke(); + } + + private static Task CaptureAfterRepaint(EditorWindow target, Parameters p) + { + var previous = EditorWindow.focusedWindow; + var selectedTabs = UnityEngine.Resources.FindObjectsOfTypeAll() + .Where(w => w != null && w != target && w.docked && w.hasFocus).ToArray(); + EditorWindow previousTab = null; + var completion = new TaskCompletionSource(); + double start = EditorApplication.timeSinceStartup; + int ticks = 0; + bool finished = false; + + void Finish(object result) + { + if (finished) return; + finished = true; + EditorApplication.update -= Tick; + AssemblyReloadEvents.beforeAssemblyReload -= CancelPendingCapture; + EditorApplication.quitting -= CancelPendingCapture; + cancelPendingCapture = null; + try + { + if (p.restore_focus && p.focus && (target == null || target.hasFocus)) + { + bool canRestoreKeyboardFocus = EditorWindow.focusedWindow == target; + if (previousTab != null) previousTab.ShowTab(); + if (canRestoreKeyboardFocus && previous != null && previous != target) previous.Focus(); + } + } + catch (Exception ex) + { + if (result is SuccessResponse success && success.Data is JObject data) + data["focus_restore_error"] = ex.Message; + } + finally + { + completion.TrySetResult(result); + } + } + + void Tick() + { + try + { + if (target == null) + { + Finish(new ErrorResponse("The target window closed before capture.")); + return; + } + double elapsed = EditorApplication.timeSinceStartup - start; + if (++ticks < 2 || elapsed < 0.2) return; + if (!target.hasFocus) + { + if (elapsed < 3) return; + Finish(new ErrorResponse("Unity did not select the target tab. Retry after the Editor is ready.")); + return; + } + Finish(Capture(target, p)); + } + catch (Exception ex) + { + Finish(new ErrorResponse("Window capture failed: " + ex.Message)); + } + } + + cancelPendingCapture = () => Finish(new ErrorResponse("Editor reload or shutdown interrupted capture. Retry after the Editor is ready.")); + EditorApplication.update += Tick; + AssemblyReloadEvents.beforeAssemblyReload += CancelPendingCapture; + EditorApplication.quitting += CancelPendingCapture; + try + { + if (p.focus) + { + target.ShowTab(); + previousTab = target.docked ? selectedTabs.FirstOrDefault(w => w.position == target.position) : null; + } + target.Repaint(); + EditorApplication.QueuePlayerLoopUpdate(); + } + catch (Exception ex) + { + Finish(new ErrorResponse("Window focus failed: " + ex.Message)); + } + return completion.Task; + } + + private static object Capture(EditorWindow target, Parameters p) + { + float scale = EditorWindowScreenshotUtility.GetWindowPixelsPerPoint(target); + int width = Mathf.RoundToInt(target.position.width * scale); + int height = Mathf.RoundToInt(target.position.height * scale); + if (width <= 0 || height <= 0 || (long)width * height > 16777216) + return new ErrorResponse("The target has an empty or excessive capture area."); + Texture2D full = null; + Texture2D image = null; + try + { + full = EditorWindowScreenshotUtility.CaptureWindowPixels(target, width, height); + string path = null; + if (p.save_file) + { + string folder = Path.GetFullPath(Path.Combine(Application.dataPath, "../Library/McpEditorScreenshots")); + Directory.CreateDirectory(folder); + path = Path.Combine(folder, DateTime.UtcNow.ToString("yyyyMMdd-HHmmss-fff") + + "-" + Guid.NewGuid().ToString("N") + ".png"); + File.WriteAllBytes(path, full.EncodeToPNG()); + } + var data = new JObject + { + ["window"] = JObject.FromObject(Describe(target)), + ["capture_source"] = "editor_window_buffer", + ["captured_at_utc"] = DateTime.UtcNow.ToString("O"), + ["width"] = width, ["height"] = height, + ["pixels_per_point"] = scale, ["path"] = path, + ["mimeType"] = "image/png" + }; + if (p.include_image) + { + image = Mathf.Max(full.width, full.height) > p.max_resolution + ? ScreenshotUtility.DownscaleTexture(full, p.max_resolution) : full; + data["imageBase64"] = Convert.ToBase64String(image.EncodeToPNG()); + data["imageWidth"] = image.width; + data["imageHeight"] = image.height; + } + return new SuccessResponse("Editor window captured.", data); + } + finally + { + if (image != null && image != full) Object.DestroyImmediate(image); + if (full != null) Object.DestroyImmediate(full); + } + } + + private static object Describe(EditorWindow window) + { + Rect rect = window.position; + return new + { + window_id = window.GetInstanceID(), title = window.titleContent.text, + type = window.GetType().FullName, focused = EditorWindow.focusedWindow == window, + selected_tab = window.hasFocus, docked = window.docked, + position = new { x = rect.x, y = rect.y, width = rect.width, height = rect.height } + }; + } + + private static Task Failure(string message) => Task.FromResult(new ErrorResponse(message)); + } +} diff --git a/MCPForUnity/Editor/Tools/ManageEditorWindows.cs.meta b/MCPForUnity/Editor/Tools/ManageEditorWindows.cs.meta new file mode 100644 index 000000000..784b80258 --- /dev/null +++ b/MCPForUnity/Editor/Tools/ManageEditorWindows.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 01a07b40b63e4687bd2cac1e05f4690f diff --git a/Server/src/cli/CLI_USAGE_GUIDE.md b/Server/src/cli/CLI_USAGE_GUIDE.md index a276bd15a..4be121024 100644 --- a/Server/src/cli/CLI_USAGE_GUIDE.md +++ b/Server/src/cli/CLI_USAGE_GUIDE.md @@ -501,6 +501,13 @@ unity-mcp material set-renderer-color "Cube" 1 0 0 1 ### Editor Commands +Use `editor windows` to list open Editor tabs, then +`editor screenshot --window-id ID` to save a unique full-size PNG under the +selected project's `Library/McpEditorScreenshots`. Only one of `--window-id`, +`--window-title`, or `--window-type` is accepted. Without a selector, the current +keyboard-focused window is captured. Screenshots may contain private Editor data. +The previous tab and focus are restored by default. Requires a graphical Editor. + ```bash # Play mode control unity-mcp editor play diff --git a/Server/src/cli/commands/editor.py b/Server/src/cli/commands/editor.py index 5b0dce795..fc257e2a7 100644 --- a/Server/src/cli/commands/editor.py +++ b/Server/src/cli/commands/editor.py @@ -17,6 +17,42 @@ def editor(): pass +@editor.command("windows") +@handle_unity_errors +def windows(): + """List open Editor windows, including inactive docked tabs.""" + config = get_config() + result = run_command("manage_editor_windows", {"action": "list"}, config) + click.echo(format_output(result, config.format)) + + +@editor.command("screenshot") +@click.option("--window-id", type=int, help="Current ID from editor windows.") +@click.option("--window-title", help="Exact title (case-insensitive).") +@click.option("--window-type", help="Exact short or full C# type name.") +@click.option("--focus/--no-focus", default=True, help="Select the target tab before capture.") +@click.option("--restore-focus/--no-restore-focus", default=True, help="Restore the previous tab and focus.") +@handle_unity_errors +def screenshot(window_id: Optional[int], window_title: Optional[str], window_type: Optional[str], + focus: bool, restore_focus: bool): + """Save a full-size PNG in the project's Library/McpEditorScreenshots. + + Use one selector; without one, capture the window with keyboard focus. + Screenshots can contain private Editor data. Requires a graphical Editor. + """ + selectors = {key: value for key, value in { + "window_id": window_id, "window_title": window_title, "window_type": window_type, + }.items() if value is not None and (not isinstance(value, str) or value.strip())} + if len(selectors) > 1: + raise click.UsageError("Use only one window selector.") + config = get_config() + result = run_command("manage_editor_windows", { + "action": "screenshot", **selectors, "focus": focus, "restore_focus": restore_focus, + "include_image": False, "save_file": True, + }, config) + click.echo(format_output(result, config.format)) + + @editor.command("play") @handle_unity_errors def play(): diff --git a/Server/src/services/tools/manage_editor_windows.py b/Server/src/services/tools/manage_editor_windows.py new file mode 100644 index 000000000..dc5c95f10 --- /dev/null +++ b/Server/src/services/tools/manage_editor_windows.py @@ -0,0 +1,64 @@ +from typing import Annotated, Any, Literal + +from fastmcp import Context +from fastmcp.server.server import ToolResult +from mcp.types import ToolAnnotations + +from services.registry import mcp_for_unity_tool +from services.tools import get_unity_instance_from_context +from services.tools.utils import extract_screenshot_images +from transport.legacy.unity_connection import async_send_command_with_retry +from transport.unity_transport import send_with_unity_instance + + +@mcp_for_unity_tool( + description=( + "List open Unity Editor tabs/windows or capture one as an MCP PNG image. " + "Use action=list to obtain window IDs; inactive docked tabs are included. " + "Screenshots read Unity's own window buffer, including covered windows. " + "Selecting an inactive tab temporarily changes focus; the previous tab is restored by default. " + "No mouse/keyboard control is used. Captures may contain private Editor data. " + "One image per capture is intended for the assistant; client display may vary. " + "No file is saved unless save_file=true. Requires a graphical Editor; " + "batch mode, native OS dialogs and minimized windows are unsupported." + ), + annotations=ToolAnnotations(title="Manage Editor windows", readOnlyHint=False, destructiveHint=True), +) +async def manage_editor_windows( + ctx: Context, + action: Annotated[Literal["list", "screenshot"], "List open windows or capture one selected tab."] = "list", + window_id: Annotated[int | None, "Current window ID from action=list. Use one selector only."] = None, + window_title: Annotated[str | None, "Exact title, ignoring case. Duplicate titles require window_id."] = None, + window_type: Annotated[str | None, "Exact C# type name or full name. Duplicate types require window_id."] = None, + focus: Annotated[bool, "Select the target tab before capture. False requires an already selected tab."] = True, + restore_focus: Annotated[bool, "Restore the previous tab and keyboard focus after capture."] = True, + include_image: Annotated[bool, "Return a PNG image block and separate metadata."] = True, + save_file: Annotated[bool, "Save a unique full-size PNG in Library/McpEditorScreenshots. Default false."] = False, + max_resolution: Annotated[int, "Maximum edge for the inline image, from 64 to 4096. Does not resize saved files."] = 1600, +) -> dict[str, Any] | ToolResult: + """Observe open Editor windows through the selected Unity instance.""" + if action not in ("list", "screenshot"): + return {"success": False, "message": "action must be list or screenshot."} + params: dict[str, Any] = {"action": action} + if action == "screenshot": + selectors = {key: value for key, value in { + "window_id": window_id, "window_title": window_title, "window_type": window_type, + }.items() if value is not None and (not isinstance(value, str) or value.strip())} + if len(selectors) > 1: + return {"success": False, "message": "Use only one window selector."} + if type(max_resolution) is not int or not 64 <= max_resolution <= 4096: + return {"success": False, "message": "max_resolution must be from 64 to 4096."} + if not include_image and not save_file: + return {"success": False, "message": "Set include_image or save_file to true."} + params.update(selectors) + params.update(focus=focus, restore_focus=restore_focus, include_image=include_image, + save_file=save_file, max_resolution=max_resolution) + instance = await get_unity_instance_from_context(ctx) + response = await send_with_unity_instance(async_send_command_with_retry, instance, "manage_editor_windows", params) + if not isinstance(response, dict): + return {"success": False, "message": str(response)} + if action == "screenshot": + images = extract_screenshot_images(response, image_audience=["assistant"]) + if images is not None: + return images + return response diff --git a/Server/src/services/tools/utils.py b/Server/src/services/tools/utils.py index 93c700858..0cb4f8081 100644 --- a/Server/src/services/tools/utils.py +++ b/Server/src/services/tools/utils.py @@ -4,7 +4,7 @@ import json import math -from typing import Any +from typing import Any, Literal _TRUTHY = {"true", "1", "yes", "on"} _FALSY = {"false", "0", "no", "off"} @@ -402,15 +402,22 @@ def _to_output_range(components: list[float], from_hex: bool = False) -> list: return None, f"color must be a list, dict, hex string, or JSON string, got {type(value).__name__}" -def extract_screenshot_images(response: dict[str, Any]) -> "ToolResult | None": +def extract_screenshot_images( + response: dict[str, Any], + *, + image_audience: list[Literal["user", "assistant"]] | None = None, +) -> "ToolResult | None": """If a Unity response contains inline base64 images, return a ToolResult with TextContent + ImageContent blocks. Returns None for normal text-only responses. - Shared screenshot handling (used by manage_camera). + Shared screenshot handling. image_audience is an optional client display + hint, not a privacy or visibility guarantee. """ from fastmcp.server.server import ToolResult from mcp.types import TextContent, ImageContent + image_options = {"annotations": {"audience": image_audience}} if image_audience is not None else {} + if not isinstance(response, dict) or not response.get("success"): return None @@ -439,7 +446,7 @@ def extract_screenshot_images(response: dict[str, Any]) -> "ToolResult | None": b64 = s.get("imageBase64") if b64: blocks.append(TextContent(type="text", text=f"[Angle: {s.get('angle', '?')}]")) - blocks.append(ImageContent(type="image", data=b64, mimeType="image/png")) + blocks.append(ImageContent(type="image", data=b64, mimeType="image/png", **image_options)) return ToolResult(content=blocks) # Single image (include_image or positioned capture) or contact sheet @@ -451,7 +458,7 @@ def extract_screenshot_images(response: dict[str, Any]) -> "ToolResult | None": return ToolResult( content=[ TextContent(type="text", text=json.dumps(text_result)), - ImageContent(type="image", data=image_b64, mimeType="image/png"), + ImageContent(type="image", data=image_b64, mimeType="image/png", **image_options), ], ) diff --git a/Server/tests/test_cli_editor_windows.py b/Server/tests/test_cli_editor_windows.py new file mode 100644 index 000000000..f7e67a818 --- /dev/null +++ b/Server/tests/test_cli_editor_windows.py @@ -0,0 +1,49 @@ +from unittest.mock import Mock + +import pytest +from click.testing import CliRunner + +from cli.commands.editor import editor +from cli.utils.config import CLIConfig, set_config + + +@pytest.fixture +def transport(monkeypatch): + config = CLIConfig(format="json", unity_instance="isolated-test@hash") + set_config(config) + send = Mock(return_value={"success": True, "data": {"path": "Library/McpEditorScreenshots/test.png"}}) + monkeypatch.setattr("cli.commands.editor.run_command", send) + return send, config + + +def test_list_routes_through_cli_configuration(transport): + send, config = transport + result = CliRunner().invoke(editor, ["windows"]) + assert result.exit_code == 0, result.output + send.assert_called_once_with("manage_editor_windows", {"action": "list"}, config) + + +@pytest.mark.parametrize("selector,params", [ + ([], {}), (["--window-id", "42"], {"window_id": 42}), + (["--window-title", "Inspector"], {"window_title": "Inspector"}), + (["--window-type", "UnityEditor.ConsoleWindow"], {"window_type": "UnityEditor.ConsoleWindow"}), +]) +def test_capture_saves_file_without_printing_base64(transport, selector, params): + send, config = transport + result = CliRunner().invoke(editor, ["screenshot", *selector, "--no-focus", "--no-restore-focus"]) + assert result.exit_code == 0, result.output + send.assert_called_once_with("manage_editor_windows", { + "action": "screenshot", **params, "focus": False, "restore_focus": False, + "include_image": False, "save_file": True, + }, config) + assert "Library/McpEditorScreenshots/test.png" in result.output + + +@pytest.mark.parametrize("args", [ + ["--window-id", "invalid"], ["--window-id", "42", "--window-title", "Inspector"], + ["--output-folder", "../private"], +]) +def test_invalid_cli_capture_does_not_connect(transport, args): + result = CliRunner().invoke(editor, ["screenshot", *args]) + assert result.exit_code == 2 + transport[0].assert_not_called() diff --git a/Server/tests/test_manage_editor_windows.py b/Server/tests/test_manage_editor_windows.py new file mode 100644 index 000000000..29f66b3f4 --- /dev/null +++ b/Server/tests/test_manage_editor_windows.py @@ -0,0 +1,123 @@ +import asyncio +import copy +import json +import os +from pathlib import Path +import subprocess +import sys +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest +from mcp.types import ImageContent, TextContent + +from services.tools.manage_editor_windows import manage_editor_windows + + +@pytest.fixture +def transport(monkeypatch): + target = "services.tools.manage_editor_windows" + send = AsyncMock(return_value={"success": True, "data": {"windows": []}}) + monkeypatch.setattr(target + ".get_unity_instance_from_context", AsyncMock(return_value="test-instance")) + monkeypatch.setattr(target + ".send_with_unity_instance", send) + return send + + +def call(**kwargs): + return asyncio.run(manage_editor_windows(SimpleNamespace(), **kwargs)) + + +def test_list_uses_request_instance_without_capture_side_effects(transport): + assert call()["success"] + assert transport.call_args.args[1:] == ("test-instance", "manage_editor_windows", {"action": "list"}) + + +@pytest.mark.parametrize("selector", [{"window_id": 42}, {"window_title": "Inspector"}, {"window_type": "UnityEditor.ConsoleWindow"}, {}]) +def test_capture_routes_selector_and_privacy_defaults(transport, selector): + call(action="screenshot", **selector) + params = transport.call_args.args[3] + assert all(params[key] == value for key, value in selector.items()) + assert params["save_file"] is False + assert params["include_image"] is True + assert params["restore_focus"] is True + + +@pytest.mark.parametrize("kwargs", [ + {"action": "close"}, {"action": "screenshot", "window_id": 1, "window_title": "Inspector"}, + {"action": "screenshot", "max_resolution": 63}, {"action": "screenshot", "max_resolution": 4097}, + {"action": "screenshot", "max_resolution": True}, + {"action": "screenshot", "include_image": False, "save_file": False}, +]) +def test_invalid_requests_do_not_reach_unity(transport, kwargs): + assert call(**kwargs)["success"] is False + transport.assert_not_called() + + +def test_image_bytes_appear_once_with_metadata_preserved(transport): + value = {"success": True, "message": "captured", "data": { + "imageBase64": "aW1hZ2U=", "mimeType": "image/png", "path": None, "window": {"window_id": 42}, + }} + transport.return_value = value + original = copy.deepcopy(value) + result = call(action="screenshot") + assert len(result.content) == 2 + assert isinstance(result.content[0], TextContent) + assert isinstance(result.content[1], ImageContent) + assert result.content[1].data == "aW1hZ2U=" + annotations = result.content[1].annotations + audience = annotations["audience"] if isinstance(annotations, dict) else annotations.audience + assert audience == ["assistant"] + metadata = json.loads(result.content[0].text) + assert metadata["data"]["window"]["window_id"] == 42 + assert "imageBase64" not in metadata["data"] + assert value == original + + +def test_file_only_capture_and_unity_errors_are_preserved(transport): + transport.return_value = {"success": True, "data": {"path": "Library/McpEditorScreenshots/test.png"}} + assert call(action="screenshot", include_image=False, save_file=True) == transport.return_value + transport.return_value = {"success": False, "error": "busy"} + assert call(action="screenshot") == transport.return_value + + +def test_native_mcp_protocol_returns_image_and_rejects_unknown_arguments(): + # The integration suite installs FastMCP stubs during collection. A child + # process verifies real wire types without changing that suite's environment. + script = ''' +import asyncio +from unittest.mock import AsyncMock, patch +from fastmcp import Client, FastMCP +from mcp.types import ImageContent +from services.registry.tool_registry import get_registered_tools +from services.tools.manage_editor_windows import manage_editor_windows + +async def run(): + server = FastMCP("screenshot-tests") + registration = next(t for t in get_registered_tools() if t["name"] == "manage_editor_windows") + server.tool(name=registration["name"], description=registration["description"], + **registration["kwargs"])(registration["func"]) + async with Client(server) as client: + tool = next(t for t in await client.list_tools() if t.name == "manage_editor_windows") + assert tool.annotations.readOnlyHint is False + assert tool.annotations.destructiveHint is True + result = await client.call_tool("manage_editor_windows", {"action": "screenshot", "window_id": 42}) + assert len(result.content) == 2 + assert isinstance(result.content[1], ImageContent) + assert result.content[1].annotations.audience == ["assistant"] + assert result.content[1].model_dump()["annotations"]["audience"] == ["assistant"] + assert "imageBase64" not in result.content[0].text + try: + await client.call_tool("manage_editor_windows", {"action": "screenshot", "output_folder": "../private"}) + except Exception: + pass + else: + raise AssertionError("Unknown output path argument was accepted") + +with patch("services.tools.manage_editor_windows.get_unity_instance_from_context", AsyncMock(return_value="test-instance")), \\ + patch("services.tools.manage_editor_windows.send_with_unity_instance", AsyncMock(return_value={"success": True, "data": {"imageBase64": "aW1hZ2U=", "path": None}})): + asyncio.run(run()) +''' + env = os.environ.copy() + env["PYTHONPATH"] = str(Path(__file__).resolve().parents[1] / "src") + result = subprocess.run([sys.executable, "-c", script], env=env, capture_output=True, text=True, timeout=45) + assert result.returncode == 0, result.stdout + result.stderr diff --git a/TestProjects/UnityMCPTests/Assets/Tests/EditMode/Tools/ManageEditorWindowsTests.cs b/TestProjects/UnityMCPTests/Assets/Tests/EditMode/Tools/ManageEditorWindowsTests.cs new file mode 100644 index 000000000..8c6e16fb7 --- /dev/null +++ b/TestProjects/UnityMCPTests/Assets/Tests/EditMode/Tools/ManageEditorWindowsTests.cs @@ -0,0 +1,450 @@ +using System; +using System.Collections; +using System.IO; +using System.Reflection; +using System.Threading.Tasks; +using MCPForUnity.Editor.Helpers; +using MCPForUnity.Editor.Tools; +using Newtonsoft.Json.Linq; +using NUnit.Framework; +using UnityEditor; +using UnityEngine; +using UnityEngine.TestTools; +using Object = UnityEngine.Object; + +namespace MCPForUnityTests.Editor.Tools +{ + public class ScreenshotColorWindow : EditorWindow + { + protected void OnGUI() + { + EditorGUI.DrawRect(new Rect(0, 0, position.width, position.height / 2), Color.blue); + EditorGUI.DrawRect(new Rect(0, position.height / 2, position.width, position.height / 2), Color.red); + } + } + + public class ScreenshotDockWindow : ScreenshotColorWindow { } + + public class ScreenshotDarkWindow : EditorWindow + { + private void OnGUI() => EditorGUI.DrawRect(new Rect(0, 0, position.width, position.height), + new Color(0.12f, 0.18f, 0.24f, 1)); + } + + [Category("EditorWindowScreenshots")] + public class ManageEditorWindowsTests + { + private ScreenshotColorWindow first; + private ScreenshotColorWindow second; + private ScreenshotDockWindow docked; + private EditorWindow previous; + + [SetUp] + public void SetUp() + { + previous = EditorWindow.focusedWindow; + first = ScriptableObject.CreateInstance(); + second = ScriptableObject.CreateInstance(); + first.titleContent = new GUIContent("MCP first fixture"); + second.titleContent = new GUIContent("MCP second fixture"); + } + + [TearDown] + public void TearDown() + { + ManageEditorWindows.CancelPendingCapture(); + if (first != null) Object.DestroyImmediate(first); + if (second != null) Object.DestroyImmediate(second); + if (docked != null) docked.Close(); + if (previous != null) previous.Focus(); + } + + [UnityTearDown] + public IEnumerator LeavePlayMode() + { + if (EditorApplication.isPlaying) yield return new ExitPlayMode(); + } + + [Test] + public void IdSelectsOneWindowWithDuplicateTitles() + { + second.titleContent = first.titleContent; + Assert.That(Resolve(new ManageEditorWindows.Parameters { window_id = second.GetInstanceID() }, out var error), Is.SameAs(second)); + Assert.That(error, Is.Null); + } + + [Test] + public void TitleIgnoresCaseAndOuterSpaces() + { + Assert.That(Resolve(new ManageEditorWindows.Parameters { window_title = " MCP FIRST fixture " }, out _), Is.SameAs(first)); + } + + [Test] + public void DuplicateTitlesAndTypesRequireIds() + { + second.titleContent = first.titleContent; + Assert.That(Resolve(new ManageEditorWindows.Parameters { window_title = "MCP first fixture" }, out var error), Is.Null); + Assert.That(error, Does.Contain("More than one")); + Assert.That(Resolve(new ManageEditorWindows.Parameters { window_type = typeof(ScreenshotColorWindow).FullName }, out _), Is.Null); + } + + [Test] + public void MultipleSelectorsAreRejected() + { + Assert.That(Resolve(new ManageEditorWindows.Parameters { window_id = first.GetInstanceID(), window_title = "other" }, out var error), Is.Null); + Assert.That(error, Does.Contain("only one")); + } + + [Test] + public void ClosedIdCannotSelectAnotherWindow() + { + int id = second.GetInstanceID(); + Object.DestroyImmediate(second); + Assert.That(ManageEditorWindows.Resolve(new EditorWindow[] { first }, + new ManageEditorWindows.Parameters { window_id = id }, out var error), Is.Null); + Assert.That(error, Does.Contain("No matching")); + } + + [Test] + public void InvalidActionDoesNotChangeFocus() + { + var focused = EditorWindow.focusedWindow; + Assert.That(ManageEditorWindows.HandleCommand(new JObject { ["action"] = "close" }).Result, Is.TypeOf()); + Assert.That(EditorWindow.focusedWindow, Is.SameAs(focused)); + } + + [Test] + public void BatchCaptureHasAnExplicitError() + { + if (!Application.isBatchMode) Assert.Ignore("Batch-mode guard requires a batch Editor."); + var result = (ErrorResponse)ManageEditorWindows.HandleCommand(new JObject { ["action"] = "screenshot" }).Result; + Assert.That(result.Error, Does.Contain("Batch mode")); + } + + [Test] + public void UnsupportedWindowHostFailsWithoutDesktopFallback() + { + Assert.Throws(() => EditorWindowScreenshotUtility.CaptureWindowPixels(first, 64, 64)); + } + + [TestCase(0, 64)] + [TestCase(4097, 4097)] + public void EmptyAndExcessiveBufferSizesAreRejected(int width, int height) + { + Assert.Throws(() => EditorWindowScreenshotUtility.CaptureWindowPixels(first, width, height)); + } + + [UnityTest] + public IEnumerator FloatingBufferCapturePreservesOrientationAndFocus() + { + RequireGraphics(); + yield return ShowFixtures(); + var task = Screenshot(second, 160); + yield return Await(task); + var data = ((SuccessResponse)task.Result).Data as JObject; + Assert.That(data, Is.Not.Null); + Assert.That((string)data["path"], Is.Null); + Assert.That(EditorWindow.focusedWindow, Is.SameAs(first)); + var image = new Texture2D(2, 2); + try + { + Assert.That(image.LoadImage(Convert.FromBase64String((string)data["imageBase64"])), Is.True); + Assert.That(Mathf.Max(image.width, image.height), Is.LessThanOrEqualTo(160)); + Assert.That(image.GetPixel(image.width / 2, image.height * 3 / 4).b, Is.GreaterThan(0.9)); + Assert.That(image.GetPixel(image.width / 2, image.height / 4).r, Is.GreaterThan(0.9)); + } + finally { Object.DestroyImmediate(image); } + } + + [UnityTest] + public IEnumerator FileOnlyCaptureIsUniqueAndInsideLibrary() + { + RequireGraphics(); + yield return ShowFixtures(); + var firstTask = ManageEditorWindows.HandleCommand(new JObject { + ["action"] = "screenshot", ["window_id"] = second.GetInstanceID(), + ["include_image"] = false, ["save_file"] = true + }); + yield return Await(firstTask); + var firstData = (JObject)((SuccessResponse)firstTask.Result).Data; + string firstPath = (string)firstData["path"]; + string secondPath = null; + try + { + Assert.That(firstPath, Does.StartWith(Path.GetFullPath(Path.Combine(Application.dataPath, "../Library/McpEditorScreenshots")))); + Assert.That(File.Exists(firstPath), Is.True); + Assert.That(firstData["imageBase64"], Is.Null); + var secondTask = ManageEditorWindows.HandleCommand(new JObject { + ["action"] = "screenshot", ["window_id"] = second.GetInstanceID(), + ["include_image"] = false, ["save_file"] = true + }); + yield return Await(secondTask); + secondPath = (string)((JObject)((SuccessResponse)secondTask.Result).Data)["path"]; + Assert.That(secondPath, Is.Not.EqualTo(firstPath)); + Assert.That(File.Exists(firstPath), Is.True); + } + finally + { + if (File.Exists(firstPath)) File.Delete(firstPath); + if (secondPath != null && File.Exists(secondPath)) File.Delete(secondPath); + } + } + + [UnityTest] + public IEnumerator CancellationRestoresFocusAndAllowsTheNextCapture() + { + RequireGraphics(); + yield return ShowFixtures(); + var cancelled = Screenshot(second); + var concurrent = Screenshot(second); + Assert.That(concurrent.Result, Is.TypeOf()); + ManageEditorWindows.CancelPendingCapture(); + Assert.That(cancelled.IsCompleted, Is.True); + Assert.That(((ErrorResponse)cancelled.Result).Error, Does.Contain("interrupted")); + Assert.That(EditorWindow.focusedWindow, Is.SameAs(first)); + var retry = Screenshot(second); + yield return Await(retry); + Assert.That(retry.Result, Is.TypeOf()); + } + + [UnityTest] + public IEnumerator ClosingTargetCompletesWithErrorAndReleasesTheCaptureGate() + { + RequireGraphics(); + yield return ShowFixtures(); + var task = Screenshot(second); + second.Close(); + yield return Await(task); + Assert.That(((ErrorResponse)task.Result).Error, Does.Contain("closed")); + var retry = Screenshot(first); + yield return Await(retry); + Assert.That(retry.Result, Is.TypeOf()); + } + + [UnityTest] + public IEnumerator InactiveDockedTabIsListedCapturedAndRestored() + { + RequireGraphics(); + var scene = EditorWindow.GetWindow(); + docked = EditorWindow.GetWindow("MCP dock fixture", false, typeof(SceneView)); + scene.ShowTab(); + scene.Focus(); + yield return null; + if (!docked.docked) Assert.Fail("Fixture was not docked; tab restoration was not exercised."); + Assert.That(docked.hasFocus, Is.False); + var listing = JObject.FromObject(((SuccessResponse)ManageEditorWindows.HandleCommand(new JObject { ["action"] = "list" }).Result).Data); + Assert.That(listing["windows"].ToString(), Does.Contain("MCP dock fixture")); + var noFocus = ManageEditorWindows.HandleCommand(new JObject { + ["action"] = "screenshot", ["window_id"] = docked.GetInstanceID(), ["focus"] = false + }); + Assert.That(noFocus.Result, Is.TypeOf()); + var task = Screenshot(docked); + yield return Await(task); + Assert.That(task.Result, Is.TypeOf()); + Assert.That(scene.hasFocus, Is.True); + Assert.That(EditorWindow.focusedWindow, Is.SameAs(scene)); + } + + [UnityTest] + public IEnumerator CaptureDoesNotOverrideUserFocusChanges() + { + RequireGraphics(); + yield return ShowFixtures(); + var task = Screenshot(second); + var third = ScriptableObject.CreateInstance(); + try + { + third.Show(); + third.Focus(); + yield return Await(task); + Assert.That(task.Result, Is.TypeOf()); + Assert.That(EditorWindow.focusedWindow, Is.SameAs(third)); + } + finally { third.Close(); } + } + + [UnityTest] + public IEnumerator DownscaledDarkPixelsMatchTheFullSizeCapture() + { + RequireGraphics(); + var dark = ScriptableObject.CreateInstance(); + dark.position = new Rect(450, 100, 320, 240); + dark.Show(); + var full = new Texture2D(2, 2); + var small = new Texture2D(2, 2); + try + { + yield return null; + var fullTask = Screenshot(dark, 4096); + yield return Await(fullTask); + var smallTask = Screenshot(dark, 64); + yield return Await(smallTask); + Assert.That(fullTask.Result, Is.TypeOf()); + Assert.That(smallTask.Result, Is.TypeOf()); + Assert.That(full.LoadImage(Convert.FromBase64String((string)((JObject)((SuccessResponse)fullTask.Result).Data)["imageBase64"])), Is.True); + Assert.That(small.LoadImage(Convert.FromBase64String((string)((JObject)((SuccessResponse)smallTask.Result).Data)["imageBase64"])), Is.True); + Assert.That(Mathf.Max(small.width, small.height), Is.EqualTo(64)); + Color before = full.GetPixel(full.width / 2, full.height / 2); + Color after = small.GetPixel(small.width / 2, small.height / 2); + Assert.That(before.r, Is.GreaterThan(0.02f)); + Assert.That(after.r, Is.EqualTo(before.r).Within(0.02f)); + Assert.That(after.g, Is.EqualTo(before.g).Within(0.02f)); + Assert.That(after.b, Is.EqualTo(before.b).Within(0.02f)); + } + finally + { + dark.Close(); + Object.DestroyImmediate(full); + Object.DestroyImmediate(small); + } + } + + [UnityTest] + public IEnumerator GameViewBufferCaptureInPlayModeDoesNotStepOrPauseThePlayer() + { + RequireGraphics(); + yield return new EnterPlayMode(); + var gameType = typeof(EditorWindow).Assembly.GetType("UnityEditor.GameView", true); + var game = EditorWindow.GetWindow(gameType); + game.ShowTab(); + yield return null; + int before = Time.frameCount; + for (int i = 0; i < 3; i++) + { + var task = Screenshot(game, 512); + yield return Await(task); + Assert.That(task.Result, Is.TypeOf()); + Assert.That(EditorApplication.isPlaying, Is.True); + Assert.That(EditorApplication.isPaused, Is.False); + } + Assert.That(Time.frameCount, Is.GreaterThan(before)); + LogAssert.NoUnexpectedReceived(); + yield return new ExitPlayMode(); + } + + [UnityTest] + public IEnumerator AssemblyReloadCancelsThePendingCaptureAndAllowsRetry() + { + RequireGraphics(); + yield return ShowFixtures(); + var task = Screenshot(second); + const string resultKey = "MCP.ScreenshotTests.ReloadResult"; + const string focusKey = "MCP.ScreenshotTests.ReloadFocus"; + SessionState.EraseString(resultKey); + SessionState.EraseBool(focusKey); + void ObserveCancellation() + { + SessionState.SetString(resultKey, task.IsCompleted && task.Result is ErrorResponse error ? error.Error : "not cancelled"); + SessionState.SetBool(focusKey, EditorWindow.focusedWindow == first); + } + // Production's callback was subscribed first. Observe its result + // before domain state is discarded, then verify persisted evidence. + AssemblyReloadEvents.beforeAssemblyReload += ObserveCancellation; + EditorUtility.RequestScriptReload(); + yield return new WaitForDomainReload(); + Assert.That(SessionState.GetString(resultKey, "missing"), Does.Contain("interrupted")); + Assert.That(SessionState.GetBool(focusKey, false), Is.True); + SessionState.EraseString(resultKey); + SessionState.EraseBool(focusKey); + var target = EditorWindow.GetWindow(); + var retry = Screenshot(target); + yield return Await(retry); + Assert.That(retry.Result, Is.TypeOf()); + } + + [UnityTest] + public IEnumerator MinimizedWindowCompletesAndSubsequentCaptureRecovers() + { + RequireGraphics(); + string title = "MCP minimize fixture " + Guid.NewGuid().ToString("N"); + second.titleContent = new GUIContent(title); + yield return ShowFixtures(); +#if UNITY_EDITOR_WIN + // Minimize only this uniquely named fixture, after verifying its + // process ID. Confirm the native minimized state on every version. + IntPtr handle = IntPtr.Zero; + uint thisProcess = (uint)System.Diagnostics.Process.GetCurrentProcess().Id; + EnumWindows((candidate, _) => { + GetWindowThreadProcessId(candidate, out uint candidateOwner); + if (candidateOwner != thisProcess) return true; + var caption = new System.Text.StringBuilder(512); + GetWindowText(candidate, caption, caption.Capacity); + if (!caption.ToString().Contains(title)) return true; + handle = candidate; + return false; + }, IntPtr.Zero); + Assert.That(handle, Is.Not.EqualTo(IntPtr.Zero), "Fixture native window was not found."); + GetWindowThreadProcessId(handle, out uint owner); + Assert.That(owner, Is.EqualTo((uint)System.Diagnostics.Process.GetCurrentProcess().Id)); + ShowWindow(handle, 6); + Assert.That(IsIconic(handle), Is.True, "Fixture was not minimized."); +#else + var host = typeof(EditorWindow).GetField("m_Parent", BindingFlags.Instance | BindingFlags.NonPublic)?.GetValue(second); + var container = host?.GetType().GetProperty("window", BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic)?.GetValue(host); + var minimize = container?.GetType().GetMethod("Minimize", BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); + if (minimize == null) Assert.Ignore("This Editor has no available minimize API on this platform."); + minimize.Invoke(container, null); +#endif + first.Focus(); + yield return null; + var task = ManageEditorWindows.HandleCommand(new JObject { + ["action"] = "screenshot", ["window_id"] = second.GetInstanceID(), ["focus"] = false + }); + yield return Await(task); + Assert.That(task.Result, Is.AssignableTo()); + Debug.Log("[EditorWindowScreenshots] Unsupported minimized capture: " + + (task.Result is ErrorResponse error ? error.Error : "buffer returned; freshness is not guaranteed")); + second.Close(); + var retry = Screenshot(first); + yield return Await(retry); + Assert.That(retry.Result, Is.TypeOf()); + } + +#if UNITY_EDITOR_WIN + private delegate bool WindowCallback(IntPtr handle, IntPtr state); + [System.Runtime.InteropServices.DllImport("user32.dll")] + private static extern bool EnumWindows(WindowCallback callback, IntPtr state); + [System.Runtime.InteropServices.DllImport("user32.dll", CharSet = System.Runtime.InteropServices.CharSet.Unicode)] + private static extern int GetWindowText(IntPtr handle, System.Text.StringBuilder text, int length); + [System.Runtime.InteropServices.DllImport("user32.dll")] + private static extern uint GetWindowThreadProcessId(IntPtr handle, out uint processId); + [System.Runtime.InteropServices.DllImport("user32.dll")] + private static extern bool ShowWindow(IntPtr handle, int command); + [System.Runtime.InteropServices.DllImport("user32.dll")] + private static extern bool IsIconic(IntPtr handle); +#endif + + private IEnumerator ShowFixtures() + { + first.position = new Rect(100, 100, 320, 240); + second.position = new Rect(450, 100, 320, 240); + first.ShowUtility(); + second.ShowUtility(); + Assert.That(first.docked, Is.False); + Assert.That(second.docked, Is.False); + first.Focus(); + yield return null; + yield return null; + } + + private static void RequireGraphics() + { + if (Application.isBatchMode || SystemInfo.graphicsDeviceType == UnityEngine.Rendering.GraphicsDeviceType.Null) + Assert.Ignore("Pixel capture requires a graphical Editor; run this fixture without -batchmode/-nographics."); + } + + private static Task Screenshot(EditorWindow window, int resolution = 1600) => + ManageEditorWindows.HandleCommand(new JObject { ["action"] = "screenshot", ["window_id"] = window.GetInstanceID(), ["max_resolution"] = resolution }); + + private static IEnumerator Await(Task task) + { + double deadline = EditorApplication.timeSinceStartup + 10; + while (!task.IsCompleted && EditorApplication.timeSinceStartup < deadline) yield return null; + Assert.That(task.IsCompleted, Is.True, "Capture did not complete before the test deadline."); + Assert.That(task.IsFaulted, Is.False, task.Exception?.ToString()); + } + + private EditorWindow Resolve(ManageEditorWindows.Parameters p, out string error) => + ManageEditorWindows.Resolve(new EditorWindow[] { first, second }, p, out error); + } +} diff --git a/TestProjects/UnityMCPTests/Assets/Tests/EditMode/Tools/ManageEditorWindowsTests.cs.meta b/TestProjects/UnityMCPTests/Assets/Tests/EditMode/Tools/ManageEditorWindowsTests.cs.meta new file mode 100644 index 000000000..13ad87f7f --- /dev/null +++ b/TestProjects/UnityMCPTests/Assets/Tests/EditMode/Tools/ManageEditorWindowsTests.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 4ea8ea45fd1b43b3bfadf71e8024ac03 diff --git a/manifest.json b/manifest.json index 8a242f74f..578b17ebc 100644 --- a/manifest.json +++ b/manifest.json @@ -97,6 +97,10 @@ "name": "manage_editor", "description": "Control Unity Editor state, play mode, preferences, undo/redo" }, + { + "name": "manage_editor_windows", + "description": "List and capture open Unity Editor tabs and windows" + }, { "name": "manage_gameobject", "description": "Create, modify, transform, and delete GameObjects" diff --git a/website/docs/guides/editor-window-screenshots.md b/website/docs/guides/editor-window-screenshots.md new file mode 100644 index 000000000..3f43a8660 --- /dev/null +++ b/website/docs/guides/editor-window-screenshots.md @@ -0,0 +1,95 @@ +--- +title: Editor window screenshots +--- + +# Editor window screenshots + +`manage_editor_windows` lets an agent inspect an open Unity Editor tab or +window without mouse or keyboard control. It captures Unity's own image +buffer, so another desktop window can cover Unity. Use `manage_camera` +for camera renders and Scene View viewport screenshots. + +Select the Unity instance for the request, then list its windows: + +```json +{"action": "list"} +``` + +The result includes IDs, titles, C# types, positions, keyboard focus and +selected tabs. Inactive docked tabs are included. IDs change when a window +closes or the Editor restarts; get a new list when an ID fails. + +Capture a window with its current ID: + +```json +{"action": "screenshot", "window_id": 12345, "max_resolution": 1600} +``` + +You can use an exact `window_title` (case-insensitive) or `window_type` +instead. Use one selector only. Duplicate titles/types require an ID. +Without a selector, capture uses the window with keyboard focus. + +The response has an MCP PNG image block and separate metadata. Image bytes +are not repeated in the text. The previous docked tab and keyboard focus +are restored by default. `focus=false` requires an already selected tab; +`restore_focus=false` leaves the target tab selected. Capture does not +replace a later user focus change. + +## Model images and client display + +Each capture returns one PNG image in the MCP tool result so the model can +inspect its pixels. It uses `annotations.audience=["assistant"]` to identify +the intended audience. This is an optional client hint, not a guarantee that +Claude, Codex or another host hides the image. The host controls thumbnails, +attachments, storage and transcript display. The tool does not create or +upload a separate user-facing artifact. + +Image bytes stay in the image block; they are not repeated in text or emitted +as a gallery. Files are saved only when requested. Returning a resource link +instead would require the client to fetch and include the image, and would +still leave display to that client. A path-only result does not give the model +pixels to inspect. + +See the MCP [content annotations](https://modelcontextprotocol.io/specification/2025-11-25/schema#annotations) +and [resource interaction model](https://modelcontextprotocol.io/specification/2025-11-25/server/resources#user-interaction-model). + +## Files and private data + +No screenshot file is saved by default. `save_file=true` saves a unique, +full-size PNG in `Library/McpEditorScreenshots`; `max_resolution` only +limits the inline image. Use `include_image=false, save_file=true` for a +file-only result. Output paths are not supplied by the caller. + +Screenshots can contain private source code, asset names, file paths, +Console messages or credentials displayed in Editor windows. The image is +sent to the selected MCP client. Enable and use the tool only with clients +that may access that data. Saved PNGs have no automatic retention cleanup; +delete them when no longer needed and exclude them from public PR evidence. +Existing instance routing, tool visibility and remote authentication apply. + +## Capture limits + +Capture requires a graphical Editor. Batch mode, minimized windows and +native operating-system dialogs are unsupported. Only open `EditorWindow` +objects are listed. Captures contain the selected tab area and exclude +operating-system borders. An inactive tab must be selected to repaint it. +Another capture is rejected until the pending capture finishes. +Minimized windows may return an old buffer; restore them before capture. + +The capture helper uses Unity's internal `GUIView.GrabPixels` API, also +used by Scene View capture. Missing internal APIs return explicit errors; +capture does not fall back to desktop pixels. Capture waits for Editor +updates without `EditorApplication.Step` or a synchronous player-loop pump. +Reload, shutdown or a closed target ends the pending capture. Retry after +the Editor is ready. Capture does not save scenes or change Play Mode state. + +## CLI + +```shell +unity-mcp --instance MyProject@hash editor windows +unity-mcp --instance MyProject@hash editor screenshot --window-id 12345 +``` + +The CLI saves a PNG and returns its metadata. The MCP tool returns the image +directly through the existing server; no extra adapter or client entry is +required. diff --git a/website/docs/reference/cli.md b/website/docs/reference/cli.md index 7bcc1af69..279576c65 100644 --- a/website/docs/reference/cli.md +++ b/website/docs/reference/cli.md @@ -78,6 +78,21 @@ The CLI mirrors the MCP tool catalog. Each command group wraps one or more `mana ## Discovering subcommands and flags +### Editor window screenshots + +`editor windows` lists IDs, titles and types of open Editor tabs. `editor screenshot` +saves a unique full-size PNG in the selected project's `Library/McpEditorScreenshots` +and prints metadata. Use one of `--window-id`, `--window-title`, or `--window-type`; +without a selector, it captures the window with keyboard focus. The previous tab +and focus are restored unless `--no-restore-focus` is supplied. `--no-focus` +requires an already selected tab. Screenshots can contain private Editor data. +See [Editor window screenshots](/guides/editor-window-screenshots) for limits. + +```bash +unity-mcp --instance MyProject@hash editor windows +unity-mcp --instance MyProject@hash editor screenshot --window-id 12345 +``` + Every group supports `--help`: ```bash diff --git a/website/docs/reference/tools/core/index.md b/website/docs/reference/tools/core/index.md index f0075dee8..396274853 100644 --- a/website/docs/reference/tools/core/index.md +++ b/website/docs/reference/tools/core/index.md @@ -23,6 +23,7 @@ Essential scene, script, asset & editor tools (always on by default) - **[`manage_camera`](./manage_camera.md)** — Manage cameras (Unity Camera + Cinemachine). - **[`manage_components`](./manage_components.md)** — Add, remove, or set properties on components attached to GameObjects. - **[`manage_editor`](./manage_editor.md)** — Controls and queries the Unity editor's state and settings. +- **[`manage_editor_windows`](./manage_editor_windows.md)** — List open Unity Editor tabs/windows or capture one as an MCP PNG image. - **[`manage_gameobject`](./manage_gameobject.md)** — Performs CRUD operations on GameObjects. - **[`manage_graphics`](./manage_graphics.md)** — Manage rendering graphics: volumes, post-processing, light baking, rendering stats, pipeline settings, and URP renderer features. - **[`manage_material`](./manage_material.md)** — Manages Unity materials (set properties, colors, shaders, etc). diff --git a/website/docs/reference/tools/core/manage_editor_windows.md b/website/docs/reference/tools/core/manage_editor_windows.md new file mode 100644 index 000000000..d33fe544e --- /dev/null +++ b/website/docs/reference/tools/core/manage_editor_windows.md @@ -0,0 +1,40 @@ +--- +title: manage_editor_windows +sidebar_label: manage_editor_windows +description: "List open Unity Editor tabs/windows or capture one as an MCP PNG image." +--- + +# `manage_editor_windows` + +> **Auto-generated** from the Python tool registry. Do not hand-edit outside `` blocks — the generator (`tools/generate_docs_reference.py`) will overwrite them. + +**Group:** `core`  ·  **Module:** `services.tools.manage_editor_windows` + +## Description + +List open Unity Editor tabs/windows or capture one as an MCP PNG image. Use action=list to obtain window IDs; inactive docked tabs are included. Screenshots read Unity's own window buffer, including covered windows. Selecting an inactive tab temporarily changes focus; the previous tab is restored by default. No mouse/keyboard control is used. Captures may contain private Editor data. One image per capture is intended for the assistant; client display may vary. No file is saved unless save_file=true. Requires a graphical Editor; batch mode, native OS dialogs and minimized windows are unsupported. + +## Parameters + +| Name | Type | Required | Description | +|------|------|----------|-------------| +| `action` | `Literal['list', 'screenshot']` | — | List open windows or capture one selected tab. | +| `window_id` | `int \| None` | — | Current window ID from action=list. Use one selector only. | +| `window_title` | `str \| None` | — | Exact title, ignoring case. Duplicate titles require window_id. | +| `window_type` | `str \| None` | — | Exact C# type name or full name. Duplicate types require window_id. | +| `focus` | `bool` | — | Select the target tab before capture. False requires an already selected tab. | +| `restore_focus` | `bool` | — | Restore the previous tab and keyboard focus after capture. | +| `include_image` | `bool` | — | Return a PNG image block and separate metadata. | +| `save_file` | `bool` | — | Save a unique full-size PNG in Library/McpEditorScreenshots. Default false. | +| `max_resolution` | `int` | — | Maximum edge for the inline image, from 64 to 4096. Does not resize saved files. | + +## Returns + +A `dict` containing the Unity response. The exact shape depends on the action. + +## Examples + + +*No examples yet. Add usage examples here — they will be preserved across regenerations.* + + diff --git a/website/docs/reference/tools/index.md b/website/docs/reference/tools/index.md index a7466a134..35c3c0c82 100644 --- a/website/docs/reference/tools/index.md +++ b/website/docs/reference/tools/index.md @@ -24,7 +24,7 @@ AI asset generation – 3D model gen/import, 2D image gen & audio gen (bring-you - **[`import_model`](./asset_gen/import_model.md)** — Import 3D models from the Sketchfab marketplace into the Unity project. - **[`import_model_file`](./asset_gen/import_model_file.md)** — Import a local 3D model file that already exists on disk (e.g. an FBX/OBJ/glTF exported from Blender or another DCC tool) into the Unity project. -## `core`   (30 tools) +## `core`   (31 tools) Essential scene, script, asset & editor tools (always on by default) - **[`apply_text_edits`](./core/apply_text_edits.md)** — Apply small text edits to a C# script identified by URI. - **[`batch_execute`](./core/batch_execute.md)** — Executes multiple MCP commands in a single batch for dramatically better performance. @@ -41,6 +41,7 @@ Essential scene, script, asset & editor tools (always on by default) - **[`manage_camera`](./core/manage_camera.md)** — Manage cameras (Unity Camera + Cinemachine). - **[`manage_components`](./core/manage_components.md)** — Add, remove, or set properties on components attached to GameObjects. - **[`manage_editor`](./core/manage_editor.md)** — Controls and queries the Unity editor's state and settings. +- **[`manage_editor_windows`](./core/manage_editor_windows.md)** — List open Unity Editor tabs/windows or capture one as an MCP PNG image. - **[`manage_gameobject`](./core/manage_gameobject.md)** — Performs CRUD operations on GameObjects. - **[`manage_graphics`](./core/manage_graphics.md)** — Manage rendering graphics: volumes, post-processing, light baking, rendering stats, pipeline settings, and URP renderer features. - **[`manage_material`](./core/manage_material.md)** — Manages Unity materials (set properties, colors, shaders, etc). diff --git a/website/sidebars.js b/website/sidebars.js index 843fab471..fb3b09809 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -27,6 +27,7 @@ const sidebars = { 'guides/claude-code-cli', 'guides/client-configurators', 'guides/multi-instance', + 'guides/editor-window-screenshots', 'guides/tool-groups', 'guides/cli', 'guides/cli-examples', From 3c1ee1695247b4d08e7f68000cb955f8e46fbe75 Mon Sep 17 00:00:00 2001 From: Anton Guljajev Date: Thu, 1 Oct 2026 11:36:06 +0900 Subject: [PATCH 2/4] fix: preserve screenshot selectors, tabs and capture orientation --- .../Helpers/EditorWindowScreenshotUtility.cs | 21 +- .../Editor/Tools/ManageEditorWindows.cs | 42 +++- .../services/tools/manage_editor_windows.py | 7 +- Server/tests/test_manage_editor_windows.py | 6 +- Server/tests/test_tool_annotations.py | 3 + .../Tools/ManageEditorWindowsTests.cs | 230 +++++++++++++++++- .../docs/guides/editor-window-screenshots.md | 18 +- 7 files changed, 296 insertions(+), 31 deletions(-) diff --git a/MCPForUnity/Editor/Helpers/EditorWindowScreenshotUtility.cs b/MCPForUnity/Editor/Helpers/EditorWindowScreenshotUtility.cs index 4913d511e..4c795f48f 100644 --- a/MCPForUnity/Editor/Helpers/EditorWindowScreenshotUtility.cs +++ b/MCPForUnity/Editor/Helpers/EditorWindowScreenshotUtility.cs @@ -179,11 +179,14 @@ private static Rect GetViewportLocalRectPoints(SceneView sceneView, float pixels internal static float GetWindowPixelsPerPoint(EditorWindow window) { - // IPanel.scaledPixelsPerPoint is not public in the oldest supported Editors. - var panel = window.rootVisualElement.panel; - var property = panel?.GetType().GetProperty("scaledPixelsPerPoint", + // The native backing scale measures physical pixels. UI Toolkit's + // scaledPixelsPerPoint also includes panel zoom, which must not resize + // a capture of the whole Editor window. + var host = GetHostView(window); + var method = host?.GetType().GetMethod("GetBackingScaleFactor", BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); - if (property?.GetValue(panel, null) is float scale && scale > 0 && !float.IsInfinity(scale)) + if (method?.Invoke(host, null) is float scale && scale > 0 + && !float.IsNaN(scale) && !float.IsInfinity(scale)) return scale; return EditorGUIUtility.pixelsPerPoint; } @@ -194,10 +197,12 @@ internal static Texture2D CaptureWindowPixels(EditorWindow window, int width, in if (width <= 0 || height <= 0 || (long)width * height > 16777216) throw new ArgumentOutOfRangeException(nameof(width), "Empty or excessive capture area."); InvokeMethodIfExists(GetHostView(window), "RepaintImmediately"); - return CaptureViewRect(window, new Rect(0, 0, width, height)); + // GrabPixels uses physical pixels. A point-sized source rectangle can + // crop fractional-DPI buffers. Readback orientation depends on the GPU API. + return CaptureViewRect(window, new Rect(0, 0, width, height), SystemInfo.graphicsUVStartsAtTop); } - private static Texture2D CaptureViewRect(EditorWindow window, Rect viewportRectPixels) + private static Texture2D CaptureViewRect(EditorWindow window, Rect viewportRectPixels, bool flipVertically = true) { object hostView = GetHostView(window); if (hostView == null) @@ -238,7 +243,7 @@ private static Texture2D CaptureViewRect(EditorWindow window, Rect viewportRectP texture = new Texture2D(width, height, TextureFormat.RGBA32, false); texture.ReadPixels(new Rect(0, 0, width, height), 0, 0); texture.Apply(); - FlipTextureVertically(texture); + if (flipVertically) FlipTextureVertically(texture); var result = texture; texture = null; return result; @@ -260,7 +265,7 @@ private static Texture2D CaptureViewRect(EditorWindow window, Rect viewportRectP } } - private static object GetHostView(EditorWindow window) + internal static object GetHostView(EditorWindow window) { if (window == null) return null; diff --git a/MCPForUnity/Editor/Tools/ManageEditorWindows.cs b/MCPForUnity/Editor/Tools/ManageEditorWindows.cs index aa5f4f993..47580518d 100644 --- a/MCPForUnity/Editor/Tools/ManageEditorWindows.cs +++ b/MCPForUnity/Editor/Tools/ManageEditorWindows.cs @@ -33,7 +33,7 @@ public static Task HandleCommand(JObject args) { try { - var p = args?.ToObject() ?? new Parameters(); + var p = ParseParameters(args); var windows = UnityEngine.Resources.FindObjectsOfTypeAll() .Where(w => w != null && (w.docked || w.hasFocus)) .OrderBy(w => w.titleContent.text, StringComparer.OrdinalIgnoreCase) @@ -71,6 +71,33 @@ public static Task HandleCommand(JObject args) } } + internal static Parameters ParseParameters(JObject args) + { + var values = new ToolParams(args ?? new JObject()); + var p = new Parameters + { + action = values.Get("action", "list"), + window_id = values.GetInt("window_id"), + window_title = values.Get("window_title"), + window_type = values.Get("window_type"), + focus = values.GetBool("focus", true), + restore_focus = values.GetBool("restore_focus", true), + include_image = values.GetBool("include_image", true), + save_file = values.GetBool("save_file", false), + max_resolution = values.GetInt("max_resolution", 1600).Value + }; + // An explicit invalid selector must never become a focused-window request. + if (values.Has("window_id") && !p.window_id.HasValue) + throw new ArgumentException("window_id must be an integer."); + foreach (string key in new[] { "window_title", "window_type" }) + if (values.Has(key) && (values.GetRaw(key).Type != JTokenType.String + || string.IsNullOrWhiteSpace(values.Get(key)))) + throw new ArgumentException(key + " must be a non-empty string."); + if (values.Has("max_resolution") && !values.GetInt("max_resolution").HasValue) + throw new ArgumentException("max_resolution must be an integer from 64 to 4096."); + return p; + } + internal static EditorWindow Resolve(EditorWindow[] windows, Parameters p, out string error) { error = null; @@ -107,7 +134,10 @@ private static Task CaptureAfterRepaint(EditorWindow target, Parameters var previous = EditorWindow.focusedWindow; var selectedTabs = UnityEngine.Resources.FindObjectsOfTypeAll() .Where(w => w != null && w != target && w.docked && w.hasFocus).ToArray(); - EditorWindow previousTab = null; + var targetHost = EditorWindowScreenshotUtility.GetHostView(target); + EditorWindow previousTab = target.docked && targetHost != null + ? selectedTabs.FirstOrDefault(w => ReferenceEquals( + EditorWindowScreenshotUtility.GetHostView(w), targetHost)) : null; var completion = new TaskCompletionSource(); double start = EditorApplication.timeSinceStartup; int ticks = 0; @@ -175,7 +205,6 @@ void Tick() if (p.focus) { target.ShowTab(); - previousTab = target.docked ? selectedTabs.FirstOrDefault(w => w.position == target.position) : null; } target.Repaint(); EditorApplication.QueuePlayerLoopUpdate(); @@ -200,13 +229,15 @@ private static object Capture(EditorWindow target, Parameters p) { full = EditorWindowScreenshotUtility.CaptureWindowPixels(target, width, height); string path = null; + byte[] fullPng = null; if (p.save_file) { string folder = Path.GetFullPath(Path.Combine(Application.dataPath, "../Library/McpEditorScreenshots")); Directory.CreateDirectory(folder); path = Path.Combine(folder, DateTime.UtcNow.ToString("yyyyMMdd-HHmmss-fff") + "-" + Guid.NewGuid().ToString("N") + ".png"); - File.WriteAllBytes(path, full.EncodeToPNG()); + fullPng = full.EncodeToPNG(); + File.WriteAllBytes(path, fullPng); } var data = new JObject { @@ -221,7 +252,8 @@ private static object Capture(EditorWindow target, Parameters p) { image = Mathf.Max(full.width, full.height) > p.max_resolution ? ScreenshotUtility.DownscaleTexture(full, p.max_resolution) : full; - data["imageBase64"] = Convert.ToBase64String(image.EncodeToPNG()); + data["imageBase64"] = Convert.ToBase64String(image == full + ? fullPng ?? full.EncodeToPNG() : image.EncodeToPNG()); data["imageWidth"] = image.width; data["imageHeight"] = image.height; } diff --git a/Server/src/services/tools/manage_editor_windows.py b/Server/src/services/tools/manage_editor_windows.py index dc5c95f10..65ef844e8 100644 --- a/Server/src/services/tools/manage_editor_windows.py +++ b/Server/src/services/tools/manage_editor_windows.py @@ -22,7 +22,7 @@ "No file is saved unless save_file=true. Requires a graphical Editor; " "batch mode, native OS dialogs and minimized windows are unsupported." ), - annotations=ToolAnnotations(title="Manage Editor windows", readOnlyHint=False, destructiveHint=True), + annotations=ToolAnnotations(title="Manage Editor windows", readOnlyHint=False, destructiveHint=False), ) async def manage_editor_windows( ctx: Context, @@ -41,6 +41,11 @@ async def manage_editor_windows( return {"success": False, "message": "action must be list or screenshot."} params: dict[str, Any] = {"action": action} if action == "screenshot": + if window_id is not None and type(window_id) is not int: + return {"success": False, "message": "window_id must be an integer."} + for name, value in {"window_title": window_title, "window_type": window_type}.items(): + if value is not None and (not isinstance(value, str) or not value.strip()): + return {"success": False, "message": f"{name} must be a non-empty string."} selectors = {key: value for key, value in { "window_id": window_id, "window_title": window_title, "window_type": window_type, }.items() if value is not None and (not isinstance(value, str) or value.strip())} diff --git a/Server/tests/test_manage_editor_windows.py b/Server/tests/test_manage_editor_windows.py index 29f66b3f4..88bf0948a 100644 --- a/Server/tests/test_manage_editor_windows.py +++ b/Server/tests/test_manage_editor_windows.py @@ -46,6 +46,10 @@ def test_capture_routes_selector_and_privacy_defaults(transport, selector): {"action": "close"}, {"action": "screenshot", "window_id": 1, "window_title": "Inspector"}, {"action": "screenshot", "max_resolution": 63}, {"action": "screenshot", "max_resolution": 4097}, {"action": "screenshot", "max_resolution": True}, + {"action": "screenshot", "window_id": True}, + {"action": "screenshot", "window_id": "not-an-id"}, + {"action": "screenshot", "window_title": " "}, + {"action": "screenshot", "window_type": ""}, {"action": "screenshot", "include_image": False, "save_file": False}, ]) def test_invalid_requests_do_not_reach_unity(transport, kwargs): @@ -99,7 +103,7 @@ async def run(): async with Client(server) as client: tool = next(t for t in await client.list_tools() if t.name == "manage_editor_windows") assert tool.annotations.readOnlyHint is False - assert tool.annotations.destructiveHint is True + assert tool.annotations.destructiveHint is False result = await client.call_tool("manage_editor_windows", {"action": "screenshot", "window_id": 42}) assert len(result.content) == 2 assert isinstance(result.content[1], ImageContent) diff --git a/Server/tests/test_tool_annotations.py b/Server/tests/test_tool_annotations.py index 933259ce4..6f5c25d98 100644 --- a/Server/tests/test_tool_annotations.py +++ b/Server/tests/test_tool_annotations.py @@ -48,6 +48,9 @@ # 'clear' empties the ephemeral Editor console buffer; Unity still mirrors # every entry to the Editor log file on disk. "read_console", + # Selects/restores tabs; optional files are unique PNGs in Library, never + # project assets or caller-specified paths. No existing data is overwritten. + "manage_editor_windows", # Session-local routing only. "set_active_instance", # Toggles which tools are visible to this session. diff --git a/TestProjects/UnityMCPTests/Assets/Tests/EditMode/Tools/ManageEditorWindowsTests.cs b/TestProjects/UnityMCPTests/Assets/Tests/EditMode/Tools/ManageEditorWindowsTests.cs index 8c6e16fb7..5e9e2971f 100644 --- a/TestProjects/UnityMCPTests/Assets/Tests/EditMode/Tools/ManageEditorWindowsTests.cs +++ b/TestProjects/UnityMCPTests/Assets/Tests/EditMode/Tools/ManageEditorWindowsTests.cs @@ -18,8 +18,12 @@ public class ScreenshotColorWindow : EditorWindow { protected void OnGUI() { - EditorGUI.DrawRect(new Rect(0, 0, position.width, position.height / 2), Color.blue); - EditorGUI.DrawRect(new Rect(0, position.height / 2, position.width, position.height / 2), Color.red); + float halfWidth = position.width / 2; + float halfHeight = position.height / 2; + EditorGUI.DrawRect(new Rect(0, 0, halfWidth, halfHeight), Color.blue); + EditorGUI.DrawRect(new Rect(halfWidth, 0, halfWidth, halfHeight), Color.green); + EditorGUI.DrawRect(new Rect(0, halfHeight, halfWidth, halfHeight), Color.red); + EditorGUI.DrawRect(new Rect(halfWidth, halfHeight, halfWidth, halfHeight), new Color(1, 1, 0, 1)); } } @@ -134,6 +138,107 @@ public void EmptyAndExcessiveBufferSizesAreRejected(int width, int height) Assert.Throws(() => EditorWindowScreenshotUtility.CaptureWindowPixels(first, width, height)); } + [TestCase(false)] + [TestCase(true)] + public void ParameterAccessorsPreserveEveryOptionAndDefault(bool camelCase) + { + string Key(string snake, string camel) => camelCase ? camel : snake; + var args = new JObject { + ["action"] = "screenshot", [Key("window_id", "windowId")] = second.GetInstanceID(), + [Key("window_title", "windowTitle")] = "Inspector", + [Key("window_type", "windowType")] = "UnityEditor.InspectorWindow", + ["focus"] = false, [Key("restore_focus", "restoreFocus")] = false, + [Key("include_image", "includeImage")] = false, + [Key("save_file", "saveFile")] = true, [Key("max_resolution", "maxResolution")] = 128 + }; + var p = ManageEditorWindows.ParseParameters(args); + Assert.That(p.action, Is.EqualTo("screenshot")); + Assert.That(p.window_id, Is.EqualTo(second.GetInstanceID())); + Assert.That(p.window_title, Is.EqualTo("Inspector")); + Assert.That(p.window_type, Is.EqualTo("UnityEditor.InspectorWindow")); + Assert.That(p.focus || p.restore_focus || p.include_image, Is.False); + Assert.That(p.save_file, Is.True); + Assert.That(p.max_resolution, Is.EqualTo(128)); + p = ManageEditorWindows.ParseParameters(null); + Assert.That(p.action, Is.EqualTo("list")); + Assert.That(p.window_id, Is.Null); + Assert.That(p.focus && p.restore_focus && p.include_image, Is.True); + Assert.That(p.save_file, Is.False); + Assert.That(p.max_resolution, Is.EqualTo(1600)); + } + + [TestCase("window_id", "not-an-id")] + [TestCase("windowId", "not-an-id")] + [TestCase("windowId", null)] + [TestCase("windowTitle", " ")] + [TestCase("window_type", "")] + [TestCase("windowType", null)] + [TestCase("maxResolution", "bad")] + public void InvalidExplicitParametersNeverBecomeFocusedWindowRequests(string key, string value) + { + var args = new JObject { ["action"] = "screenshot", + [key] = value == null ? JValue.CreateNull() : new JValue(value) }; + var focused = EditorWindow.focusedWindow; + var result = ManageEditorWindows.HandleCommand(args).Result; + Assert.That(result, Is.TypeOf()); + Assert.That(((ErrorResponse)result).Error, Does.Contain("must be")); + Assert.That(EditorWindow.focusedWindow, Is.SameAs(focused)); + } + + [UnityTest] + public IEnumerator SavedAndInlineFullSizePngBytesMatch() + { + RequireGraphics(); + yield return ShowFixtures(); + var task = ManageEditorWindows.HandleCommand(new JObject { + ["action"] = "screenshot", ["windowId"] = second.GetInstanceID(), + ["saveFile"] = true, ["includeImage"] = true, ["maxResolution"] = 4096 + }); + yield return Await(task); + Assert.That(task.Result, Is.TypeOf()); + var data = (JObject)((SuccessResponse)task.Result).Data; + string saved = (string)data["path"]; + try + { + Assert.That(File.ReadAllBytes(saved), Is.EqualTo(Convert.FromBase64String((string)data["imageBase64"]))); + Assert.That((int)data["imageWidth"], Is.EqualTo((int)data["width"])); + Assert.That((int)data["imageHeight"], Is.EqualTo((int)data["height"])); + } + finally { if (File.Exists(saved)) File.Delete(saved); } + } + + [UnityTest] + public IEnumerator DockedBufferPreservesFourCornerColorsAndPhysicalDimensions() + { + RequireGraphics(); + var scene = EditorWindow.GetWindow(); + docked = EditorWindow.GetWindow("MCP corner dock fixture", false, typeof(SceneView)); + docked.ShowTab(); + yield return null; + yield return null; + Assert.That(docked.docked, Is.True); + var task = Screenshot(docked, 4096); + yield return Await(task); + Assert.That(task.Result, Is.TypeOf()); + var data = (JObject)((SuccessResponse)task.Result).Data; + var image = new Texture2D(2, 2); + try + { + Assert.That(image.LoadImage(Convert.FromBase64String((string)data["imageBase64"])), Is.True); + float scale = (float)data["pixels_per_point"]; + Assert.That(image.width, Is.EqualTo(Mathf.RoundToInt(docked.position.width * scale))); + Assert.That(image.height, Is.EqualTo(Mathf.RoundToInt(docked.position.height * scale))); + Debug.Log("[EditorWindowScreenshots] Capture dimensions=" + image.width + "x" + image.height + + " source_points=" + data["window"]["position"] + " backing_scale=" + data["pixels_per_point"] + + " graphics=" + SystemInfo.graphicsDeviceType + " uv_top=" + SystemInfo.graphicsUVStartsAtTop + + " public_capture_api_available=" + (typeof(UnityEditorInternal.InternalEditorUtility).GetMethod("CaptureEditorWindow", + BindingFlags.Static | BindingFlags.Public, null, new[] { typeof(EditorWindow), typeof(RenderTexture) }, null) != null)); + AssertCornerColors(image); + Debug.Log("[EditorWindowScreenshots] Corner capture pixels_per_point=" + scale); + } + finally { Object.DestroyImmediate(image); scene.ShowTab(); } + } + [UnityTest] public IEnumerator FloatingBufferCapturePreservesOrientationAndFocus() { @@ -150,12 +255,108 @@ public IEnumerator FloatingBufferCapturePreservesOrientationAndFocus() { Assert.That(image.LoadImage(Convert.FromBase64String((string)data["imageBase64"])), Is.True); Assert.That(Mathf.Max(image.width, image.height), Is.LessThanOrEqualTo(160)); - Assert.That(image.GetPixel(image.width / 2, image.height * 3 / 4).b, Is.GreaterThan(0.9)); - Assert.That(image.GetPixel(image.width / 2, image.height / 4).r, Is.GreaterThan(0.9)); + Debug.Log("[EditorWindowScreenshots] Capture dimensions=" + image.width + "x" + image.height + + " source_points=" + data["window"]["position"] + " backing_scale=" + data["pixels_per_point"] + + " graphics=" + SystemInfo.graphicsDeviceType + " uv_top=" + SystemInfo.graphicsUVStartsAtTop + + " public_capture_api_available=" + (typeof(UnityEditorInternal.InternalEditorUtility).GetMethod("CaptureEditorWindow", + BindingFlags.Static | BindingFlags.Public, null, new[] { typeof(EditorWindow), typeof(RenderTexture) }, null) != null)); + AssertCornerColors(image); } finally { Object.DestroyImmediate(image); } } + [UnityTest] + public IEnumerator CamelCaseRequestSelectsTheUnfocusedWindow() + { + RequireGraphics(); + yield return ShowFixtures(); + var task = ManageEditorWindows.HandleCommand(new JObject { + ["action"] = "screenshot", ["windowId"] = second.GetInstanceID(), + ["maxResolution"] = 128, ["restoreFocus"] = true + }); + yield return Await(task); + Assert.That(task.Result, Is.TypeOf()); + var data = (JObject)((SuccessResponse)task.Result).Data; + Assert.That((int)data["window"]["window_id"], Is.EqualTo(second.GetInstanceID())); + Assert.That(Math.Max((int)data["imageWidth"], (int)data["imageHeight"]), Is.EqualTo(128)); + Assert.That(EditorWindow.focusedWindow, Is.SameAs(first)); + } + + [UnityTest] + public IEnumerator BatchRouteSelectsTheRequestedUnfocusedWindow() + { + RequireGraphics(); + yield return ShowFixtures(); + var task = BatchExecute.HandleCommand(new JObject { + ["commands"] = new JArray(new JObject { + ["tool"] = "manage_editor_windows", ["params"] = new JObject { + ["action"] = "screenshot", ["window_id"] = second.GetInstanceID(), + ["max_resolution"] = 128 + } + }) + }); + yield return Await(task); + Assert.That(task.Result, Is.TypeOf()); + var data = JObject.FromObject(((SuccessResponse)task.Result).Data); + Assert.That((int)data["results"][0]["result"]["data"]["window"]["window_id"], + Is.EqualTo(second.GetInstanceID())); + Assert.That(EditorWindow.focusedWindow, Is.SameAs(first)); + } + + [UnityTest] + public IEnumerator InvalidBatchSelectorFailsWithoutCapturingTheFocusedWindow() + { + RequireGraphics(); + yield return ShowFixtures(); + var task = BatchExecute.HandleCommand(new JObject { + ["commands"] = new JArray(new JObject { + ["tool"] = "manage_editor_windows", ["params"] = new JObject { + ["action"] = "screenshot", ["window_title"] = " " + } + }) + }); + yield return Await(task); + Assert.That(task.Result, Is.TypeOf()); + var data = JObject.FromObject(((ErrorResponse)task.Result).Data); + Assert.That((bool)data["results"][0]["callSucceeded"], Is.False); + Assert.That(data["results"][0]["result"]["data"], Is.Null); + Assert.That(EditorWindow.focusedWindow, Is.SameAs(first)); + } + + [UnityTest] + public IEnumerator ResizedDockRestoresItsSelectedTabWhenAnotherWindowHadKeyboardFocus() + { + RequireGraphics(); + yield return ShowFixtures(); + var scene = EditorWindow.GetWindow(); + docked = EditorWindow.GetWindow("MCP resized dock fixture", false, typeof(SceneView)); + docked.ShowTab(); + yield return null; + yield return null; + scene.ShowTab(); + yield return null; + var parentField = typeof(EditorWindow).GetField("m_Parent", BindingFlags.Instance | BindingFlags.NonPublic); + var host = parentField.GetValue(scene); + Assert.That(parentField.GetValue(docked), Is.SameAs(host)); + var positionProperty = host.GetType().GetProperty("position", BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); + Rect original = (Rect)positionProperty.GetValue(host); + try + { + positionProperty.SetValue(host, new Rect(original.x, original.y, original.width + 73, original.height + 41)); + scene.Repaint(); + yield return null; + yield return null; + Assert.That(docked.position, Is.Not.EqualTo(scene.position), "Background-tab rectangle must be stale to exercise the regression."); + first.Focus(); + var task = Screenshot(docked); + yield return Await(task); + Assert.That(task.Result, Is.TypeOf()); + Assert.That(scene.hasFocus, Is.True, "The previous selected tab must be restored independently of keyboard focus."); + Assert.That(EditorWindow.focusedWindow, Is.SameAs(first)); + } + finally { if (host != null) positionProperty.SetValue(host, original); } + } + [UnityTest] public IEnumerator FileOnlyCaptureIsUniqueAndInsideLibrary() { @@ -427,6 +628,26 @@ private IEnumerator ShowFixtures() yield return null; } + private static void AssertCornerColors(Texture2D image) + { + Debug.Log("[EditorWindowScreenshots] Corners TL=" + image.GetPixel(image.width / 10, image.height * 9 / 10) + + " TR=" + image.GetPixel(image.width * 9 / 10, image.height * 9 / 10) + + " BL=" + image.GetPixel(image.width / 10, image.height / 10) + + " BR=" + image.GetPixel(image.width * 9 / 10, image.height / 10)); + // Sample near all four edges, leaving room for the dock tab strip. + AssertColor(image.GetPixel(image.width / 10, image.height * 9 / 10), Color.blue, "top left"); + AssertColor(image.GetPixel(image.width * 9 / 10, image.height * 9 / 10), Color.green, "top right"); + AssertColor(image.GetPixel(image.width / 10, image.height / 10), Color.red, "bottom left"); + AssertColor(image.GetPixel(image.width * 9 / 10, image.height / 10), new Color(1, 1, 0, 1), "bottom right"); + } + + private static void AssertColor(Color actual, Color expected, string corner) + { + Assert.That(actual.r, Is.EqualTo(expected.r).Within(0.04f), corner + " red"); + Assert.That(actual.g, Is.EqualTo(expected.g).Within(0.04f), corner + " green"); + Assert.That(actual.b, Is.EqualTo(expected.b).Within(0.04f), corner + " blue"); + } + private static void RequireGraphics() { if (Application.isBatchMode || SystemInfo.graphicsDeviceType == UnityEngine.Rendering.GraphicsDeviceType.Null) @@ -442,6 +663,7 @@ private static IEnumerator Await(Task task) while (!task.IsCompleted && EditorApplication.timeSinceStartup < deadline) yield return null; Assert.That(task.IsCompleted, Is.True, "Capture did not complete before the test deadline."); Assert.That(task.IsFaulted, Is.False, task.Exception?.ToString()); + if (task.Result is ErrorResponse error) Debug.Log("[EditorWindowScreenshots] Capture result: " + error.Error); } private EditorWindow Resolve(ManageEditorWindows.Parameters p, out string error) => diff --git a/website/docs/guides/editor-window-screenshots.md b/website/docs/guides/editor-window-screenshots.md index 3f43a8660..f313be0b0 100644 --- a/website/docs/guides/editor-window-screenshots.md +++ b/website/docs/guides/editor-window-screenshots.md @@ -44,15 +44,6 @@ Claude, Codex or another host hides the image. The host controls thumbnails, attachments, storage and transcript display. The tool does not create or upload a separate user-facing artifact. -Image bytes stay in the image block; they are not repeated in text or emitted -as a gallery. Files are saved only when requested. Returning a resource link -instead would require the client to fetch and include the image, and would -still leave display to that client. A path-only result does not give the model -pixels to inspect. - -See the MCP [content annotations](https://modelcontextprotocol.io/specification/2025-11-25/schema#annotations) -and [resource interaction model](https://modelcontextprotocol.io/specification/2025-11-25/server/resources#user-interaction-model). - ## Files and private data No screenshot file is saved by default. `save_file=true` saves a unique, @@ -66,6 +57,8 @@ sent to the selected MCP client. Enable and use the tool only with clients that may access that data. Saved PNGs have no automatic retention cleanup; delete them when no longer needed and exclude them from public PR evidence. Existing instance routing, tool visibility and remote authentication apply. +The tool declares `readOnlyHint=false` because it can select tabs and save PNGs, +and `destructiveHint=false` because it does not overwrite assets or existing files. ## Capture limits @@ -76,9 +69,10 @@ operating-system borders. An inactive tab must be selected to repaint it. Another capture is rejected until the pending capture finishes. Minimized windows may return an old buffer; restore them before capture. -The capture helper uses Unity's internal `GUIView.GrabPixels` API, also -used by Scene View capture. Missing internal APIs return explicit errors; -capture does not fall back to desktop pixels. Capture waits for Editor +The helper reads Unity's internal `GUIView.GrabPixels` buffer in physical +pixels, using the window's native backing scale and graphics-API-dependent +orientation correction. Missing APIs return explicit errors; capture does not +fall back to desktop pixels. Capture waits for Editor updates without `EditorApplication.Step` or a synchronous player-loop pump. Reload, shutdown or a closed target ends the pending capture. Retry after the Editor is ready. Capture does not save scenes or change Play Mode state. From 6e005a10fbf91848a338ce13998d38ee69eadbb8 Mon Sep 17 00:00:00 2001 From: Anton Guljajev Date: Thu, 1 Oct 2026 12:44:44 +0900 Subject: [PATCH 3/4] fix: align screenshot content edges and Scene View orientation --- .../Helpers/EditorWindowScreenshotUtility.cs | 21 +- .../Tools/ManageEditorWindowsTests.cs | 196 +++++++++++++++--- .../docs/guides/editor-window-screenshots.md | 4 +- 3 files changed, 190 insertions(+), 31 deletions(-) diff --git a/MCPForUnity/Editor/Helpers/EditorWindowScreenshotUtility.cs b/MCPForUnity/Editor/Helpers/EditorWindowScreenshotUtility.cs index 4c795f48f..84cb32a09 100644 --- a/MCPForUnity/Editor/Helpers/EditorWindowScreenshotUtility.cs +++ b/MCPForUnity/Editor/Helpers/EditorWindowScreenshotUtility.cs @@ -142,7 +142,7 @@ private static void FocusAndRepaint(SceneView sceneView) private static Rect GetSceneViewViewportPixelRect(SceneView sceneView) { - float pixelsPerPoint = EditorGUIUtility.pixelsPerPoint; + float pixelsPerPoint = GetWindowPixelsPerPoint(sceneView); Rect viewportLocalPoints = GetViewportLocalRectPoints(sceneView, pixelsPerPoint); if (viewportLocalPoints.width <= 0f || viewportLocalPoints.height <= 0f) throw new InvalidOperationException("Failed to resolve Scene view viewport rect."); @@ -199,15 +199,28 @@ internal static Texture2D CaptureWindowPixels(EditorWindow window, int width, in InvokeMethodIfExists(GetHostView(window), "RepaintImmediately"); // GrabPixels uses physical pixels. A point-sized source rectangle can // crop fractional-DPI buffers. Readback orientation depends on the GPU API. - return CaptureViewRect(window, new Rect(0, 0, width, height), SystemInfo.graphicsUVStartsAtTop); + return CaptureViewRect(window, new Rect(0, 0, width, height)); } - private static Texture2D CaptureViewRect(EditorWindow window, Rect viewportRectPixels, bool flipVertically = true) + private static Texture2D CaptureViewRect(EditorWindow window, Rect viewportRectPixels) { object hostView = GetHostView(window); if (hostView == null) throw new InvalidOperationException("Failed to resolve Editor window host view."); + // GrabPixels reads the host buffer, whose dock tabs and borders are + // outside EditorWindow.position. Resolve content margins from the + // host rather than assuming the content begins at buffer (0, 0). + PropertyInfo borderProperty = hostView.GetType().GetProperty("borderSize", + BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); + if (!(borderProperty?.GetValue(hostView) is RectOffset borders)) + throw new MissingMemberException($"{hostView.GetType().FullName}.borderSize"); + float scale = GetWindowPixelsPerPoint(window); + viewportRectPixels.x += Mathf.Round(borders.left * scale); + // The native pixel rectangle starts at the bottom of the host buffer; + // readback is normalized separately below for the active graphics API. + viewportRectPixels.y += Mathf.Round(borders.bottom * scale); + // GrabPixels is an internal extern on GUIView (parent of HostView), present since at least Unity 2021.1. // See: UnityCsReference/Editor/Mono/GUIView.bindings.cs — `internal extern void GrabPixels(RenderTexture, Rect)` // If Unity removes this, the MissingMethodException below keeps the failure explicit. @@ -243,7 +256,7 @@ private static Texture2D CaptureViewRect(EditorWindow window, Rect viewportRectP texture = new Texture2D(width, height, TextureFormat.RGBA32, false); texture.ReadPixels(new Rect(0, 0, width, height), 0, 0); texture.Apply(); - if (flipVertically) FlipTextureVertically(texture); + if (SystemInfo.graphicsUVStartsAtTop) FlipTextureVertically(texture); var result = texture; texture = null; return result; diff --git a/TestProjects/UnityMCPTests/Assets/Tests/EditMode/Tools/ManageEditorWindowsTests.cs b/TestProjects/UnityMCPTests/Assets/Tests/EditMode/Tools/ManageEditorWindowsTests.cs index 5e9e2971f..093d2e22e 100644 --- a/TestProjects/UnityMCPTests/Assets/Tests/EditMode/Tools/ManageEditorWindowsTests.cs +++ b/TestProjects/UnityMCPTests/Assets/Tests/EditMode/Tools/ManageEditorWindowsTests.cs @@ -16,6 +16,7 @@ namespace MCPForUnityTests.Editor.Tools { public class ScreenshotColorWindow : EditorWindow { + public bool paintEdgeMarkers; protected void OnGUI() { float halfWidth = position.width / 2; @@ -24,6 +25,12 @@ protected void OnGUI() EditorGUI.DrawRect(new Rect(halfWidth, 0, halfWidth, halfHeight), Color.green); EditorGUI.DrawRect(new Rect(0, halfHeight, halfWidth, halfHeight), Color.red); EditorGUI.DrawRect(new Rect(halfWidth, halfHeight, halfWidth, halfHeight), new Color(1, 1, 0, 1)); + if (paintEdgeMarkers) + { + float edge = 4 / EditorGUIUtility.pixelsPerPoint; + EditorGUI.DrawRect(new Rect(edge, edge, position.width - 2 * edge, + position.height - 2 * edge), Color.black); + } } } @@ -213,6 +220,7 @@ public IEnumerator DockedBufferPreservesFourCornerColorsAndPhysicalDimensions() RequireGraphics(); var scene = EditorWindow.GetWindow(); docked = EditorWindow.GetWindow("MCP corner dock fixture", false, typeof(SceneView)); + docked.paintEdgeMarkers = true; docked.ShowTab(); yield return null; yield return null; @@ -226,15 +234,10 @@ public IEnumerator DockedBufferPreservesFourCornerColorsAndPhysicalDimensions() { Assert.That(image.LoadImage(Convert.FromBase64String((string)data["imageBase64"])), Is.True); float scale = (float)data["pixels_per_point"]; + TestContext.CurrentContext.Test.Properties.Set("capture_backing_scale", scale); Assert.That(image.width, Is.EqualTo(Mathf.RoundToInt(docked.position.width * scale))); Assert.That(image.height, Is.EqualTo(Mathf.RoundToInt(docked.position.height * scale))); - Debug.Log("[EditorWindowScreenshots] Capture dimensions=" + image.width + "x" + image.height - + " source_points=" + data["window"]["position"] + " backing_scale=" + data["pixels_per_point"] - + " graphics=" + SystemInfo.graphicsDeviceType + " uv_top=" + SystemInfo.graphicsUVStartsAtTop - + " public_capture_api_available=" + (typeof(UnityEditorInternal.InternalEditorUtility).GetMethod("CaptureEditorWindow", - BindingFlags.Static | BindingFlags.Public, null, new[] { typeof(EditorWindow), typeof(RenderTexture) }, null) != null)); - AssertCornerColors(image); - Debug.Log("[EditorWindowScreenshots] Corner capture pixels_per_point=" + scale); + AssertContentEdgeMarkers(image); } finally { Object.DestroyImmediate(image); scene.ShowTab(); } } @@ -255,16 +258,133 @@ public IEnumerator FloatingBufferCapturePreservesOrientationAndFocus() { Assert.That(image.LoadImage(Convert.FromBase64String((string)data["imageBase64"])), Is.True); Assert.That(Mathf.Max(image.width, image.height), Is.LessThanOrEqualTo(160)); - Debug.Log("[EditorWindowScreenshots] Capture dimensions=" + image.width + "x" + image.height - + " source_points=" + data["window"]["position"] + " backing_scale=" + data["pixels_per_point"] - + " graphics=" + SystemInfo.graphicsDeviceType + " uv_top=" + SystemInfo.graphicsUVStartsAtTop - + " public_capture_api_available=" + (typeof(UnityEditorInternal.InternalEditorUtility).GetMethod("CaptureEditorWindow", - BindingFlags.Static | BindingFlags.Public, null, new[] { typeof(EditorWindow), typeof(RenderTexture) }, null) != null)); AssertCornerColors(image); } finally { Object.DestroyImmediate(image); } } + [UnityTest] + public IEnumerator FloatingFullSizeBufferMatchesContentEdges() + { + RequireGraphics(); + second.paintEdgeMarkers = true; + yield return ShowFixtures(); + var task = Screenshot(second, 4096); + yield return Await(task); + Assert.That(task.Result, Is.TypeOf()); + var data = (JObject)((SuccessResponse)task.Result).Data; + var image = new Texture2D(2, 2); + try + { + Assert.That(image.LoadImage(Convert.FromBase64String((string)data["imageBase64"])), Is.True); + float scale = (float)data["pixels_per_point"]; + TestContext.CurrentContext.Test.Properties.Set("capture_backing_scale", scale); + Assert.That(image.width, Is.EqualTo(Mathf.RoundToInt(second.position.width * scale))); + Assert.That(image.height, Is.EqualTo(Mathf.RoundToInt(second.position.height * scale))); + AssertContentEdgeMarkers(image); + } + finally { Object.DestroyImmediate(image); } + } + + [UnityTest] + public IEnumerator SceneViewViewportPreservesContentEdgesAndOrientation() => + CaptureSceneViewFixture(false); + + [UnityTest] + public IEnumerator DockedSceneViewViewportPreservesContentEdgesAndOrientation() => + CaptureSceneViewFixture(true); + + private static IEnumerator CaptureSceneViewFixture(bool dockedScene) + { + RequireGraphics(); + var scene = dockedScene ? EditorWindow.GetWindow() : ScriptableObject.CreateInstance(); + bool previousGizmos = scene.drawGizmos; + if (!dockedScene) + { + scene.titleContent = new GUIContent("MCP Scene viewport fixture"); + scene.position = new Rect(200, 150, 400, 300); + } + scene.drawGizmos = false; + // Controls are outside this fixture's painted viewport. Disable + // overlays only on the owned Scene View so they cannot cover markers. + var canvas = typeof(SceneView).GetProperty("overlayCanvas", + BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic)?.GetValue(scene); + var overlaysProperty = canvas?.GetType().GetProperty("overlaysEnabled", + BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); + object previousOverlays = overlaysProperty?.GetValue(canvas); + void SetOverlays(bool enabled) + { + if (canvas == null) return; + if (overlaysProperty?.CanWrite == true) overlaysProperty.SetValue(canvas, enabled); + else + { + var setter = canvas.GetType().GetMethod("SetOverlaysEnabled", + BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); + Assert.That(setter, Is.Not.Null, "The Scene View fixture cannot hide its own overlays."); + setter.Invoke(canvas, new object[] { enabled }); + } + } + var image = new Texture2D(2, 2); + string folder = Path.GetFullPath(Path.Combine(Application.dataPath, + "../Library/McpSceneViewportTests", Guid.NewGuid().ToString("N"))); + int repaints = 0; + void PaintViewport(SceneView view) + { + if (view != scene || Event.current.type != EventType.Repaint) return; + var viewport = (Rect)typeof(SceneView).GetProperty("cameraViewport", + BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic).GetValue(scene); + Handles.BeginGUI(); + try + { + float halfWidth = viewport.width / 2; + float halfHeight = viewport.height / 2; + EditorGUI.DrawRect(new Rect(0, 0, halfWidth, halfHeight), Color.blue); + EditorGUI.DrawRect(new Rect(halfWidth, 0, halfWidth, halfHeight), Color.green); + EditorGUI.DrawRect(new Rect(0, halfHeight, halfWidth, halfHeight), Color.red); + EditorGUI.DrawRect(new Rect(halfWidth, halfHeight, halfWidth, halfHeight), new Color(1, 1, 0, 1)); + float edge = 4 / EditorGUIUtility.pixelsPerPoint; + EditorGUI.DrawRect(new Rect(edge, edge, viewport.width - 2 * edge, + viewport.height - 2 * edge), Color.black); + repaints++; + } + finally { Handles.EndGUI(); } + } + SceneView.duringSceneGui += PaintViewport; + try + { + if (dockedScene) scene.ShowTab(); else scene.ShowUtility(); + Assert.That(scene.docked, Is.EqualTo(dockedScene)); + SetOverlays(false); + scene.Focus(); + double deadline = EditorApplication.timeSinceStartup + 5; + while (repaints < 2 && EditorApplication.timeSinceStartup < deadline) + { + scene.Repaint(); + yield return null; + } + Assert.That(repaints, Is.GreaterThanOrEqualTo(2), "Scene View fixture never painted its viewport."); + var result = EditorWindowScreenshotUtility.CaptureSceneViewViewportToProject(scene, + "viewport.png", 1, true, true, 4096, out int width, out int height, folder); + Assert.That(image.LoadImage(Convert.FromBase64String(result.ImageBase64)), Is.True); + TestContext.CurrentContext.Test.Properties.Set("capture_backing_scale", + EditorWindowScreenshotUtility.GetWindowPixelsPerPoint(scene)); + Assert.That(image.width, Is.EqualTo(width)); + Assert.That(image.height, Is.EqualTo(height)); + AssertContentEdgeMarkers(image); + } + finally + { + SceneView.duringSceneGui -= PaintViewport; + scene.drawGizmos = previousGizmos; + if (previousOverlays is bool enabled) SetOverlays(enabled); + if (!dockedScene) scene.Close(); + Object.DestroyImmediate(image); + string file = Path.Combine(folder, "viewport.png"); + if (File.Exists(file)) File.Delete(file); + if (Directory.Exists(folder)) Directory.Delete(folder); + } + } + [UnityTest] public IEnumerator CamelCaseRequestSelectsTheUnfocusedWindow() { @@ -593,8 +713,6 @@ public IEnumerator MinimizedWindowCompletesAndSubsequentCaptureRecovers() }); yield return Await(task); Assert.That(task.Result, Is.AssignableTo()); - Debug.Log("[EditorWindowScreenshots] Unsupported minimized capture: " + - (task.Result is ErrorResponse error ? error.Error : "buffer returned; freshness is not guaranteed")); second.Close(); var retry = Screenshot(first); yield return Await(retry); @@ -623,22 +741,51 @@ private IEnumerator ShowFixtures() second.ShowUtility(); Assert.That(first.docked, Is.False); Assert.That(second.docked, Is.False); - first.Focus(); + // Let both native utility-window creations finish before focusing. + // A fixed two-tick wait after Focus can leave delayed activation queued. yield return null; yield return null; + first.Focus(); + double deadline = EditorApplication.timeSinceStartup + 5; + while (EditorWindow.focusedWindow != first && EditorApplication.timeSinceStartup < deadline) + yield return null; + Assert.That(EditorWindow.focusedWindow, Is.SameAs(first), + "The fixture must establish keyboard focus before requesting capture."); } private static void AssertCornerColors(Texture2D image) { - Debug.Log("[EditorWindowScreenshots] Corners TL=" + image.GetPixel(image.width / 10, image.height * 9 / 10) - + " TR=" + image.GetPixel(image.width * 9 / 10, image.height * 9 / 10) - + " BL=" + image.GetPixel(image.width / 10, image.height / 10) - + " BR=" + image.GetPixel(image.width * 9 / 10, image.height / 10)); - // Sample near all four edges, leaving room for the dock tab strip. - AssertColor(image.GetPixel(image.width / 10, image.height * 9 / 10), Color.blue, "top left"); - AssertColor(image.GetPixel(image.width * 9 / 10, image.height * 9 / 10), Color.green, "top right"); - AssertColor(image.GetPixel(image.width / 10, image.height / 10), Color.red, "bottom left"); - AssertColor(image.GetPixel(image.width * 9 / 10, image.height / 10), new Color(1, 1, 0, 1), "bottom right"); + // Coordinates are actual output pixels, not a percentage of the buffer. + // The fixture paints from content (0, 0) to position.size: a tab strip, + // host border or vertical flip must fail even on a large docked pane. + TestContext.CurrentContext.Test.Properties.Set("capture_graphics_api", SystemInfo.graphicsDeviceType.ToString()); + TestContext.CurrentContext.Test.Properties.Set("capture_uv_starts_at_top", SystemInfo.graphicsUVStartsAtTop); + const int inset = 2; + AssertPixel(image, inset, image.height - 1 - inset, Color.blue, "top left content edge"); + AssertPixel(image, image.width - 1 - inset, image.height - 1 - inset, Color.green, "top right content edge"); + AssertPixel(image, inset, inset, Color.red, "bottom left content edge"); + AssertPixel(image, image.width - 1 - inset, inset, new Color(1, 1, 0, 1), "bottom right content edge"); + } + + private static void AssertContentEdgeMarkers(Texture2D image) + { + AssertCornerColors(image); + // Four physical pixels of colored edge surround a black interior. + // Also sample just inside it: a shifted/cropped rectangle cannot pass + // merely because a large quadrant still has the expected color. + const int inner = 6; + AssertPixel(image, inner, image.height - 1 - inner, Color.black, "top left interior"); + AssertPixel(image, image.width - 1 - inner, image.height - 1 - inner, Color.black, "top right interior"); + AssertPixel(image, inner, inner, Color.black, "bottom left interior"); + AssertPixel(image, image.width - 1 - inner, inner, Color.black, "bottom right interior"); + } + + private static void AssertPixel(Texture2D image, int x, int y, Color expected, string edge) + { + Color actual = image.GetPixel(x, y); + AssertColor(actual, expected, $"{edge} at ({x}, {y}) in {image.width}x{image.height}; " + + $"actual={actual}, expected={expected}, graphics={SystemInfo.graphicsDeviceType}, " + + $"uv_top={SystemInfo.graphicsUVStartsAtTop}"); } private static void AssertColor(Color actual, Color expected, string corner) @@ -663,7 +810,6 @@ private static IEnumerator Await(Task task) while (!task.IsCompleted && EditorApplication.timeSinceStartup < deadline) yield return null; Assert.That(task.IsCompleted, Is.True, "Capture did not complete before the test deadline."); Assert.That(task.IsFaulted, Is.False, task.Exception?.ToString()); - if (task.Result is ErrorResponse error) Debug.Log("[EditorWindowScreenshots] Capture result: " + error.Error); } private EditorWindow Resolve(ManageEditorWindows.Parameters p, out string error) => diff --git a/website/docs/guides/editor-window-screenshots.md b/website/docs/guides/editor-window-screenshots.md index f313be0b0..67034ceb6 100644 --- a/website/docs/guides/editor-window-screenshots.md +++ b/website/docs/guides/editor-window-screenshots.md @@ -64,8 +64,8 @@ and `destructiveHint=false` because it does not overwrite assets or existing fil Capture requires a graphical Editor. Batch mode, minimized windows and native operating-system dialogs are unsupported. Only open `EditorWindow` -objects are listed. Captures contain the selected tab area and exclude -operating-system borders. An inactive tab must be selected to repaint it. +objects are listed. Captures contain the tab content and exclude the dock tab +strip, host borders and operating-system borders. An inactive tab must be selected to repaint it. Another capture is rejected until the pending capture finishes. Minimized windows may return an old buffer; restore them before capture. From 51f11ee0a59f26486d469180c280d3d1c533b5b7 Mon Sep 17 00:00:00 2001 From: Anton Guljajev Date: Fri, 2 Oct 2026 10:41:41 +0900 Subject: [PATCH 4/4] fix: report screenshot persistence outcomes on failures --- .../Helpers/EditorWindowScreenshotUtility.cs | 4 + .../Editor/Tools/ManageEditorWindows.cs | 70 ++++++++- .../services/tools/manage_editor_windows.py | 20 ++- Server/tests/test_cli_editor_windows.py | 4 + Server/tests/test_manage_editor_windows.py | 33 ++++ .../Tools/ManageEditorWindowsTests.cs | 147 ++++++++++++++++++ .../docs/guides/editor-window-screenshots.md | 7 + 7 files changed, 276 insertions(+), 9 deletions(-) diff --git a/MCPForUnity/Editor/Helpers/EditorWindowScreenshotUtility.cs b/MCPForUnity/Editor/Helpers/EditorWindowScreenshotUtility.cs index 84cb32a09..8c7789784 100644 --- a/MCPForUnity/Editor/Helpers/EditorWindowScreenshotUtility.cs +++ b/MCPForUnity/Editor/Helpers/EditorWindowScreenshotUtility.cs @@ -177,6 +177,7 @@ private static Rect GetViewportLocalRectPoints(SceneView sceneView, float pixels Mathf.Min(windowRect.height, viewportHeight)); } + /// Uses native backing scale rather than panel zoom, falling back to the current Editor scale. internal static float GetWindowPixelsPerPoint(EditorWindow window) { // The native backing scale measures physical pixels. UI Toolkit's @@ -191,6 +192,7 @@ internal static float GetWindowPixelsPerPoint(EditorWindow window) return EditorGUIUtility.pixelsPerPoint; } + /// Reads a bounded window content rectangle in physical pixels without a desktop-capture fallback. internal static Texture2D CaptureWindowPixels(EditorWindow window, int width, int height) { if (window == null) throw new ArgumentNullException(nameof(window)); @@ -202,6 +204,7 @@ internal static Texture2D CaptureWindowPixels(EditorWindow window, int width, in return CaptureViewRect(window, new Rect(0, 0, width, height)); } + /// Offsets for host borders, normalizes GPU orientation, and restores render-target state on every exit. private static Texture2D CaptureViewRect(EditorWindow window, Rect viewportRectPixels) { object hostView = GetHostView(window); @@ -278,6 +281,7 @@ private static Texture2D CaptureViewRect(EditorWindow window, Rect viewportRectP } } + /// Resolves the native host identity used for buffer capture and dock-tab restoration across supported Editors. internal static object GetHostView(EditorWindow window) { if (window == null) diff --git a/MCPForUnity/Editor/Tools/ManageEditorWindows.cs b/MCPForUnity/Editor/Tools/ManageEditorWindows.cs index 47580518d..5bb09de6c 100644 --- a/MCPForUnity/Editor/Tools/ManageEditorWindows.cs +++ b/MCPForUnity/Editor/Tools/ManageEditorWindows.cs @@ -29,6 +29,7 @@ internal sealed class Parameters public int max_resolution = 1600; } + /// Lists eligible windows or schedules one validated capture; errors never select a replacement target. public static Task HandleCommand(JObject args) { try @@ -71,6 +72,7 @@ public static Task HandleCommand(JObject args) } } + /// Reads direct and batch-route aliases, retaining defaults while rejecting invalid explicit selectors. internal static Parameters ParseParameters(JObject args) { var values = new ToolParams(args ?? new JObject()); @@ -98,6 +100,7 @@ internal static Parameters ParseParameters(JObject args) return p; } + /// Requires one unambiguous selector; only an omitted selector may use the currently focused window. internal static EditorWindow Resolve(EditorWindow[] windows, Parameters p, out string error) { error = null; @@ -124,11 +127,13 @@ internal static EditorWindow Resolve(EditorWindow[] windows, Parameters p, out s return null; } + /// Completes the current request through its normal cleanup path during reload, shutdown, or cancellation. internal static void CancelPendingCapture() { cancelPendingCapture?.Invoke(); } + /// Waits for the selected tab to repaint and restores its former host selection without overriding later user focus. private static Task CaptureAfterRepaint(EditorWindow target, Parameters p) { var previous = EditorWindow.focusedWindow; @@ -143,6 +148,7 @@ private static Task CaptureAfterRepaint(EditorWindow target, Parameters int ticks = 0; bool finished = false; + /// Completes once, detaches lifecycle callbacks, releases the capture gate, and conditionally restores focus. void Finish(object result) { if (finished) return; @@ -171,6 +177,7 @@ void Finish(object result) } } + /// Bounds tab-selection waiting and handles a closed target before synchronous pixel capture. void Tick() { try @@ -216,6 +223,7 @@ void Tick() return completion.Task; } + /// Checks the physical pixel area and owns the full-size texture until response construction finishes. private static object Capture(EditorWindow target, Parameters p) { float scale = EditorWindowScreenshotUtility.GetWindowPixelsPerPoint(target); @@ -224,48 +232,95 @@ private static object Capture(EditorWindow target, Parameters p) if (width <= 0 || height <= 0 || (long)width * height > 16777216) return new ErrorResponse("The target has an empty or excessive capture area."); Texture2D full = null; - Texture2D image = null; try { full = EditorWindowScreenshotUtility.CaptureWindowPixels(target, width, height); + return BuildCaptureResponse(target, p, full, scale); + } + finally + { + if (full != null) Object.DestroyImmediate(full); + } + } + + /// + /// Prepares metadata and inline pixels before persisting an optional full-size PNG. + /// The caller owns full; this method releases only its downscaled texture. + /// + internal static object BuildCaptureResponse(EditorWindow target, Parameters p, Texture2D full, + float scale, Func downscale = null) + { + Texture2D image = null; + try + { string path = null; byte[] fullPng = null; if (p.save_file) { string folder = Path.GetFullPath(Path.Combine(Application.dataPath, "../Library/McpEditorScreenshots")); - Directory.CreateDirectory(folder); path = Path.Combine(folder, DateTime.UtcNow.ToString("yyyyMMdd-HHmmss-fff") + "-" + Guid.NewGuid().ToString("N") + ".png"); fullPng = full.EncodeToPNG(); - File.WriteAllBytes(path, fullPng); } var data = new JObject { ["window"] = JObject.FromObject(Describe(target)), ["capture_source"] = "editor_window_buffer", ["captured_at_utc"] = DateTime.UtcNow.ToString("O"), - ["width"] = width, ["height"] = height, + ["width"] = full.width, ["height"] = full.height, ["pixels_per_point"] = scale, ["path"] = path, ["mimeType"] = "image/png" }; if (p.include_image) { image = Mathf.Max(full.width, full.height) > p.max_resolution - ? ScreenshotUtility.DownscaleTexture(full, p.max_resolution) : full; + ? (downscale ?? ScreenshotUtility.DownscaleTexture)(full, p.max_resolution) : full; data["imageBase64"] = Convert.ToBase64String(image == full ? fullPng ?? full.EncodeToPNG() : image.EncodeToPNG()); data["imageWidth"] = image.width; data["imageHeight"] = image.height; } - return new SuccessResponse("Editor window captured.", data); + var response = new SuccessResponse("Editor window captured.", data); + return path == null ? response : SaveCaptureFile(path, fullPng, response); + } + catch (Exception ex) + { + return new ErrorResponse("Window capture failed: " + ex.Message); } finally { if (image != null && image != full) Object.DestroyImmediate(image); - if (full != null) Object.DestroyImmediate(full); } } + /// + /// Saves a prepared response's unique PNG, removing incomplete output on failure. + /// If removal fails, the error exposes the retained path and cleanup outcome. + /// + internal static object SaveCaptureFile(string path, byte[] png, SuccessResponse response) + { + try + { + Directory.CreateDirectory(Path.GetDirectoryName(path)); + File.WriteAllBytes(path, png); + return response; + } + catch (Exception ex) + { + try + { + File.Delete(path); + } + catch (Exception cleanupError) + { + return new ErrorResponse("Screenshot file write failed and its output could not be removed: " + ex.Message, + new { path, cleanup_failed = true, cleanup_error = cleanupError.Message }); + } + return new ErrorResponse("Screenshot file write failed; incomplete output was removed: " + ex.Message); + } + } + + /// Reports current window identity, tab selection, keyboard focus, and content position in Editor points. private static object Describe(EditorWindow window) { Rect rect = window.position; @@ -278,6 +333,7 @@ private static object Describe(EditorWindow window) }; } + /// Returns a completed command task containing a structured error instead of a transport exception. private static Task Failure(string message) => Task.FromResult(new ErrorResponse(message)); } } diff --git a/Server/src/services/tools/manage_editor_windows.py b/Server/src/services/tools/manage_editor_windows.py index 65ef844e8..5939e7f8e 100644 --- a/Server/src/services/tools/manage_editor_windows.py +++ b/Server/src/services/tools/manage_editor_windows.py @@ -36,7 +36,14 @@ async def manage_editor_windows( save_file: Annotated[bool, "Save a unique full-size PNG in Library/McpEditorScreenshots. Default false."] = False, max_resolution: Annotated[int, "Maximum edge for the inline image, from 64 to 4096. Does not resize saved files."] = 1600, ) -> dict[str, Any] | ToolResult: - """Observe open Editor windows through the selected Unity instance.""" + """Observe open windows through the context-selected Unity instance. + + Validate selectors before dispatch. Successful captures return one image + block and metadata, or file-only metadata when requested. If image-block + conversion fails after Unity saves a file, retain its path in a structured + error without returning encoded pixels. Audience annotations are client + display hints and do not enforce privacy or remove saved files. + """ if action not in ("list", "screenshot"): return {"success": False, "message": "action must be list or screenshot."} params: dict[str, Any] = {"action": action} @@ -63,7 +70,16 @@ async def manage_editor_windows( if not isinstance(response, dict): return {"success": False, "message": str(response)} if action == "screenshot": - images = extract_screenshot_images(response, image_audience=["assistant"]) + try: + images = extract_screenshot_images(response, image_audience=["assistant"]) + except (TypeError, ValueError): + data = response.get("data") + metadata = {key: value for key, value in data.items() if key != "imageBase64"} if isinstance(data, dict) else {} + return { + "success": False, + "message": "Could not build the image response. Any saved screenshot remains at data.path.", + "data": metadata, + } if images is not None: return images return response diff --git a/Server/tests/test_cli_editor_windows.py b/Server/tests/test_cli_editor_windows.py index f7e67a818..f43fddc9d 100644 --- a/Server/tests/test_cli_editor_windows.py +++ b/Server/tests/test_cli_editor_windows.py @@ -9,6 +9,7 @@ @pytest.fixture def transport(monkeypatch): + """Use an isolated CLI configuration and mock transport without connecting to Unity.""" config = CLIConfig(format="json", unity_instance="isolated-test@hash") set_config(config) send = Mock(return_value={"success": True, "data": {"path": "Library/McpEditorScreenshots/test.png"}}) @@ -17,6 +18,7 @@ def transport(monkeypatch): def test_list_routes_through_cli_configuration(transport): + """The window-list command retains configured instance routing.""" send, config = transport result = CliRunner().invoke(editor, ["windows"]) assert result.exit_code == 0, result.output @@ -29,6 +31,7 @@ def test_list_routes_through_cli_configuration(transport): (["--window-type", "UnityEditor.ConsoleWindow"], {"window_type": "UnityEditor.ConsoleWindow"}), ]) def test_capture_saves_file_without_printing_base64(transport, selector, params): + """CLI captures request file-only output and preserve explicit selector and focus options.""" send, config = transport result = CliRunner().invoke(editor, ["screenshot", *selector, "--no-focus", "--no-restore-focus"]) assert result.exit_code == 0, result.output @@ -44,6 +47,7 @@ def test_capture_saves_file_without_printing_base64(transport, selector, params) ["--output-folder", "../private"], ]) def test_invalid_cli_capture_does_not_connect(transport, args): + """Invalid selectors and unsupported output paths fail before connection.""" result = CliRunner().invoke(editor, ["screenshot", *args]) assert result.exit_code == 2 transport[0].assert_not_called() diff --git a/Server/tests/test_manage_editor_windows.py b/Server/tests/test_manage_editor_windows.py index 88bf0948a..b303a5911 100644 --- a/Server/tests/test_manage_editor_windows.py +++ b/Server/tests/test_manage_editor_windows.py @@ -16,6 +16,7 @@ @pytest.fixture def transport(monkeypatch): + """Isolate instance selection and Unity transport so tool tests cannot reach a running Editor.""" target = "services.tools.manage_editor_windows" send = AsyncMock(return_value={"success": True, "data": {"windows": []}}) monkeypatch.setattr(target + ".get_unity_instance_from_context", AsyncMock(return_value="test-instance")) @@ -24,16 +25,19 @@ def transport(monkeypatch): def call(**kwargs): + """Invoke the asynchronous tool with a minimal context in a private event loop.""" return asyncio.run(manage_editor_windows(SimpleNamespace(), **kwargs)) def test_list_uses_request_instance_without_capture_side_effects(transport): + """Listing sends only the list action through the context-selected instance.""" assert call()["success"] assert transport.call_args.args[1:] == ("test-instance", "manage_editor_windows", {"action": "list"}) @pytest.mark.parametrize("selector", [{"window_id": 42}, {"window_title": "Inspector"}, {"window_type": "UnityEditor.ConsoleWindow"}, {}]) def test_capture_routes_selector_and_privacy_defaults(transport, selector): + """Every selector preserves inline-only output and default focus restoration.""" call(action="screenshot", **selector) params = transport.call_args.args[3] assert all(params[key] == value for key, value in selector.items()) @@ -53,11 +57,13 @@ def test_capture_routes_selector_and_privacy_defaults(transport, selector): {"action": "screenshot", "include_image": False, "save_file": False}, ]) def test_invalid_requests_do_not_reach_unity(transport, kwargs): + """Invalid actions, selectors, sizes, and output combinations fail before transport.""" assert call(**kwargs)["success"] is False transport.assert_not_called() def test_image_bytes_appear_once_with_metadata_preserved(transport): + """One assistant-audience image carries pixels while metadata and the source response remain intact.""" value = {"success": True, "message": "captured", "data": { "imageBase64": "aW1hZ2U=", "mimeType": "image/png", "path": None, "window": {"window_id": 42}, }} @@ -78,6 +84,7 @@ def test_image_bytes_appear_once_with_metadata_preserved(transport): def test_file_only_capture_and_unity_errors_are_preserved(transport): + """File-only results and structured Unity errors pass through without image conversion.""" transport.return_value = {"success": True, "data": {"path": "Library/McpEditorScreenshots/test.png"}} assert call(action="screenshot", include_image=False, save_file=True) == transport.return_value transport.return_value = {"success": False, "error": "busy"} @@ -87,6 +94,7 @@ def test_file_only_capture_and_unity_errors_are_preserved(transport): def test_native_mcp_protocol_returns_image_and_rejects_unknown_arguments(): # The integration suite installs FastMCP stubs during collection. A child # process verifies real wire types without changing that suite's environment. + """A child process checks real MCP image types, annotations, and rejection of unknown output paths.""" script = ''' import asyncio from unittest.mock import AsyncMock, patch @@ -125,3 +133,28 @@ async def run(): env["PYTHONPATH"] = str(Path(__file__).resolve().parents[1] / "src") result = subprocess.run([sys.executable, "-c", script], env=env, capture_output=True, text=True, timeout=45) assert result.returncode == 0, result.stdout + result.stderr + + +@pytest.mark.parametrize("saved_path", [None, "Library/McpEditorScreenshots/requested.png"]) +@pytest.mark.parametrize("failure", [ValueError, TypeError]) +def test_image_conversion_failure_reports_saved_path_without_pixels(transport, monkeypatch, saved_path, failure): + """Response conversion errors must retain cleanup metadata without leaking encoded pixels.""" + value = {"success": True, "message": "captured", "data": { + "imageBase64": "private-pixels", "path": saved_path, "window": {"window_id": 42}, + }} + original = copy.deepcopy(value) + transport.return_value = value + + def fail_conversion(*args, **kwargs): + """Fail after Unity has returned its successful optional-file result.""" + raise failure("image conversion failed") + + monkeypatch.setattr("services.tools.manage_editor_windows.extract_screenshot_images", fail_conversion) + result = call(action="screenshot", save_file=saved_path is not None) + assert result["success"] is False + assert "image response" in result["message"] + assert result["data"]["path"] == saved_path + assert result["data"]["window"]["window_id"] == 42 + assert "imageBase64" not in result["data"] + assert "private-pixels" not in json.dumps(result) + assert value == original diff --git a/TestProjects/UnityMCPTests/Assets/Tests/EditMode/Tools/ManageEditorWindowsTests.cs b/TestProjects/UnityMCPTests/Assets/Tests/EditMode/Tools/ManageEditorWindowsTests.cs index 093d2e22e..f433f6bbe 100644 --- a/TestProjects/UnityMCPTests/Assets/Tests/EditMode/Tools/ManageEditorWindowsTests.cs +++ b/TestProjects/UnityMCPTests/Assets/Tests/EditMode/Tools/ManageEditorWindowsTests.cs @@ -17,6 +17,7 @@ namespace MCPForUnityTests.Editor.Tools public class ScreenshotColorWindow : EditorWindow { public bool paintEdgeMarkers; + /// Paints deterministic color regions used to detect orientation, cropping, and color-space changes. protected void OnGUI() { float halfWidth = position.width / 2; @@ -38,6 +39,7 @@ public class ScreenshotDockWindow : ScreenshotColorWindow { } public class ScreenshotDarkWindow : EditorWindow { + /// Paints deterministic color regions used to detect orientation, cropping, and color-space changes. private void OnGUI() => EditorGUI.DrawRect(new Rect(0, 0, position.width, position.height), new Color(0.12f, 0.18f, 0.24f, 1)); } @@ -50,6 +52,7 @@ public class ManageEditorWindowsTests private ScreenshotDockWindow docked; private EditorWindow previous; + /// Creates two unshown windows with distinct identities and records the previous keyboard focus. [SetUp] public void SetUp() { @@ -60,6 +63,7 @@ public void SetUp() second.titleContent = new GUIContent("MCP second fixture"); } + /// Cancels any pending request, disposes owned fixtures, and restores the earlier focused window. [TearDown] public void TearDown() { @@ -70,12 +74,14 @@ public void TearDown() if (previous != null) previous.Focus(); } + /// Returns the owned test Editor to Edit Mode after tests that enter Play Mode. [UnityTearDown] public IEnumerator LeavePlayMode() { if (EditorApplication.isPlaying) yield return new ExitPlayMode(); } + /// Duplicate titles must not prevent an exact instance ID from selecting its intended window. [Test] public void IdSelectsOneWindowWithDuplicateTitles() { @@ -84,12 +90,14 @@ public void IdSelectsOneWindowWithDuplicateTitles() Assert.That(error, Is.Null); } + /// Title matching accepts case differences and surrounding whitespace. [Test] public void TitleIgnoresCaseAndOuterSpaces() { Assert.That(Resolve(new ManageEditorWindows.Parameters { window_title = " MCP FIRST fixture " }, out _), Is.SameAs(first)); } + /// Ambiguous title and type selectors fail rather than capture an arbitrary window. [Test] public void DuplicateTitlesAndTypesRequireIds() { @@ -99,6 +107,7 @@ public void DuplicateTitlesAndTypesRequireIds() Assert.That(Resolve(new ManageEditorWindows.Parameters { window_type = typeof(ScreenshotColorWindow).FullName }, out _), Is.Null); } + /// Conflicting selectors fail before target selection. [Test] public void MultipleSelectorsAreRejected() { @@ -106,6 +115,7 @@ public void MultipleSelectorsAreRejected() Assert.That(error, Does.Contain("only one")); } + /// A destroyed target ID must not fall back to another open window. [Test] public void ClosedIdCannotSelectAnotherWindow() { @@ -116,6 +126,7 @@ public void ClosedIdCannotSelectAnotherWindow() Assert.That(error, Does.Contain("No matching")); } + /// Unsupported actions return a structured error without changing keyboard focus. [Test] public void InvalidActionDoesNotChangeFocus() { @@ -124,6 +135,7 @@ public void InvalidActionDoesNotChangeFocus() Assert.That(EditorWindow.focusedWindow, Is.SameAs(focused)); } + /// Headless Editors reject window pixel capture with an actionable batch-mode error. [Test] public void BatchCaptureHasAnExplicitError() { @@ -132,12 +144,14 @@ public void BatchCaptureHasAnExplicitError() Assert.That(result.Error, Does.Contain("Batch mode")); } + /// An unshown window has no host and cannot trigger desktop capture. [Test] public void UnsupportedWindowHostFailsWithoutDesktopFallback() { Assert.Throws(() => EditorWindowScreenshotUtility.CaptureWindowPixels(first, 64, 64)); } + /// Empty and over-limit pixel buffers fail before graphics allocation. [TestCase(0, 64)] [TestCase(4097, 4097)] public void EmptyAndExcessiveBufferSizesAreRejected(int width, int height) @@ -145,10 +159,12 @@ public void EmptyAndExcessiveBufferSizesAreRejected(int width, int height) Assert.Throws(() => EditorWindowScreenshotUtility.CaptureWindowPixels(first, width, height)); } + /// Direct snake-case and batch camel-case inputs preserve every capture option and omitted default. [TestCase(false)] [TestCase(true)] public void ParameterAccessorsPreserveEveryOptionAndDefault(bool camelCase) { + /// Chooses the input alias for each parameter without changing the expected value. string Key(string snake, string camel) => camelCase ? camel : snake; var args = new JObject { ["action"] = "screenshot", [Key("window_id", "windowId")] = second.GetInstanceID(), @@ -174,6 +190,7 @@ public void ParameterAccessorsPreserveEveryOptionAndDefault(bool camelCase) Assert.That(p.max_resolution, Is.EqualTo(1600)); } + /// Malformed explicit selectors and sizes fail without silently capturing the focused window. [TestCase("window_id", "not-an-id")] [TestCase("windowId", "not-an-id")] [TestCase("windowId", null)] @@ -192,6 +209,105 @@ public void InvalidExplicitParametersNeverBecomeFocusedWindowRequests(string key Assert.That(EditorWindow.focusedWindow, Is.SameAs(focused)); } + /// Late inline-processing failures must not leave encoded private pixels on disk. + [Test] + public void InlineProcessingFailureDoesNotPersistFullSizePng() + { + string folder = Path.GetFullPath(Path.Combine(Application.dataPath, "../Library/McpEditorScreenshots")); + var before = Directory.Exists(folder) ? Directory.GetFiles(folder) : Array.Empty(); + var full = new Texture2D(128, 128, TextureFormat.RGBA32, false); + bool reachedInlineProcessing = false; + try + { + var result = ManageEditorWindows.BuildCaptureResponse(first, + new ManageEditorWindows.Parameters { save_file = true, include_image = true, max_resolution = 64 }, + full, 1, (_, __) => { + reachedInlineProcessing = true; + throw new InvalidOperationException("injected inline processing failure"); + }); + Assert.That(reachedInlineProcessing, Is.True, "The full PNG must be encoded before the injected failure."); + Assert.That(result, Is.TypeOf()); + Assert.That(((ErrorResponse)result).Error, Does.Contain("injected inline processing failure")); + Assert.That(Directory.Exists(folder) ? Directory.GetFiles(folder) : Array.Empty(), + Is.EquivalentTo(before), "A failed response must not persist a new screenshot."); + Assert.That(full != null, Is.True, "Response construction must not destroy its caller's texture."); + } + finally { Object.DestroyImmediate(full); } + } + + /// CPU pixel fixtures verify successful file-only and unscaled file/image output without window focus. + [TestCase(false)] + [TestCase(true)] + public void PreparedCapturePersistsFullPngAndReusesInlineBytes(bool includeImage) + { + var full = new Texture2D(16, 16, TextureFormat.RGBA32, false); + string path = null; + try + { + full.SetPixel(0, 0, Color.blue); + full.Apply(); + var result = ManageEditorWindows.BuildCaptureResponse(first, + new ManageEditorWindows.Parameters { save_file = true, include_image = includeImage }, full, 1); + Assert.That(result, Is.TypeOf()); + var data = (JObject)((SuccessResponse)result).Data; + path = (string)data["path"]; + Assert.That(Path.GetDirectoryName(path), Is.EqualTo(Path.GetFullPath( + Path.Combine(Application.dataPath, "../Library/McpEditorScreenshots")))); + Assert.That(File.ReadAllBytes(path), Is.EqualTo(full.EncodeToPNG())); + if (includeImage) Assert.That(Convert.FromBase64String((string)data["imageBase64"]), Is.EqualTo(File.ReadAllBytes(path))); + else Assert.That(data["imageBase64"], Is.Null); + } + finally + { + if (path != null && File.Exists(path)) File.Delete(path); + Object.DestroyImmediate(full); + } + } + + /// Failed writes remove the generated partial file rather than returning an orphaned path. + [Test] + public void FailedFileWriteRemovesIncompleteOutput() + { + string path = Path.GetFullPath(Path.Combine(Application.dataPath, + "../Library/McpEditorScreenshots", Guid.NewGuid().ToString("N") + ".png")); + Directory.CreateDirectory(Path.GetDirectoryName(path)); + File.WriteAllBytes(path, new byte[] { 137, 80 }); + try + { + var result = ManageEditorWindows.SaveCaptureFile(path, null, new SuccessResponse("prepared")); + Assert.That(result, Is.TypeOf()); + Assert.That(File.Exists(path), Is.False); + Assert.That(((ErrorResponse)result).Error, Does.Contain("incomplete output was removed")); + } + finally { if (File.Exists(path)) File.Delete(path); } + } + + /// Windows sharing violations expose the retained file path when write and cleanup both fail. + [Test, Platform("Win")] + public void FailedCleanupReportsRetainedPath() + { + string path = Path.GetFullPath(Path.Combine(Application.dataPath, + "../Library/McpEditorScreenshots", Guid.NewGuid().ToString("N") + ".png")); + Directory.CreateDirectory(Path.GetDirectoryName(path)); + try + { + using (var locked = new FileStream(path, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None)) + { + locked.WriteByte(137); + locked.Flush(); + var result = ManageEditorWindows.SaveCaptureFile(path, new byte[] { 137, 80 }, new SuccessResponse("prepared")); + Assert.That(result, Is.TypeOf()); + var data = JObject.FromObject(((ErrorResponse)result).Data); + Assert.That((string)data["path"], Is.EqualTo(path)); + Assert.That((bool)data["cleanup_failed"], Is.True); + Assert.That((string)data["cleanup_error"], Is.Not.Empty); + } + Assert.That(File.Exists(path), Is.True, "The error must reference the retained output."); + } + finally { if (File.Exists(path)) File.Delete(path); } + } + + /// Combined graphical output contains the same PNG bytes in the full-size file and inline image. [UnityTest] public IEnumerator SavedAndInlineFullSizePngBytesMatch() { @@ -214,6 +330,7 @@ public IEnumerator SavedAndInlineFullSizePngBytesMatch() finally { if (File.Exists(saved)) File.Delete(saved); } } + /// Docked capture preserves four corner colors and dimensions derived from native backing scale. [UnityTest] public IEnumerator DockedBufferPreservesFourCornerColorsAndPhysicalDimensions() { @@ -242,6 +359,7 @@ public IEnumerator DockedBufferPreservesFourCornerColorsAndPhysicalDimensions() finally { Object.DestroyImmediate(image); scene.ShowTab(); } } + /// A floating capture keeps pixel orientation and restores the previously focused fixture. [UnityTest] public IEnumerator FloatingBufferCapturePreservesOrientationAndFocus() { @@ -263,6 +381,7 @@ public IEnumerator FloatingBufferCapturePreservesOrientationAndFocus() finally { Object.DestroyImmediate(image); } } + /// Thin colored rims detect host-margin offsets and cropping in floating full-size output. [UnityTest] public IEnumerator FloatingFullSizeBufferMatchesContentEdges() { @@ -286,14 +405,17 @@ public IEnumerator FloatingFullSizeBufferMatchesContentEdges() finally { Object.DestroyImmediate(image); } } + /// The floating Scene View viewport shares correct content margins and readback orientation. [UnityTest] public IEnumerator SceneViewViewportPreservesContentEdgesAndOrientation() => CaptureSceneViewFixture(false); + /// The docked Scene View viewport excludes host chrome and preserves its edge markers. [UnityTest] public IEnumerator DockedSceneViewViewportPreservesContentEdgesAndOrientation() => CaptureSceneViewFixture(true); + /// Paints only the owned Scene View viewport and restores its overlays, gizmos, handlers, and output file. private static IEnumerator CaptureSceneViewFixture(bool dockedScene) { RequireGraphics(); @@ -312,6 +434,7 @@ private static IEnumerator CaptureSceneViewFixture(bool dockedScene) var overlaysProperty = canvas?.GetType().GetProperty("overlaysEnabled", BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); object previousOverlays = overlaysProperty?.GetValue(canvas); + /// Temporarily hides fixture overlays through the setter available in the tested Editor version. void SetOverlays(bool enabled) { if (canvas == null) return; @@ -328,6 +451,7 @@ void SetOverlays(bool enabled) string folder = Path.GetFullPath(Path.Combine(Application.dataPath, "../Library/McpSceneViewportTests", Guid.NewGuid().ToString("N"))); int repaints = 0; + /// Paints quadrant and edge markers only during repaints of the owned Scene View. void PaintViewport(SceneView view) { if (view != scene || Event.current.type != EventType.Repaint) return; @@ -385,6 +509,7 @@ void PaintViewport(SceneView view) } } + /// A direct camel-case request captures the specified unfocused fixture rather than the focused one. [UnityTest] public IEnumerator CamelCaseRequestSelectsTheUnfocusedWindow() { @@ -402,6 +527,7 @@ public IEnumerator CamelCaseRequestSelectsTheUnfocusedWindow() Assert.That(EditorWindow.focusedWindow, Is.SameAs(first)); } + /// The real batch route preserves selector identity after parameter-key normalization. [UnityTest] public IEnumerator BatchRouteSelectsTheRequestedUnfocusedWindow() { @@ -423,6 +549,7 @@ public IEnumerator BatchRouteSelectsTheRequestedUnfocusedWindow() Assert.That(EditorWindow.focusedWindow, Is.SameAs(first)); } + /// The batch route rejects invalid explicit IDs and leaves the focused window unchanged. [UnityTest] public IEnumerator InvalidBatchSelectorFailsWithoutCapturingTheFocusedWindow() { @@ -443,6 +570,7 @@ public IEnumerator InvalidBatchSelectorFailsWithoutCapturingTheFocusedWindow() Assert.That(EditorWindow.focusedWindow, Is.SameAs(first)); } + /// Dock restoration uses host identity even when an inactive tab rectangle is stale after resize. [UnityTest] public IEnumerator ResizedDockRestoresItsSelectedTabWhenAnotherWindowHadKeyboardFocus() { @@ -477,6 +605,7 @@ public IEnumerator ResizedDockRestoresItsSelectedTabWhenAnotherWindowHadKeyboard finally { if (host != null) positionProperty.SetValue(host, original); } } + /// Explicit file-only captures use distinct generated names inside Library and return no inline pixels. [UnityTest] public IEnumerator FileOnlyCaptureIsUniqueAndInsideLibrary() { @@ -511,6 +640,7 @@ public IEnumerator FileOnlyCaptureIsUniqueAndInsideLibrary() } } + /// Cancellation restores prior focus and releases the gate for a subsequent successful capture. [UnityTest] public IEnumerator CancellationRestoresFocusAndAllowsTheNextCapture() { @@ -528,6 +658,7 @@ public IEnumerator CancellationRestoresFocusAndAllowsTheNextCapture() Assert.That(retry.Result, Is.TypeOf()); } + /// Closing a target completes its request with an error and permits a later capture. [UnityTest] public IEnumerator ClosingTargetCompletesWithErrorAndReleasesTheCaptureGate() { @@ -542,6 +673,7 @@ public IEnumerator ClosingTargetCompletesWithErrorAndReleasesTheCaptureGate() Assert.That(retry.Result, Is.TypeOf()); } + /// Inactive docked tabs remain discoverable and their prior selected tab is restored after capture. [UnityTest] public IEnumerator InactiveDockedTabIsListedCapturedAndRestored() { @@ -566,6 +698,7 @@ public IEnumerator InactiveDockedTabIsListedCapturedAndRestored() Assert.That(EditorWindow.focusedWindow, Is.SameAs(scene)); } + /// A focus change after capture begins takes precedence over automatic focus restoration. [UnityTest] public IEnumerator CaptureDoesNotOverrideUserFocusChanges() { @@ -584,6 +717,7 @@ public IEnumerator CaptureDoesNotOverrideUserFocusChanges() finally { third.Close(); } } + /// Resizing retains the full-size dark colors within readback tolerance. [UnityTest] public IEnumerator DownscaledDarkPixelsMatchTheFullSizeCapture() { @@ -620,6 +754,7 @@ public IEnumerator DownscaledDarkPixelsMatchTheFullSizeCapture() } } + /// Composited Game View capture leaves Play Mode running and introduces no PlayerLoop step. [UnityTest] public IEnumerator GameViewBufferCaptureInPlayModeDoesNotStepOrPauseThePlayer() { @@ -643,6 +778,7 @@ public IEnumerator GameViewBufferCaptureInPlayModeDoesNotStepOrPauseThePlayer() yield return new ExitPlayMode(); } + /// Assembly reload interrupts the request through normal cleanup and leaves capture retry usable. [UnityTest] public IEnumerator AssemblyReloadCancelsThePendingCaptureAndAllowsRetry() { @@ -653,6 +789,7 @@ public IEnumerator AssemblyReloadCancelsThePendingCaptureAndAllowsRetry() const string focusKey = "MCP.ScreenshotTests.ReloadFocus"; SessionState.EraseString(resultKey); SessionState.EraseBool(focusKey); + /// Records that reload cancellation completed before the test assembly is replaced. void ObserveCancellation() { SessionState.SetString(resultKey, task.IsCompleted && task.Result is ErrorResponse error ? error.Error : "not cancelled"); @@ -673,6 +810,7 @@ void ObserveCancellation() Assert.That(retry.Result, Is.TypeOf()); } + /// Minimized-window capture is bounded and capture recovers after restoring the owned Editor window. [UnityTest] public IEnumerator MinimizedWindowCompletesAndSubsequentCaptureRecovers() { @@ -733,6 +871,7 @@ public IEnumerator MinimizedWindowCompletesAndSubsequentCaptureRecovers() private static extern bool IsIconic(IntPtr handle); #endif + /// Shows only owned utility windows and waits for a stable first-fixture focus before assertions. private IEnumerator ShowFixtures() { first.position = new Rect(100, 100, 320, 240); @@ -753,6 +892,7 @@ private IEnumerator ShowFixtures() "The fixture must establish keyboard focus before requesting capture."); } + /// Checks all four image corners to detect axis flips and mismatched color quadrants. private static void AssertCornerColors(Texture2D image) { // Coordinates are actual output pixels, not a percentage of the buffer. @@ -767,6 +907,7 @@ private static void AssertCornerColors(Texture2D image) AssertPixel(image, image.width - 1 - inset, inset, new Color(1, 1, 0, 1), "bottom right content edge"); } + /// Checks the colored outer rim and black interior to detect content offsets as well as flips. private static void AssertContentEdgeMarkers(Texture2D image) { AssertCornerColors(image); @@ -780,6 +921,7 @@ private static void AssertContentEdgeMarkers(Texture2D image) AssertPixel(image, image.width - 1 - inner, inner, Color.black, "bottom right interior"); } + /// Samples one expected edge or interior position with an explanatory assertion label. private static void AssertPixel(Texture2D image, int x, int y, Color expected, string edge) { Color actual = image.GetPixel(x, y); @@ -788,6 +930,7 @@ private static void AssertPixel(Texture2D image, int x, int y, Color expected, s + $"uv_top={SystemInfo.graphicsUVStartsAtTop}"); } + /// Compares RGB channels within tolerance while preserving a precise failing region label. private static void AssertColor(Color actual, Color expected, string corner) { Assert.That(actual.r, Is.EqualTo(expected.r).Within(0.04f), corner + " red"); @@ -795,15 +938,18 @@ private static void AssertColor(Color actual, Color expected, string corner) Assert.That(actual.b, Is.EqualTo(expected.b).Within(0.04f), corner + " blue"); } + /// Skips window-buffer cases in headless Editors and records the active graphics API for real captures. private static void RequireGraphics() { if (Application.isBatchMode || SystemInfo.graphicsDeviceType == UnityEngine.Rendering.GraphicsDeviceType.Null) Assert.Ignore("Pixel capture requires a graphical Editor; run this fixture without -batchmode/-nographics."); } + /// Requests an inline-only capture of the explicit fixture ID at the requested resolution. private static Task Screenshot(EditorWindow window, int resolution = 1600) => ManageEditorWindows.HandleCommand(new JObject { ["action"] = "screenshot", ["window_id"] = window.GetInstanceID(), ["max_resolution"] = resolution }); + /// Bounds asynchronous fixture requests so capture lifecycle regressions fail instead of hanging the suite. private static IEnumerator Await(Task task) { double deadline = EditorApplication.timeSinceStartup + 10; @@ -812,6 +958,7 @@ private static IEnumerator Await(Task task) Assert.That(task.IsFaulted, Is.False, task.Exception?.ToString()); } + /// Runs selector resolution against only the two owned windows. private EditorWindow Resolve(ManageEditorWindows.Parameters p, out string error) => ManageEditorWindows.Resolve(new EditorWindow[] { first, second }, p, out error); } diff --git a/website/docs/guides/editor-window-screenshots.md b/website/docs/guides/editor-window-screenshots.md index 67034ceb6..e58f7b198 100644 --- a/website/docs/guides/editor-window-screenshots.md +++ b/website/docs/guides/editor-window-screenshots.md @@ -51,6 +51,13 @@ full-size PNG in `Library/McpEditorScreenshots`; `max_resolution` only limits the inline image. Use `include_image=false, save_file=true` for a file-only result. Output paths are not supplied by the caller. +The PNG is written after metadata and the inline image are prepared. Failed +writes attempt to remove incomplete output; if removal also fails, the error +includes `data.path` and `data.cleanup_failed=true` so you can delete it manually. +If the server cannot build the MCP image block after Unity saves the file, its +error retains `data.path` and omits the encoded pixels. +A lost client connection after a successful save does not undo that saved file. + Screenshots can contain private source code, asset names, file paths, Console messages or credentials displayed in Editor windows. The image is sent to the selected MCP client. Enable and use the tool only with clients