From e52a74405779d516d9b9dc02496d490fc22acfb1 Mon Sep 17 00:00:00 2001 From: Google Maps SDK Team Date: Fri, 21 Aug 2026 17:55:00 +0000 Subject: [PATCH] feat: Sync from internal staging. --- .github/workflows/release.yml | 8 +- .github/workflows/web-ci.yml | 14 + agent/python_agent/agent_config.py | 2 +- agent/python_agent/agent_with_grounding.py | 38 ++- agent/python_agent/agent_with_templates.py | 179 +++++++--- agent/python_agent/extractor.py | 16 +- .../instructions/shared_style_guidelines.md | 18 +- .../python_agent/test_agent_with_templates.py | 160 ++++++--- .../web_build/index.html | 0 .../web_build/src/core-shell.ts | 0 client/android/web_build/src/main.ts | 2 +- .../web_build/tsconfig.json | 0 .../web_build/vite.config.ts | 0 client/ios/web_build/index.html | 39 +++ client/ios/web_build/src/core-shell.ts | 310 ++++++++++++++++++ client/ios/web_build/src/main.ts | 2 +- client/ios/web_build/tsconfig.json | 22 ++ client/ios/web_build/vite.config.ts | 33 ++ client/mobile_core/web_build/build_defs.bzl | 37 --- client/web/package-lock.json | 4 +- client/web/package.json | 2 +- client/web/tsconfig.json | 5 +- 22 files changed, 728 insertions(+), 163 deletions(-) rename client/{mobile_core => android}/web_build/index.html (100%) rename client/{mobile_core => android}/web_build/src/core-shell.ts (100%) rename client/{mobile_core => android}/web_build/tsconfig.json (100%) rename client/{mobile_core => android}/web_build/vite.config.ts (100%) create mode 100644 client/ios/web_build/index.html create mode 100644 client/ios/web_build/src/core-shell.ts create mode 100644 client/ios/web_build/tsconfig.json create mode 100644 client/ios/web_build/vite.config.ts delete mode 100644 client/mobile_core/web_build/build_defs.bzl diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 127f11b..fc62731 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -51,12 +51,14 @@ jobs: uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4.0.1 with: - node-version: 20 + node-version: 22 registry-url: "https://wombat-dressing-room.appspot.com/" - name: Run Semantic Release - working-directory: client/web env: GITHUB_TOKEN: ${{ secrets.SYNCED_GITHUB_TOKEN_REPO }} + NODE_AUTH_TOKEN: ${{ secrets.NPM_WOMBAT_TOKEN }} NPM_TOKEN: ${{ secrets.NPM_WOMBAT_TOKEN }} - run: npm run release + NODE_PATH: ${{ github.workspace }}/client/web/node_modules + run: npx --prefix client/web semantic-release + diff --git a/.github/workflows/web-ci.yml b/.github/workflows/web-ci.yml index fde544b..b17c5e9 100644 --- a/.github/workflows/web-ci.yml +++ b/.github/workflows/web-ci.yml @@ -1,3 +1,17 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + name: Web CI on: diff --git a/agent/python_agent/agent_config.py b/agent/python_agent/agent_config.py index 78c15b3..e3151c3 100644 --- a/agent/python_agent/agent_config.py +++ b/agent/python_agent/agent_config.py @@ -47,7 +47,7 @@ class AgentConfig: generic_model: str = "gemini/gemini-3-flash-preview" router_thinking_budget: int = 0 extractor_thinking_budget: int = 0 - fallback_mode: FallbackMode = FallbackMode.DYNAMIC + fallback_mode: FallbackMode = FallbackMode.TEXT def __post_init__(self): if not isinstance(self.fallback_mode, FallbackMode): diff --git a/agent/python_agent/agent_with_grounding.py b/agent/python_agent/agent_with_grounding.py index b955615..f908900 100644 --- a/agent/python_agent/agent_with_grounding.py +++ b/agent/python_agent/agent_with_grounding.py @@ -52,11 +52,15 @@ logger.warning("Skill file not found at %s", skill_path) -async def query_vertex_map(query: str) -> str: +async def query_vertex_map( + query: str, + model_id: str = "gemini-3-flash-preview", +) -> str: """Query Google Maps via Vertex Grounding and return cleaned response. Args: query: The location query or question. + model_id: The model ID to use for Vertex Grounding. Returns: The grounded and cleaned A2UI response string. @@ -74,8 +78,6 @@ async def query_vertex_map(query: str) -> str: location = "global" logger.warning("GOOGLE_CLOUD_LOCATION is not set, defaulting to 'global'.") - model_id = "gemini-3-flash-preview" - client = genai.Client(vertexai=True, project=project_id, location=location) # Construct instruction @@ -203,8 +205,30 @@ async def query_vertex_map(query: str) -> str: class MAUIAgentWithGrounding(MAUIAgent): """An agent that finds restaurants based on user criteria, using Vertex Grounding.""" - def __init__(self, base_url: str): - super().__init__(base_url, agent_name="MAUI Agent with Grounding") + def __init__( + self, + base_url: str, + model_name: str = "gemini/gemini-3-flash-preview", + ): + super().__init__( + base_url, + agent_name="MAUI Agent with Grounding", + model_name=model_name, + ) + + async def query_vertex_map(self, query: str) -> str: + """Query Google Maps via Vertex Grounding and return cleaned response. + + Args: + query: The location query or question. + + Returns: + The grounded and cleaned A2UI response string. + """ + model_id = ( + self._model_name.removeprefix("gemini/").removeprefix("models/") + ) + return await query_vertex_map(query, model_id=model_id) def _build_llm_agent( self, schema_manager: A2uiSchemaManager | None = None @@ -223,7 +247,7 @@ def _build_llm_agent( skill_manager_tool = skill_toolset.SkillToolset(skills=skills) # Use FunctionTool for Vertex grounding - grounding_tool = FunctionTool(func=query_vertex_map) + grounding_tool = FunctionTool(func=self.query_vertex_map) agent_instruction = """You are a location routing agent. Whenever the user asks a question about a location, directions, places, or maps, @@ -245,7 +269,7 @@ def _build_llm_agent( instruction = agent_instruction return LlmAgent( - model=LiteLlm(model="gemini/gemini-3-flash-preview"), + model=LiteLlm(model=self._model_name), name="maui_agent_grounding", description=( "An agent that can provide Google Maps UI-enriched responses using" diff --git a/agent/python_agent/agent_with_templates.py b/agent/python_agent/agent_with_templates.py index 657aa49..df11c2a 100644 --- a/agent/python_agent/agent_with_templates.py +++ b/agent/python_agent/agent_with_templates.py @@ -14,6 +14,8 @@ """MAUI Agent with template-based latency optimization.""" +import asyncio +import json import logging import pathlib from types import SimpleNamespace @@ -25,6 +27,7 @@ from google.adk import skills as adk_skills from google.adk.agents import run_config from google.adk.agents.llm_agent import LlmAgent +from google.adk.events.event import Event from google.adk.models.lite_llm import LiteLlm from google.adk.models.llm_request import LlmRequest from google.adk.runners import Runner @@ -65,6 +68,17 @@ } _SUPPORTED_INTENTS = {IntentClass.LOCAL_SEARCH, IntentClass.DIRECTIONS} +_GROUNDED_TEXT_BASE_INSTRUCTION = """\ +You are an expert location and navigation assistant with access to Google Maps tools. + +## Core Rules +1. **Accuracy & Grounding**: Use Google Maps tools to look up real-time places, business hours, amenities, contact details, routes, and weather. NEVER hallucinate place facts, locations, or operational details. +2. **Parallel Tool Execution**: When researching multiple entities, neighborhoods, routes, or options, emit ALL independent tool calls in parallel within your initial response turn. Only serialize calls when Step 2 strictly depends on data returned by Step 1. +3. **Minimize Round-Trips**: Gather necessary place facts efficiently and emit independent queries in parallel. Only perform follow-up tool turns when subsequent calls strictly depend on data returned from earlier steps (e.g., retrieving details for specific place IDs or searching along a computed route polyline). Avoid redundant follow-up queries for details already retrieved. +4. **Helpful & Actionable Answers**: Fully address all constraints in the user's prompt (e.g., parking, pricing, specific dietary options, bag policies). If tools return generic listings that lack specific policy details, supplement with known facts while noting any uncertainty. +5. **No A2UI Tags**: Return standard plain text/markdown only. Do NOT output A2UI tags or JSON surfaces. +""" + class MAUIAgentWithTemplates(MAUIAgent): """MAUI Agent extending base with server-side layout templates and query intent routing.""" @@ -74,6 +88,7 @@ def __init__(self, base_url: str, config: AgentConfig | None = None) -> None: super().__init__(base_url=base_url, model_name=self.config.generic_model) self.router_client = LiteLlm(model=self.config.router_model) self.extractor_client = LiteLlm(model=self.config.template_model) + self.fallback_client = LiteLlm(model=self.config.generic_model) def _build_runner(self, agent: LlmAgent) -> Runner: runner = super()._build_runner(agent) @@ -109,6 +124,19 @@ def _on_tool_error( } return None + def _load_shared_guidelines(self) -> str: + """Loads shared conversational text style guidelines if available.""" + shared_guidelines_path = ( + _SHARED_INSTRUCTIONS_PATH / "shared_style_guidelines.md" + ) + if shared_guidelines_path.exists(): + try: + with open(shared_guidelines_path, "r", encoding="utf-8") as f: + return f.read() + except (OSError, ValueError) as e: + logger.warning("Failed to load shared style guidelines: %s", e) + return "" + def _build_dynamic_extractor_agent( self, skill_name: str, @@ -118,16 +146,9 @@ def _build_dynamic_extractor_agent( skill_dir = _SKILL_BASE_PATH / skill_name skill = adk_skills.load_skill_from_dir(skill_dir) skill_instructions = skill.instructions - shared_guidelines_path = ( - _SHARED_INSTRUCTIONS_PATH / "shared_style_guidelines.md" - ) - if shared_guidelines_path.exists(): - try: - with open(shared_guidelines_path, "r", encoding="utf-8") as f: - shared_guidelines = f.read() - skill_instructions = f"{skill_instructions}\n\n{shared_guidelines}" - except (OSError, ValueError) as e: - logger.warning("Failed to load shared style guidelines: %s", e) + shared_guidelines = self._load_shared_guidelines() + if shared_guidelines: + skill_instructions = f"{skill_instructions}\n\n{shared_guidelines}" # Extractors use template_model, generic UI uses generic_model if skill_name.endswith("-template-response"): @@ -385,9 +406,11 @@ async def stream( else: logger.info( "Router matched OTHER_SPATIAL and fallback_mode is TEXT. " - "Executing fast text response flow." + "Executing grounded text fallback flow." + ) + final_parts = await self._handle_grounded_text_fallback( + cleaned_query, session_id ) - final_parts = await self._handle_text_only(cleaned_query, session_id) yield { "is_task_complete": True, "parts": final_parts, @@ -410,7 +433,22 @@ async def stream( yield part return - # Fallback for un-implemented spatial intents + # Fallback for un-implemented spatial intents or validation failures + if self.config.fallback_mode == FallbackMode.TEXT: + logger.info( + "FallbackMode.TEXT enabled: routing intent %s to grounded text" + " handler.", + intent, + ) + final_parts = await self._handle_grounded_text_fallback( + cleaned_query, session_id + ) + yield { + "is_task_complete": True, + "parts": final_parts, + } + return + logger.warning( "Intent %s not supported by template extractors. Falling back to base" " UI stream.", @@ -483,49 +521,90 @@ def _wrap_in_text_only(self, text: str, session_id: str) -> list[Part]: ) return [create_a2ui_part(action) for action in merged_actions] - async def _handle_text_only( - self, cleaned_query: str, session_id: str - ) -> list[Part]: - """Generates a plain text response and wraps it in the text_only template. - - Args: - cleaned_query: The cleaned user query. - session_id: Context session ID. + def _get_grounded_text_instruction(self) -> str: + """Builds system instruction for grounded text responses.""" + return _GROUNDED_TEXT_BASE_INSTRUCTION - Returns: - List of A2A Parts. - """ - extractor_config = { - "system_instruction": ( - "You are a helpful location assistant. Answer the user's" - " question directly. Keep it relatively concise. Do NOT" - " output A2UI tags." - ), - } - if self.config.extractor_thinking_budget > 0: - extractor_config["thinking_config"] = types.ThinkingConfig( - thinking_budget=self.config.extractor_thinking_budget + async def _handle_grounded_text( + self, cleaned_query: str, session_id: str, client: LiteLlm + ) -> list[Part]: + """Generates a grounded plain text response using the provided model client with GroundingLite tools.""" + system_instruction = self._get_grounded_text_instruction() + generate_content_config = None + if ( + client == self.extractor_client + and self.config.extractor_thinking_budget > 0 + ): + generate_content_config = types.GenerateContentConfig( + thinking_config=types.ThinkingConfig( + thinking_budget=self.config.extractor_thinking_budget + ) ) - answer_req = LlmRequest( - contents=[ - types.Content( - role="user", - parts=[types.Part.from_text(text=cleaned_query)], - ) - ], - config=types.GenerateContentConfig(**extractor_config), + tools = [self.make_grounding_lite_mcp()] + agent = LlmAgent( + model=client, + name="maui_grounded_text_agent", + description="Agent for text responses with Maps grounding", + instruction=system_instruction, + tools=tools, + generate_content_config=generate_content_config, + ) + runner = self._build_runner(agent) + current_message = types.Content( + role="user", parts=[types.Part.from_text(text=cleaned_query)] ) - answer_text = "" - async for res in self.extractor_client.generate_content_async(answer_req): - if res.content and res.content.parts: - for p in res.content.parts: - if p.text: - answer_text += p.text + try: + async for event in runner.run_async( + user_id=self._user_id, + session_id=session_id, + run_config=run_config.RunConfig( + streaming_mode=run_config.StreamingMode.SSE, + ), + new_message=current_message, + state_delta={ + "expression": "{expression}", + "base_url": self.base_url, + }, + ): + if event.content and event.content.parts: + if event.partial: + for p in event.content.parts: + if p.text: + answer_text += p.text + else: + answer_text = "" + for p in event.content.parts: + if p.text: + answer_text += p.text + except Exception as e: + logger.warning("Grounded text generation failed: %s", e) + + if not answer_text: + answer_text = ( + "I'm sorry, I encountered an issue retrieving location details" + " right now." + ) return self._wrap_in_text_only(answer_text, session_id) + async def _handle_text_only( + self, cleaned_query: str, session_id: str + ) -> list[Part]: + """Generates a plain text response for TEXT_ONLY intent using template_model with grounding.""" + return await self._handle_grounded_text( + cleaned_query, session_id, client=self.extractor_client + ) + + async def _handle_grounded_text_fallback( + self, cleaned_query: str, session_id: str + ) -> list[Part]: + """Generates a grounded plain text response for fallback/complex spatial queries using generic_model.""" + return await self._handle_grounded_text( + cleaned_query, session_id, client=self.fallback_client + ) + async def _handle_extracted_intent( self, intent: IntentClass, @@ -571,7 +650,9 @@ async def _handle_extracted_intent( intent, ) if not fallback_text: - final_parts = await self._handle_text_only(query, session_id) + final_parts = await self._handle_grounded_text_fallback( + query, session_id + ) else: final_parts = self._wrap_in_text_only(fallback_text, session_id) yield { diff --git a/agent/python_agent/extractor.py b/agent/python_agent/extractor.py index bd38574..53749a0 100644 --- a/agent/python_agent/extractor.py +++ b/agent/python_agent/extractor.py @@ -80,10 +80,11 @@ class LocalSearchExtractorSchema(BaseModel): summary: str = Field( description=( - "A detailed response summarizing the search results, answering the" - " user's query fully. Use markdown formatting (bullet points," - " bolding, tables) and break into paragraphs as needed. Bold place" - " names." + "A detailed response summarizing the search results that fully and" + " clearly answers all aspects of the user's prompt (including" + " qualitative criteria, preferences, and comparisons). Use markdown" + " formatting (bullet points, bolding, tables) and break into" + " paragraphs as needed. Bold place names." ) ) center_lat: float = Field(description="Latitude of the center of results") @@ -157,9 +158,10 @@ class DirectionsExtractorSchema(BaseModel): summary: str = Field( description=( - "A detailed response summarizing the travel directions, including" - " key steps, estimated time, and travel mode. Use markdown" - " formatting and break into paragraphs if helpful." + "A detailed response summarizing the travel directions and route" + " options that fully answers all user questions, route comparisons," + " and travel context requested in the prompt. Use markdown formatting" + " and break into paragraphs if helpful." ) ) center_lat: float = Field( diff --git a/agent/python_agent/shared/instructions/shared_style_guidelines.md b/agent/python_agent/shared/instructions/shared_style_guidelines.md index f27e0b4..1c29c12 100644 --- a/agent/python_agent/shared/instructions/shared_style_guidelines.md +++ b/agent/python_agent/shared/instructions/shared_style_guidelines.md @@ -3,11 +3,19 @@ When generating conversational text (such as summaries, descriptions, or directions), you must follow these formatting and content rules: -* **Content**: Always fully and clearly answer each aspect of the prompt. -* **Quantity**: Make sure that the answer is useful and actionable. Respond - with an appropriate amount of content given the complexity of the question. - E.g., if helping someone differentiate between places, consider responding - with details about each place. +* **Content & Completeness**: Always fully and clearly answer each aspect of + the user's prompt. Address all explicit constraints, qualitative criteria, + comparisons, preferences, and sub-questions asked. Explain *why* places or + routes fit the user's specific needs rather than providing a bare listing. +* **Quantity & Nuance**: Make sure the answer is substantive, useful, and + actionable. Respond with an appropriate depth of detail given the complexity + of the question: + * If comparing places or route alternatives, explicitly analyze their + trade-offs (e.g. transit vs driving, travel time, convenience, cost, or + atmosphere). + * If the user asks about commute, context, or travel conditions, describe + relevant timing and real-world nuances (e.g. rush-hour delays, + navigation landmarks). * **Formatting**: Use markdown to apply formatting elements like bullet points, bolding, and tables to break up the text. Break content into multiple paragraphs as needed. diff --git a/agent/python_agent/test_agent_with_templates.py b/agent/python_agent/test_agent_with_templates.py index 25410d1..3fcf6c9 100644 --- a/agent/python_agent/test_agent_with_templates.py +++ b/agent/python_agent/test_agent_with_templates.py @@ -36,14 +36,14 @@ class MockPart: """Mock Part helper for test streaming.""" - def __init__(self, text): + def __init__(self, text=""): self.text = text class MockContent: """Mock Content helper for test streaming.""" - def __init__(self, parts): + def __init__(self, parts=None): self.parts = parts @@ -68,8 +68,12 @@ async def __anext__(self): raise StopAsyncIteration return self.items.pop(0) + async def aclose(self): + pass + class MockFunctionCall: + """Mock FunctionCall helper for test streaming.""" def __init__(self, name, args): self.name = name @@ -77,6 +81,7 @@ def __init__(self, name, args): class MockEvent: + """Mock Event helper for ADK runner streaming.""" def __init__(self, function_calls=None, content=None, partial=False): self.function_calls = function_calls or [] @@ -93,16 +98,24 @@ class TestAgentOrchestration(unittest.IsolatedAsyncioTestCase): def setUp(self): super().setUp() self.mock_router = mock.MagicMock(spec=LiteLlm) + self.mock_router.model = "gemini/router-model" self.mock_extractor = mock.MagicMock(spec=LiteLlm) + self.mock_extractor.model = "gemini/template-model" def _setup_mock_llm(self, mock_lite_llm_class): def lite_llm_side_effect(*args, **kwargs): model = kwargs.get("model") or (args[0] if args else None) if model == "gemini/router-model": return self.mock_router - elif model == "gemini/template-model": + elif model in ( + "gemini/template-model", + "gemini/gemini-3-flash-preview", + "gemini/generic-model", + ): return self.mock_extractor - return mock.MagicMock(spec=LiteLlm) + m = mock.MagicMock(spec=LiteLlm) + m.model = model or "mock-model" + return m mock_lite_llm_class.side_effect = lite_llm_side_effect @@ -162,10 +175,14 @@ async def test_agent_text_only_flow(self, mock_lite_llm_class): self._mock_llm_stream('{"intent": "TEXT_ONLY", "query": "hello"}') ) - # Mock extractor response stream yielding text answer - self.mock_extractor.generate_content_async.return_value = ( - self._mock_llm_stream("This is a fast text-only response.") - ) + mock_runner = mock.MagicMock() + mock_runner.run_async.return_value = MockAsyncIterator([ + MockEvent( + content=MockContent( + [MockPart("This is a fast text-only response.")] + ) + ) + ]) config = AgentConfig( fallback_mode=FallbackMode.TEXT, @@ -174,12 +191,8 @@ async def test_agent_text_only_flow(self, mock_lite_llm_class): ) agent = MAUIAgentWithTemplates(base_url="http://test-url", config=config) - # Run stream - results = [] - async for item in agent.stream( - query="hello", session_id="session_123", ui_version="v0.9" - ): - results.append(item) + with mock.patch.object(agent, "_build_runner", return_value=mock_runner): + results = await self._collect_stream(agent, "hello") self.assertEqual(len(results), 1) self.assertTrue(results[0]["is_task_complete"]) @@ -205,9 +218,12 @@ async def test_agent_router_failure_fallback(self, mock_lite_llm_class): self.mock_router.generate_content_async.side_effect = Exception( "Router error" ) - self.mock_extractor.generate_content_async.return_value = ( - self._mock_llm_stream("Response after router failure.") - ) + mock_runner = mock.MagicMock() + mock_runner.run_async.return_value = MockAsyncIterator([ + MockEvent( + content=MockContent([MockPart("Response after router failure.")]) + ) + ]) config = AgentConfig( fallback_mode=FallbackMode.TEXT, @@ -216,11 +232,8 @@ async def test_agent_router_failure_fallback(self, mock_lite_llm_class): ) agent = MAUIAgentWithTemplates(base_url="http://test-url", config=config) - results = [] - async for item in agent.stream( - query="hello", session_id="session_123", ui_version="v0.9" - ): - results.append(item) + with mock.patch.object(agent, "_build_runner", return_value=mock_runner): + results = await self._collect_stream(agent, "hello") self.assertEqual(len(results), 1) self.assertTrue(results[0]["is_task_complete"]) @@ -242,9 +255,9 @@ async def test_agent_extractor_empty_parts_handled_safely( self._mock_llm_stream('{"intent": "TEXT_ONLY", "query": "hello"}') ) - # Mock extractor response with None parts - self.mock_extractor.generate_content_async.return_value = MockAsyncIterator( - [MockResponse(MockContent(None))] + mock_runner = mock.MagicMock() + mock_runner.run_async.return_value = MockAsyncIterator( + [MockEvent(content=MockContent(None))] ) config = AgentConfig( @@ -254,17 +267,18 @@ async def test_agent_extractor_empty_parts_handled_safely( ) agent = MAUIAgentWithTemplates(base_url="http://test-url", config=config) - results = [] - async for item in agent.stream( - query="hello", session_id="session_123", ui_version="v0.9" - ): - results.append(item) + with mock.patch.object(agent, "_build_runner", return_value=mock_runner): + results = await self._collect_stream(agent, "hello") self.assertEqual(len(results), 1) self.assertTrue(results[0]["is_task_complete"]) parts = results[0]["parts"] text_comp = self._get_component_by_id(parts, "text-content") - self.assertEqual(text_comp["text"], "") + self.assertEqual( + text_comp["text"], + "I'm sorry, I encountered an issue retrieving location details right" + " now.", + ) @mock.patch(_LITELLM_PATH) async def test_agent_unsupported_intent_fallback(self, mock_lite_llm_class): @@ -307,6 +321,7 @@ def test_init_without_config_uses_default(self): self.assertEqual( agent.config.template_model, "gemini/gemini-3.1-flash-lite" ) + self.assertEqual(agent.config.fallback_mode, FallbackMode.TEXT) @mock.patch(_LITELLM_PATH) async def test_agent_directions_flow(self, mock_lite_llm_class): @@ -623,7 +638,6 @@ async def test_agent_directions_flow_missing_travel_mode_fallback( self._mock_llm_stream("Fallback plain text directions.") ) - mock_runner = mock.MagicMock() mock_fc = MockFunctionCall( name="set_model_response", args={ @@ -648,7 +662,17 @@ async def test_agent_directions_flow_missing_travel_mode_fallback( }, ) mock_event = MockEvent(function_calls=[mock_fc]) - mock_runner.run_async.return_value = MockAsyncIterator([mock_event]) + mock_extractor_runner = mock.MagicMock() + mock_extractor_runner.run_async.return_value = MockAsyncIterator( + [mock_event] + ) + + mock_fallback_runner = mock.MagicMock() + mock_fallback_runner.run_async.return_value = MockAsyncIterator([ + MockEvent( + content=MockContent([MockPart("Fallback plain text directions.")]) + ) + ]) config = AgentConfig( fallback_mode="TEXT", @@ -657,7 +681,11 @@ async def test_agent_directions_flow_missing_travel_mode_fallback( ) agent = MAUIAgentWithTemplates(base_url="http://test-url", config=config) - with mock.patch.object(agent, "_build_runner", return_value=mock_runner): + with mock.patch.object( + agent, + "_build_runner", + side_effect=[mock_extractor_runner, mock_fallback_runner], + ): results = await self._collect_stream(agent, query="directions to work") self.assertEqual(len(results), 1) @@ -833,7 +861,14 @@ async def test_agent_local_search_flow_catalog_validation_failure_fallback( mock_schema_manager.get_catalog.return_value = mock_catalog agent._schema_managers = {"v0.9": mock_schema_manager} - with mock.patch.object(agent, "_build_runner", return_value=mock_runner): + mock_fallback_runner = mock.MagicMock() + mock_fallback_runner.run_async.return_value = MockAsyncIterator( + [MockEvent(content=MockContent([MockPart("Fallback text from LLM.")]))] + ) + + with mock.patch.object( + agent, "_build_runner", side_effect=[mock_runner, mock_fallback_runner] + ): results = await self._collect_stream(agent, "coffee") self.assertEqual(len(results), 1) @@ -849,13 +884,26 @@ async def test_agent_fallback_mode_text_on_extractor_failure( self._setup_mock_llm(mock_lite_llm_class) self._mock_llm_responses( router_resp='{"intent": "LOCAL_SEARCH", "query": "sushi Seattle"}', - extractor_resp="I could not search places right now.", ) - mock_runner = mock.MagicMock() - mock_runner.run_async.return_value = MockAsyncIterator([MockEvent()]) + mock_extractor_runner = mock.MagicMock() + mock_extractor_runner.run_async.return_value = MockAsyncIterator( + [MockEvent()] + ) + mock_fallback_runner = mock.MagicMock() + mock_fallback_runner.run_async.return_value = MockAsyncIterator([ + MockEvent( + content=MockContent( + [MockPart("I could not search places right now.")] + ) + ) + ]) agent = self._setup_agent(fallback_mode="TEXT") - with mock.patch.object(agent, "_build_runner", return_value=mock_runner): + with mock.patch.object( + agent, + "_build_runner", + side_effect=[mock_extractor_runner, mock_fallback_runner], + ): results = await self._collect_stream(agent, "sushi Seattle") self.assertEqual(len(results), 1) @@ -873,13 +921,26 @@ async def test_agent_fallback_mode_dynamic_on_extractor_failure( self._setup_mock_llm(mock_lite_llm_class) self._mock_llm_responses( router_resp='{"intent": "LOCAL_SEARCH", "query": "sushi Seattle"}', - extractor_resp="Fast response after extraction failure.", ) - mock_runner = mock.MagicMock() - mock_runner.run_async.return_value = MockAsyncIterator([MockEvent()]) + mock_extractor_runner = mock.MagicMock() + mock_extractor_runner.run_async.return_value = MockAsyncIterator( + [MockEvent()] + ) + mock_fallback_runner = mock.MagicMock() + mock_fallback_runner.run_async.return_value = MockAsyncIterator([ + MockEvent( + content=MockContent( + [MockPart("Fast response after extraction failure.")] + ) + ) + ]) agent = self._setup_agent(fallback_mode="DYNAMIC") - with mock.patch.object(agent, "_build_runner", return_value=mock_runner): + with mock.patch.object( + agent, + "_build_runner", + side_effect=[mock_extractor_runner, mock_fallback_runner], + ): with mock.patch( "python_agent.agent.MAUIAgent.stream", ) as mock_super_stream: @@ -903,10 +964,18 @@ async def test_agent_other_spatial_fallback_mode_text( self._setup_mock_llm(mock_lite_llm_class) self._mock_llm_responses( router_resp='{"intent": "OTHER_SPATIAL", "query": "weather Yosemite"}', - extractor_resp="The weather in Yosemite is sunny, 75 degrees.", ) + mock_runner = mock.MagicMock() + mock_runner.run_async.return_value = MockAsyncIterator([ + MockEvent( + content=MockContent( + [MockPart("The weather in Yosemite is sunny, 75 degrees.")] + ) + ) + ]) agent = self._setup_agent(fallback_mode="TEXT") - results = await self._collect_stream(agent, "weather Yosemite") + with mock.patch.object(agent, "_build_runner", return_value=mock_runner): + results = await self._collect_stream(agent, "weather Yosemite") self.assertEqual(len(results), 1) self.assertTrue(results[0]["is_task_complete"]) @@ -1033,6 +1102,5 @@ def test_build_dynamic_extractor_agent_handles_file_read_error(self): ) self.assertIn("Base skill instructions", extractor_agent.instruction) - if __name__ == "__main__": unittest.main() diff --git a/client/mobile_core/web_build/index.html b/client/android/web_build/index.html similarity index 100% rename from client/mobile_core/web_build/index.html rename to client/android/web_build/index.html diff --git a/client/mobile_core/web_build/src/core-shell.ts b/client/android/web_build/src/core-shell.ts similarity index 100% rename from client/mobile_core/web_build/src/core-shell.ts rename to client/android/web_build/src/core-shell.ts diff --git a/client/android/web_build/src/main.ts b/client/android/web_build/src/main.ts index 9f520c9..55a5b14 100644 --- a/client/android/web_build/src/main.ts +++ b/client/android/web_build/src/main.ts @@ -15,7 +15,7 @@ import {customElement} from 'lit/decorators.js'; -import {A2UICoreShell} from '../../../mobile_core/web_build/src/core-shell'; +import {A2UICoreShell} from './core-shell'; (window as any)['A2UI_ATTRIBUTION_ID'] = 'gmp_web_maui_v0.1.7_exp,gmp_android_maui_v0.1.7_exp'; diff --git a/client/mobile_core/web_build/tsconfig.json b/client/android/web_build/tsconfig.json similarity index 100% rename from client/mobile_core/web_build/tsconfig.json rename to client/android/web_build/tsconfig.json diff --git a/client/mobile_core/web_build/vite.config.ts b/client/android/web_build/vite.config.ts similarity index 100% rename from client/mobile_core/web_build/vite.config.ts rename to client/android/web_build/vite.config.ts diff --git a/client/ios/web_build/index.html b/client/ios/web_build/index.html new file mode 100644 index 0000000..a3da007 --- /dev/null +++ b/client/ios/web_build/index.html @@ -0,0 +1,39 @@ + + + + + + + + + A2UI Bridge + + + + + + + + diff --git a/client/ios/web_build/src/core-shell.ts b/client/ios/web_build/src/core-shell.ts new file mode 100644 index 0000000..ea3d5a2 --- /dev/null +++ b/client/ios/web_build/src/core-shell.ts @@ -0,0 +1,310 @@ +// +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +import {css, html, LitElement, nothing, type PropertyValues} from 'lit'; +import {state} from 'lit/decorators.js'; + +import {A2UIRenderer, type TimelineItem, themeStyleSheet} from '@googlemaps/a2ui/lit'; + +export interface A2UIComponentNode { + id?: string; + component: string; + child?: string; + children?: string[] | A2UIComponentNode[] | { componentId: string }; + center?: { path?: string; lat?: number; lng?: number }; + [key: string]: unknown; +} + +export interface A2UIMessage { + createSurface?: { surfaceId: string; catalogId: string }; + updateComponents?: { surfaceId?: string; components: A2UIComponentNode[] }; + updateDataModel?: { surfaceId?: string; data?: Record; value?: Record }; + version?: string; + [key: string]: unknown; +} + +export abstract class A2UICoreShell extends LitElement { + @state() + protected timeline: TimelineItem[] = []; + + protected rendererRef = new A2UIRenderer(); + protected globalDataModelRef: Record = {}; + protected resizeObserver!: ResizeObserver; + protected timeoutId: ReturnType | null = null; + + static override styles = [ + themeStyleSheet, + css` + :host { + display: flex; + flex-direction: column; + width: 100%; + height: 100vh; + overflow-y: auto; + overflow-x: hidden; + background: var(--social-bg, #f1f3f4); + } + .chat-messages { + height: auto; + padding: 16px; + overflow: visible; + display: block; + } + .surface-message { + margin-bottom: 16px; + } + .loading { + opacity: 0.5; + text-align: center; + margin-top: 20px; + font-family: sans-serif; + } + `]; + + override connectedCallback() { + super.connectedCallback(); + + // Ensure global typography and Material theme definitions are present on the document + if (!document.adoptedStyleSheets.includes(themeStyleSheet)) { + document.adoptedStyleSheets = [...document.adoptedStyleSheets, themeStyleSheet]; + } + + this.setupResizer(); + this.notifyJsReady(); + } + + override disconnectedCallback() { + super.disconnectedCallback(); + if (this.resizeObserver) { + this.resizeObserver.disconnect(); + } + if (this.timeoutId) { + clearTimeout(this.timeoutId); + } + } + + protected abstract notifyWebpageResized(height: number): void; + protected abstract notifyJsReady(): void; + + processA2uiMessages(json: unknown) { + try { + let messages: A2UIMessage[] = (typeof json === 'string' ? JSON.parse(json) : json) as A2UIMessage[]; + if (typeof messages === 'string') { + messages = JSON.parse(messages as string) as A2UIMessage[]; + } + + if (!Array.isArray(messages)) { + messages = [messages]; + } + + // 1. Auto-fix common LLM hallucinated keys ('latitude' -> 'lat', + // 'title' -> 'label'). + const fixKeys = (obj: any) => { + if (Array.isArray(obj)) { + obj.forEach(fixKeys); + } else if (obj !== null && typeof obj === 'object') { + if (obj.latitude !== undefined) { + obj.lat = obj.latitude; + delete obj.latitude; + } + if (obj.longitude !== undefined) { + obj.lng = obj.longitude; + delete obj.longitude; + } + if (obj.title !== undefined && obj.label === undefined) { + obj.label = obj.title; + delete obj.title; + } + Object.values(obj).forEach(fixKeys); + } + }; + fixKeys(messages); + + + // 2. Track global data model and resolve 'path' references (e.g., Paris + // map bug). We must track the model globally because components and + // data often arrive in separate SSE chunks. + let hasUiInstructions = false; + const UI_KEYS = ['createSurface', 'updateComponents', 'updateDataModel', 'deleteSurface', 'beginRendering', 'surfaceUpdate']; + + messages.forEach((item) => { + if (UI_KEYS.some(key => Object.prototype.hasOwnProperty.call(item, key))) { + hasUiInstructions = true; + } + if (item.updateDataModel) { + const payload = item.updateDataModel.data || item.updateDataModel.value; + if (payload) { + this.globalDataModelRef = { ...this.globalDataModelRef, ...payload }; + } + } + }); + + // 3. Deduplication: Only process this chunk in the WebView IF it + // contains actual UI instructions. If it's just pure conversational + // text, we ignore it here because Android's native bubble handles it. + if (!hasUiInstructions) { + return; + } + + const resolvePath = (pathStr: string) => { + if (!pathStr || !pathStr.startsWith('/')) return null; + let parts = pathStr.split('/').filter(Boolean); + let curr: any = this.globalDataModelRef; + for (let p of parts) { + if (curr && Object.prototype.hasOwnProperty.call(curr, p)) + curr = curr[p]; + else + return null; + } + return curr; + }; + + const fixGoogleMap = (comp: any) => { + if (comp.component === 'GoogleMap') { + if (comp.center && comp.center.path) { + let resolved = resolvePath(comp.center.path); + if (resolved) comp.center = resolved; + } + } + if (comp.children && Array.isArray(comp.children)) { + comp.children.forEach((c: any) => { + if (typeof c === 'object') fixGoogleMap(c); + }); + } + }; + + messages.forEach((item: any) => { + if (item.updateComponents && item.updateComponents.components) { + let comps = item.updateComponents.components; + comps.forEach(fixGoogleMap); + + // Ensure 'root' Column exists for the A2UI Renderer + let hasRoot = comps.some((c: any) => c.id === 'root'); + if (!hasRoot && comps.length > 0) { + // If no "root" exists, generating a new "root" container. + let referencedChildIds = new Set(); + comps.forEach((c: any) => { + if (typeof c.child === 'string') + referencedChildIds.add(c.child); + if (c.children) { + if (Array.isArray(c.children)) { + c.children.forEach((child: any) => { + if (typeof child === 'string') + referencedChildIds.add(child); + else if (child && typeof child === 'object' && child.id) + referencedChildIds.add(child.id); + }); + } else if ( + typeof c.children === 'object' && + c.children.componentId) { + referencedChildIds.add(c.children.componentId); + } + } + }); + + // Filter the components that are not claimed as a child by anyone + let rootChildren = + comps + .filter((c: any) => c.id && !referencedChildIds.has(c.id)) + .map((c: any) => c.id); + + comps.unshift( + {id: 'root', component: 'Column', children: rootChildren}); + } + } + }); + + // Injects a mandatory createSurface command if missing so isolated + // WebView chunks won't render blank. + const hasCreate = messages.some((item) => item.createSurface); + if (!hasCreate) { + let surfaceId: string | undefined = undefined; + for (const m of messages) { + if (m.updateComponents) surfaceId = m.updateComponents.surfaceId; + if (m.updateDataModel) surfaceId = m.updateDataModel.surfaceId; + if (surfaceId) break; + } + if (surfaceId) { + messages.unshift({ + createSurface: { + surfaceId, + catalogId: 'a2ui://maps-agentic-ui-catalog.json' + }, + version: 'v0.9' + }); + } + } + + this.rendererRef.processResponse(messages.map((msg) => ({ type: "a2ui", message: msg }))); + this.timeline = [...this.rendererRef.timeline]; + } catch (e) { + console.error("Failed to process A2UI JSON:", e); + } + } + + private setupResizer() { + this.resizeObserver = new ResizeObserver(() => { + if (this.timeoutId) { + clearTimeout(this.timeoutId); + } + this.timeoutId = setTimeout(() => { + const rootWrapper = this.shadowRoot?.querySelector('.chat-messages'); + if (rootWrapper) { + const newHeight = rootWrapper.scrollHeight; + this.notifyWebpageResized(newHeight); + } + }, 100); + }); + + + const chatMessagesEl = this.shadowRoot?.querySelector('.chat-messages'); + if (chatMessagesEl) { + this.resizeObserver.observe(chatMessagesEl); + } + } + + protected override updated(changedProperties: PropertyValues) { + super.updated(changedProperties); + + // Fallback: If setupResizer ran before shadow DOM rendered chat-messages, observe it now. + const chatMessagesEl = this.shadowRoot?.querySelector('.chat-messages'); + if (chatMessagesEl && this.resizeObserver) { + this.resizeObserver.observe(chatMessagesEl); + } + } + + override render() { + return html` +
+ + ${this.timeline.length === 0 ? html`

Waiting for payload...

` : nothing} + ${this.timeline.map((item) => { + if (item.type === 'surface') { + const surface = this.rendererRef.getSurface(item.surfaceId); + if (!surface) return nothing; + return html` +
+ +
+ `; + } + return nothing; + })} + +
+ `; + } +} diff --git a/client/ios/web_build/src/main.ts b/client/ios/web_build/src/main.ts index 88812eb..30a2236 100644 --- a/client/ios/web_build/src/main.ts +++ b/client/ios/web_build/src/main.ts @@ -16,7 +16,7 @@ import {customElement} from 'lit/decorators.js'; import {type PropertyValues} from 'lit'; -import {A2UICoreShell} from '../../../mobile_core/web_build/src/core-shell'; +import {A2UICoreShell} from './core-shell'; (window as any)['A2UI_ATTRIBUTION_ID'] = 'gmp_web_maui_v0.1.7_exp,gmp_ios_maui_v0.1.7_exp'; diff --git a/client/ios/web_build/tsconfig.json b/client/ios/web_build/tsconfig.json new file mode 100644 index 0000000..398398c --- /dev/null +++ b/client/ios/web_build/tsconfig.json @@ -0,0 +1,22 @@ +{ + "compilerOptions": { + "target": "es2022", + "module": "esnext", + "lib": ["es2023", "DOM", "DOM.Iterable"], + "skipLibCheck": true, + "experimentalDecorators": true, + "useDefineForClassFields": false, + "rootDir": ".", + "outDir": "dist", + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "resolveJsonModule": true, + "isolatedModules": true, + "noEmit": true, + "strict": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noFallthroughCasesInSwitch": true + }, + "include": ["src/**/*.ts"] +} diff --git a/client/ios/web_build/vite.config.ts b/client/ios/web_build/vite.config.ts new file mode 100644 index 0000000..83ce639 --- /dev/null +++ b/client/ios/web_build/vite.config.ts @@ -0,0 +1,33 @@ +// +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +import {defineConfig} from 'vite'; +import {viteSingleFile} from 'vite-plugin-singlefile'; + +export default defineConfig({ + plugins: [viteSingleFile()], + resolve: { + dedupe: ['lit', '@lit/context', '@lit-labs/signals'], + }, + build: { + outDir: 'dist', + rollupOptions: { + input: { + app: 'index.html', + }, + }, + }, +}); diff --git a/client/mobile_core/web_build/build_defs.bzl b/client/mobile_core/web_build/build_defs.bzl deleted file mode 100644 index 228dbd7..0000000 --- a/client/mobile_core/web_build/build_defs.bzl +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2026 Google LLC - -"""Shared build definitions for A2UI mobile web builds.""" - -def generate_mobile_index_html(name, html_template, js_bundle, out_html): - """Takes a base HTML template and inline-injects a compiled JS bundle. - - Args: - name: Name of the generated rule. - html_template: The base index.html target to use as a shell. - js_bundle: The compiled JS bundle target (e.g. from closure_js_binary). - out_html: The filename of the resulting self-contained HTML file. - """ - native.genrule( - name = name, - srcs = [html_template, js_bundle], - outs = [out_html], - cmd = """ - # Extract the .js file from the bundle outputs. - for f in $(locations {js_bundle}); do - case $$f in *.js) JS_FILE=$$f ;; esac - done - - # Wrap the raw JS inside " - ) > tmp_inject_js.js - - # Replace the old module