diff --git a/ai-logic/firebase-ai-ondevice-interop/CHANGELOG.md b/ai-logic/firebase-ai-ondevice-interop/CHANGELOG.md index 4c8e0254149..a537d7cbf5f 100644 --- a/ai-logic/firebase-ai-ondevice-interop/CHANGELOG.md +++ b/ai-logic/firebase-ai-ondevice-interop/CHANGELOG.md @@ -1,5 +1,7 @@ # Unreleased +- [feature] Added support for function calling in on-device content generation + requests and responses. (#8758) - [changed] **Breaking Change**: Updated `GenerateContentRequest` to accept a `List` instead of a single `TextPart` and optional `ImagePart`. (#8754) diff --git a/ai-logic/firebase-ai-ondevice-interop/api.txt b/ai-logic/firebase-ai-ondevice-interop/api.txt index f90faa169e4..9e5bd17123b 100644 --- a/ai-logic/firebase-ai-ondevice-interop/api.txt +++ b/ai-logic/firebase-ai-ondevice-interop/api.txt @@ -3,9 +3,12 @@ package com.google.firebase.ai.ondevice.interop { public final class Candidate { ctor public Candidate(String text, com.google.firebase.ai.ondevice.interop.FinishReason finishReason); + ctor public Candidate(String text, com.google.firebase.ai.ondevice.interop.FinishReason finishReason, java.util.List functionCalls = emptyList()); method public com.google.firebase.ai.ondevice.interop.FinishReason getFinishReason(); + method public java.util.List getFunctionCalls(); method public String getText(); property public final com.google.firebase.ai.ondevice.interop.FinishReason finishReason; + property public final java.util.List functionCalls; property public final String text; } @@ -78,8 +81,73 @@ package com.google.firebase.ai.ondevice.interop { ctor public FirebaseAIOnDeviceUnknownException(String message, Throwable? cause = null); } + public final class FunctionCallPart implements com.google.firebase.ai.ondevice.interop.Part { + ctor public FunctionCallPart(String name, java.util.Map args); + ctor public FunctionCallPart(String name, java.util.Map args, String? id = null); + method public java.util.Map getArgs(); + method public String? getId(); + method public String getName(); + property public final java.util.Map args; + property public final String? id; + property public final String name; + } + + public final class FunctionDeclaration { + ctor public FunctionDeclaration(String name, String description); + ctor public FunctionDeclaration(String name, String description, com.google.firebase.ai.ondevice.interop.FunctionSchema? parameters = null); + method public String getDescription(); + method public String getName(); + method public com.google.firebase.ai.ondevice.interop.FunctionSchema? getParameters(); + property public final String description; + property public final String name; + property public final com.google.firebase.ai.ondevice.interop.FunctionSchema? parameters; + } + + public final class FunctionResponsePart implements com.google.firebase.ai.ondevice.interop.Part { + ctor public FunctionResponsePart(String name, java.util.Map response); + ctor public FunctionResponsePart(String name, java.util.Map response, String? id = null); + method public String? getId(); + method public String getName(); + method public java.util.Map getResponse(); + property public final String? id; + property public final String name; + property public final java.util.Map response; + } + + public final class FunctionSchema { + ctor public FunctionSchema(String type); + ctor public FunctionSchema(String type, String? description = null); + ctor public FunctionSchema(String type, String? description = null, java.util.Map properties = emptyMap()); + ctor public FunctionSchema(String type, String? description = null, java.util.Map properties = emptyMap(), java.util.List required = emptyList()); + ctor public FunctionSchema(String type, String? description = null, java.util.Map properties = emptyMap(), java.util.List required = emptyList(), java.util.List? enum = null); + ctor public FunctionSchema(String type, String? description = null, java.util.Map properties = emptyMap(), java.util.List required = emptyList(), java.util.List? enum = null, Boolean? nullable = null); + ctor public FunctionSchema(String type, String? description = null, java.util.Map properties = emptyMap(), java.util.List required = emptyList(), java.util.List? enum = null, Boolean? nullable = null, com.google.firebase.ai.ondevice.interop.FunctionSchema? items = null); + method public String? getDescription(); + method public java.util.List? getEnum(); + method public com.google.firebase.ai.ondevice.interop.FunctionSchema? getItems(); + method public Boolean? getNullable(); + method public java.util.Map getProperties(); + method public java.util.List getRequired(); + method public String getType(); + property public final String? description; + property public final java.util.List? enum; + property public final com.google.firebase.ai.ondevice.interop.FunctionSchema? items; + property public final Boolean? nullable; + property public final java.util.Map properties; + property public final java.util.List required; + property public final String type; + } + public final class GenerateContentRequest { + ctor public GenerateContentRequest(java.util.List parts); + ctor public GenerateContentRequest(java.util.List parts, Float? temperature = null); + ctor public GenerateContentRequest(java.util.List parts, Float? temperature = null, Integer? topK = null); + ctor public GenerateContentRequest(java.util.List parts, Float? temperature = null, Integer? topK = null, Integer? seed = null); + ctor public GenerateContentRequest(java.util.List parts, Float? temperature = null, Integer? topK = null, Integer? seed = null, Integer? candidateCount = null); + ctor public GenerateContentRequest(java.util.List parts, Float? temperature = null, Integer? topK = null, Integer? seed = null, Integer? candidateCount = null, Integer? maxOutputTokens = null); + ctor public GenerateContentRequest(java.util.List parts, Float? temperature = null, Integer? topK = null, Integer? seed = null, Integer? candidateCount = null, Integer? maxOutputTokens = null, com.google.firebase.ai.ondevice.interop.TextPart? systemInstruction = null); ctor public GenerateContentRequest(java.util.List parts, Float? temperature = null, Integer? topK = null, Integer? seed = null, Integer? candidateCount = null, Integer? maxOutputTokens = null, com.google.firebase.ai.ondevice.interop.TextPart? systemInstruction = null, Boolean? enableThinking = null); + ctor public GenerateContentRequest(java.util.List parts, Float? temperature = null, Integer? topK = null, Integer? seed = null, Integer? candidateCount = null, Integer? maxOutputTokens = null, com.google.firebase.ai.ondevice.interop.TextPart? systemInstruction = null, Boolean? enableThinking = null, java.util.List tools = emptyList()); method public Integer? getCandidateCount(); method public Boolean? getEnableThinking(); method public Integer? getMaxOutputTokens(); @@ -87,6 +155,7 @@ package com.google.firebase.ai.ondevice.interop { method public Integer? getSeed(); method public com.google.firebase.ai.ondevice.interop.TextPart? getSystemInstruction(); method public Float? getTemperature(); + method public java.util.List getTools(); method public Integer? getTopK(); property public final Integer? candidateCount; property public final Boolean? enableThinking; @@ -95,6 +164,7 @@ package com.google.firebase.ai.ondevice.interop { property public final Integer? seed; property public final com.google.firebase.ai.ondevice.interop.TextPart? systemInstruction; property public final Float? temperature; + property public final java.util.List tools; property public final Integer? topK; } @@ -133,6 +203,7 @@ package com.google.firebase.ai.ondevice.interop { method public suspend Object? getBaseModelName(kotlin.coroutines.Continuation); method public suspend Object? getTokenLimit(kotlin.coroutines.Continuation); method public suspend Object? isAvailable(kotlin.coroutines.Continuation); + method public suspend Object? isFunctionCallingAvailable(kotlin.coroutines.Continuation); method public suspend Object? isThinkingModeAvailable(kotlin.coroutines.Continuation); method public suspend Object? warmup(kotlin.coroutines.Continuation); } diff --git a/ai-logic/firebase-ai-ondevice-interop/src/main/kotlin/com/google/firebase/ai/ondevice/interop/Candidate.kt b/ai-logic/firebase-ai-ondevice-interop/src/main/kotlin/com/google/firebase/ai/ondevice/interop/Candidate.kt index 0b2d54f94a5..89fc99faa46 100644 --- a/ai-logic/firebase-ai-ondevice-interop/src/main/kotlin/com/google/firebase/ai/ondevice/interop/Candidate.kt +++ b/ai-logic/firebase-ai-ondevice-interop/src/main/kotlin/com/google/firebase/ai/ondevice/interop/Candidate.kt @@ -26,4 +26,10 @@ public class FinishReason private constructor(public val name: String) { } /** A `Candidate` represents a single response generated by the model for a given request. */ -public class Candidate(public val text: String, public val finishReason: FinishReason) {} +public class Candidate +@JvmOverloads +constructor( + public val text: String, + public val finishReason: FinishReason, + public val functionCalls: List = emptyList() +) diff --git a/ai-logic/firebase-ai-ondevice-interop/src/main/kotlin/com/google/firebase/ai/ondevice/interop/FunctionDeclaration.kt b/ai-logic/firebase-ai-ondevice-interop/src/main/kotlin/com/google/firebase/ai/ondevice/interop/FunctionDeclaration.kt new file mode 100644 index 00000000000..dfcd434275d --- /dev/null +++ b/ai-logic/firebase-ai-ondevice-interop/src/main/kotlin/com/google/firebase/ai/ondevice/interop/FunctionDeclaration.kt @@ -0,0 +1,56 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.firebase.ai.ondevice.interop + +/** + * Defines a function that the model can use as a tool for on-device inference. + * + * @property name The name of the function. + * @property description The description of what the function does. + * @property parameters The schema describing the parameters the function accepts. + */ +public class FunctionDeclaration +@JvmOverloads +constructor( + public val name: String, + public val description: String, + public val parameters: FunctionSchema? = null +) + +/** + * Schema defining the structure of function parameters for on-device function calling. + * + * @property type The data type of the schema (for example, OBJECT, STRING, INTEGER, BOOLEAN, + * ARRAY). + * @property description An optional description of what the value represents. + * @property properties The map of property names to schemas when type is OBJECT. + * @property required The list of required property names when type is OBJECT. + * @property enum The list of allowed string values when applicable. + * @property nullable Indicates whether the value can be null. + * @property items The schema for array elements when type is ARRAY. + */ +public class FunctionSchema +@JvmOverloads +constructor( + public val type: String, + public val description: String? = null, + public val properties: Map = emptyMap(), + public val required: List = emptyList(), + public val enum: List? = null, + public val nullable: Boolean? = null, + public val items: FunctionSchema? = null +) diff --git a/ai-logic/firebase-ai-ondevice-interop/src/main/kotlin/com/google/firebase/ai/ondevice/interop/GenerateContentRequest.kt b/ai-logic/firebase-ai-ondevice-interop/src/main/kotlin/com/google/firebase/ai/ondevice/interop/GenerateContentRequest.kt index ea98f8c24fe..3c7ad3f703e 100644 --- a/ai-logic/firebase-ai-ondevice-interop/src/main/kotlin/com/google/firebase/ai/ondevice/interop/GenerateContentRequest.kt +++ b/ai-logic/firebase-ai-ondevice-interop/src/main/kotlin/com/google/firebase/ai/ondevice/interop/GenerateContentRequest.kt @@ -31,8 +31,11 @@ package com.google.firebase.ai.ondevice.interop * response. * @property systemInstruction Instructions that direct the model to behave a certain way. * @property enableThinking Whether to enable thinking mode for the model. + * @property tools The list of function declaration tools available to the model. */ -public class GenerateContentRequest( +public class GenerateContentRequest +@JvmOverloads +constructor( public val parts: List, public val temperature: Float? = null, public val topK: Int? = null, @@ -41,4 +44,5 @@ public class GenerateContentRequest( public val maxOutputTokens: Int? = null, public val systemInstruction: TextPart? = null, public val enableThinking: Boolean? = null, + public val tools: List = emptyList(), ) {} diff --git a/ai-logic/firebase-ai-ondevice-interop/src/main/kotlin/com/google/firebase/ai/ondevice/interop/GenerativeModel.kt b/ai-logic/firebase-ai-ondevice-interop/src/main/kotlin/com/google/firebase/ai/ondevice/interop/GenerativeModel.kt index d6f96ace9b6..2f5baa980c1 100644 --- a/ai-logic/firebase-ai-ondevice-interop/src/main/kotlin/com/google/firebase/ai/ondevice/interop/GenerativeModel.kt +++ b/ai-logic/firebase-ai-ondevice-interop/src/main/kotlin/com/google/firebase/ai/ondevice/interop/GenerativeModel.kt @@ -43,6 +43,13 @@ public interface GenerativeModel { */ public suspend fun isThinkingModeAvailable(): Boolean + /** + * Checks whether function calling is available on the model. + * + * @return `true` if function calling is available, `false` otherwise. + */ + public suspend fun isFunctionCallingAvailable(): Boolean + /** * Generates new content from the input [GenerateContentRequest] given to the model as a prompt. * diff --git a/ai-logic/firebase-ai-ondevice-interop/src/main/kotlin/com/google/firebase/ai/ondevice/interop/Part.kt b/ai-logic/firebase-ai-ondevice-interop/src/main/kotlin/com/google/firebase/ai/ondevice/interop/Part.kt index feeda34760b..d1bb25617eb 100644 --- a/ai-logic/firebase-ai-ondevice-interop/src/main/kotlin/com/google/firebase/ai/ondevice/interop/Part.kt +++ b/ai-logic/firebase-ai-ondevice-interop/src/main/kotlin/com/google/firebase/ai/ondevice/interop/Part.kt @@ -26,3 +26,50 @@ public class TextPart(public val text: String) : Part /** Represents image data. */ public class ImagePart(public val bitmap: Bitmap) : Part + +/** + * Represents function call name and parameters emitted by the model or sent in conversation + * history. + */ +public class FunctionCallPart +@JvmOverloads +constructor( + public val name: String, + public val args: Map, + public val id: String? = null +) : Part { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is FunctionCallPart) return false + return name == other.name && args == other.args && id == other.id + } + + override fun hashCode(): Int { + var result = name.hashCode() + result = 31 * result + args.hashCode() + result = 31 * result + (id?.hashCode() ?: 0) + return result + } +} + +/** Represents function call output to be returned to the model. */ +public class FunctionResponsePart +@JvmOverloads +constructor( + public val name: String, + public val response: Map, + public val id: String? = null +) : Part { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is FunctionResponsePart) return false + return name == other.name && response == other.response && id == other.id + } + + override fun hashCode(): Int { + var result = name.hashCode() + result = 31 * result + response.hashCode() + result = 31 * result + (id?.hashCode() ?: 0) + return result + } +} diff --git a/ai-logic/firebase-ai-ondevice/CHANGELOG.md b/ai-logic/firebase-ai-ondevice/CHANGELOG.md index 781c41d62f9..8633f8e5705 100644 --- a/ai-logic/firebase-ai-ondevice/CHANGELOG.md +++ b/ai-logic/firebase-ai-ondevice/CHANGELOG.md @@ -1,5 +1,7 @@ # Unreleased +- [feature] Added support for function calling in on-device content generation + requests and responses. (#8758) - [feature] Added support for multiple `TextPart` and `ImagePart` instances in on-device content generation requests. (#8754) diff --git a/ai-logic/firebase-ai-ondevice/src/main/kotlin/com/google/firebase/ai/ondevice/Converters.kt b/ai-logic/firebase-ai-ondevice/src/main/kotlin/com/google/firebase/ai/ondevice/Converters.kt index 512847fb193..6a041b521b0 100644 --- a/ai-logic/firebase-ai-ondevice/src/main/kotlin/com/google/firebase/ai/ondevice/Converters.kt +++ b/ai-logic/firebase-ai-ondevice/src/main/kotlin/com/google/firebase/ai/ondevice/Converters.kt @@ -22,11 +22,11 @@ import com.google.firebase.ai.ondevice.interop.DownloadStatusInterop import com.google.firebase.ai.ondevice.interop.FinishReason import com.google.firebase.ai.ondevice.interop.FirebaseAIOnDeviceException import com.google.firebase.ai.ondevice.interop.FirebaseAIOnDeviceInvalidRequestException -import com.google.firebase.ai.ondevice.interop.FirebaseAIOnDeviceUnknownException import com.google.firebase.ai.ondevice.interop.GenerateContentResponse import com.google.firebase.ai.ondevice.interop.GenerationConfig import com.google.firebase.ai.ondevice.interop.ModelConfig import com.google.firebase.ai.ondevice.interop.OnDeviceModelStatusInterop +import com.google.mlkit.genai.common.FunctionArgValue import com.google.mlkit.genai.prompt.GenerateContentRequest import com.google.mlkit.genai.prompt.ImagePart import com.google.mlkit.genai.prompt.ModelPreference @@ -38,6 +38,7 @@ import com.google.mlkit.genai.prompt.content import com.google.mlkit.genai.prompt.generateContentRequest import com.google.mlkit.genai.prompt.generationConfig import com.google.mlkit.genai.prompt.modelConfig +import com.google.mlkit.genai.schema.tools.FunctionType // ==================================== // `Part` converter extension functions @@ -46,6 +47,8 @@ internal fun com.google.firebase.ai.ondevice.interop.Part.toMlKit(): Part = when (this) { is com.google.firebase.ai.ondevice.interop.TextPart -> toMlKit() is com.google.firebase.ai.ondevice.interop.ImagePart -> toMlKit() + is com.google.firebase.ai.ondevice.interop.FunctionCallPart -> toMlKit() + is com.google.firebase.ai.ondevice.interop.FunctionResponsePart -> toMlKit() else -> throw FirebaseAIOnDeviceInvalidRequestException( IllegalArgumentException("Unsupported part type: ${this::class.java.simpleName}") @@ -60,6 +63,36 @@ internal fun com.google.firebase.ai.ondevice.interop.TextPart.toMlKitSystemInstr internal fun com.google.firebase.ai.ondevice.interop.ImagePart.toMlKit(): ImagePart = ImagePart(bitmap) +internal fun com.google.firebase.ai.ondevice.interop.FunctionCallPart.toMlKit(): + com.google.mlkit.genai.prompt.FunctionCallPart = + com.google.mlkit.genai.prompt.FunctionCallPart( + com.google.mlkit.genai.common.FunctionCall( + name = name, + arguments = args.mapValues { toFunctionArgValue(it.value) }, + id = id + ) + ) + +internal fun com.google.firebase.ai.ondevice.interop.FunctionResponsePart.toMlKit(): + com.google.mlkit.genai.prompt.FunctionResponsePart = + com.google.mlkit.genai.prompt.FunctionResponsePart(id = id, name = name, response = response) + +private fun toFunctionArgValue(value: Any?): FunctionArgValue = + when (value) { + is Boolean -> FunctionArgValue.BoolValue(value) + is Int -> FunctionArgValue.IntValue(value) + is Long -> FunctionArgValue.LongValue(value) + is Double -> FunctionArgValue.DoubleValue(value) + is Float -> FunctionArgValue.DoubleValue(value.toDouble()) + is String -> FunctionArgValue.StringValue(value) + is List<*> -> FunctionArgValue.ListValue(value.map { toFunctionArgValue(it) }) + is Map<*, *> -> + FunctionArgValue.MapValue( + value.entries.associate { it.key.toString() to toFunctionArgValue(it.value) } + ) + else -> FunctionArgValue.StringValue(value?.toString() ?: "") + } + // ============================================ // `CountTokens*` converter extension functions // ============================================ @@ -70,7 +103,41 @@ internal fun com.google.mlkit.genai.prompt.CountTokensResponse.toInterop(): Coun // `Candidate` converter extension functions // ========================================= internal fun com.google.mlkit.genai.prompt.Candidate.toInterop(): Candidate = - Candidate(text, finishReason?.toInterop() ?: FinishReason.OTHER) + Candidate( + text = text, + finishReason = finishReason?.toInterop() ?: FinishReason.OTHER, + functionCalls = functionCalls?.map { it.toInterop() } ?: emptyList() + ) + +internal fun com.google.mlkit.genai.common.FunctionCall.toInterop(): + com.google.firebase.ai.ondevice.interop.FunctionCallPart = + com.google.firebase.ai.ondevice.interop.FunctionCallPart( + name = name, + args = arguments.mapValues { it.value.toAny() }, + id = id + ) + +internal fun com.google.mlkit.genai.prompt.FunctionCallPart.toInterop(): + com.google.firebase.ai.ondevice.interop.FunctionCallPart = functionCall.toInterop() + +internal fun com.google.mlkit.genai.prompt.FunctionResponsePart.toInterop(): + com.google.firebase.ai.ondevice.interop.FunctionResponsePart = + com.google.firebase.ai.ondevice.interop.FunctionResponsePart( + name = name, + response = response, + id = id + ) + +internal fun FunctionArgValue.toAny(): Any? = + when (this) { + is FunctionArgValue.BoolValue -> value + is FunctionArgValue.IntValue -> value + is FunctionArgValue.LongValue -> value + is FunctionArgValue.DoubleValue -> value + is FunctionArgValue.StringValue -> value + is FunctionArgValue.ListValue -> values.map { it.toAny() } + is FunctionArgValue.MapValue -> map.mapValues { it.value.toAny() } + } private fun Int.toInterop(): FinishReason = when (this) { @@ -79,6 +146,39 @@ private fun Int.toInterop(): FinishReason = else -> FinishReason.OTHER } +// ================================================ +// `Tool` converter extension functions +// ================================================ +internal fun com.google.firebase.ai.ondevice.interop.FunctionDeclaration.toMlKit(): + com.google.mlkit.genai.schema.tools.FunctionDeclaration = + com.google.mlkit.genai.schema.tools.FunctionDeclaration( + name = name, + description = description, + parameters = parameters?.toMlKit() + ) + +internal fun com.google.firebase.ai.ondevice.interop.FunctionSchema.toMlKit(): + com.google.mlkit.genai.schema.tools.FunctionSchema = + com.google.mlkit.genai.schema.tools.FunctionSchema( + type = + when (type.uppercase()) { + "STRING" -> FunctionType.STRING + "NUMBER" -> FunctionType.NUMBER + "INTEGER" -> FunctionType.INTEGER + "BOOLEAN" -> FunctionType.BOOLEAN + "ARRAY" -> FunctionType.ARRAY + "OBJECT" -> FunctionType.OBJECT + "NULL" -> FunctionType.NULL + else -> FunctionType.UNSPECIFIED + }, + description = description, + properties = properties.mapValues { it.value.toMlKit() }.ifEmpty { null }, + required = required.ifEmpty { null }, + enum = enum, + nullable = nullable, + items = items?.toMlKit() + ) + // ================================================ // `GenerateContent*` converter extension functions // ================================================ @@ -96,6 +196,9 @@ internal fun com.google.firebase.ai.ondevice.interop.GenerateContentRequest.toMl seed = this@toMlKit.seed systemInstruction = this@toMlKit.systemInstruction?.toMlKitSystemInstruction() enableThinking = this@toMlKit.enableThinking ?: false + if (this@toMlKit.tools.isNotEmpty()) { + setTools(this@toMlKit.tools.map { it.toMlKit() }) + } } } catch (e: IllegalArgumentException) { throw FirebaseAIOnDeviceInvalidRequestException(e) @@ -152,10 +255,6 @@ internal fun com.google.mlkit.genai.common.DownloadStatus.toInterop(): DownloadS DownloadStatusInterop.DownloadCompleted() is com.google.mlkit.genai.common.DownloadStatus.DownloadFailed -> DownloadStatusInterop.DownloadFailed(FirebaseAIOnDeviceException.from(e)) - else -> - DownloadStatusInterop.DownloadFailed( - FirebaseAIOnDeviceUnknownException("Unknown download status") - ) } internal fun Int.toInteropStatus(): OnDeviceModelStatusInterop = diff --git a/ai-logic/firebase-ai-ondevice/src/main/kotlin/com/google/firebase/ai/ondevice/GenerativeModelImpl.kt b/ai-logic/firebase-ai-ondevice/src/main/kotlin/com/google/firebase/ai/ondevice/GenerativeModelImpl.kt index cccf01a9630..a939e083183 100644 --- a/ai-logic/firebase-ai-ondevice/src/main/kotlin/com/google/firebase/ai/ondevice/GenerativeModelImpl.kt +++ b/ai-logic/firebase-ai-ondevice/src/main/kotlin/com/google/firebase/ai/ondevice/GenerativeModelImpl.kt @@ -29,6 +29,7 @@ import com.google.firebase.ai.ondevice.interop.OnDeviceModelStatusInterop import com.google.mlkit.genai.common.FeatureStatus import com.google.mlkit.genai.common.GenAiException import com.google.mlkit.genai.common.GenAiException.ErrorCode +import com.google.mlkit.genai.prompt.Feature import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.catch import kotlinx.coroutines.flow.map @@ -47,7 +48,14 @@ internal class GenerativeModelImpl( override suspend fun isThinkingModeAvailable(): Boolean = try { - mlkitModel.isThinkingModeAvailable() + mlkitModel.getAvailableFeatures().contains(Feature.THINKING_MODE) + } catch (e: GenAiException) { + throw getMappingException(e) + } + + override suspend fun isFunctionCallingAvailable(): Boolean = + try { + mlkitModel.getAvailableFeatures().contains(Feature.FUNCTION_CALLING) } catch (e: GenAiException) { throw getMappingException(e) } diff --git a/ai-logic/firebase-ai-ondevice/src/test/kotlin/com/google/firebase/ai/ondevice/ConvertersTest.kt b/ai-logic/firebase-ai-ondevice/src/test/kotlin/com/google/firebase/ai/ondevice/ConvertersTest.kt index 0b2286f0cc3..d15160aaa91 100644 --- a/ai-logic/firebase-ai-ondevice/src/test/kotlin/com/google/firebase/ai/ondevice/ConvertersTest.kt +++ b/ai-logic/firebase-ai-ondevice/src/test/kotlin/com/google/firebase/ai/ondevice/ConvertersTest.kt @@ -19,11 +19,20 @@ package com.google.firebase.ai.ondevice import com.google.common.truth.Truth.assertThat import com.google.firebase.ai.ondevice.interop.FinishReason as InteropFinishReason import com.google.firebase.ai.ondevice.interop.FirebaseAIOnDeviceInvalidRequestException +import com.google.firebase.ai.ondevice.interop.FunctionCallPart as InteropFunctionCallPart +import com.google.firebase.ai.ondevice.interop.FunctionDeclaration as InteropFunctionDeclaration +import com.google.firebase.ai.ondevice.interop.FunctionResponsePart as InteropFunctionResponsePart +import com.google.firebase.ai.ondevice.interop.FunctionSchema as InteropFunctionSchema import com.google.firebase.ai.ondevice.interop.GenerateContentRequest as InteropGenerateContentRequest import com.google.firebase.ai.ondevice.interop.TextPart as InteropTextPart +import com.google.mlkit.genai.common.FunctionArgValue +import com.google.mlkit.genai.common.FunctionCall import com.google.mlkit.genai.prompt.Candidate import com.google.mlkit.genai.prompt.CountTokensResponse +import com.google.mlkit.genai.prompt.FunctionCallPart +import com.google.mlkit.genai.prompt.FunctionResponsePart import com.google.mlkit.genai.prompt.TextPart +import com.google.mlkit.genai.schema.tools.FunctionType import org.junit.Assert.assertThrows import org.junit.Test import org.mockito.Mockito.mock @@ -41,6 +50,59 @@ internal class ConvertersTest { assertThat(mlKitTextPart.textString).isEqualTo("hello") } + @Test + fun `InteropParts toMlKit should convert function call parts correctly`() { + val interopPart = + InteropFunctionCallPart("fetchWeather", mapOf("city" to "London", "temp" to 20L), "id-123") + val mlKitPart = interopPart.toMlKit() + assertThat(mlKitPart.functionCall.name).isEqualTo("fetchWeather") + assertThat(mlKitPart.functionCall.id).isEqualTo("id-123") + assertThat((mlKitPart.functionCall.arguments["city"] as FunctionArgValue.StringValue).value) + .isEqualTo("London") + assertThat((mlKitPart.functionCall.arguments["temp"] as FunctionArgValue.LongValue).value) + .isEqualTo(20L) + } + + @Test + fun `MlKit FunctionCallPart toInterop should convert correctly`() { + val mlKitPart = + FunctionCallPart( + FunctionCall( + name = "fetchWeather", + arguments = mapOf("city" to FunctionArgValue.StringValue("London")), + id = "id-123" + ) + ) + val interopPart = mlKitPart.toInterop() + assertThat(interopPart.name).isEqualTo("fetchWeather") + assertThat(interopPart.id).isEqualTo("id-123") + assertThat(interopPart.args).isEqualTo(mapOf("city" to "London")) + } + + @Test + fun `InteropParts toMlKit should convert function response parts correctly`() { + val interopPart = + InteropFunctionResponsePart("fetchWeather", mapOf("result" to "sunny"), "id-123") + val mlKitPart = interopPart.toMlKit() + assertThat(mlKitPart.name).isEqualTo("fetchWeather") + assertThat(mlKitPart.id).isEqualTo("id-123") + assertThat(mlKitPart.response).isEqualTo(mapOf("result" to "sunny")) + } + + @Test + fun `MlKit FunctionResponsePart toInterop should convert correctly`() { + val mlKitPart = + FunctionResponsePart( + id = "id-123", + name = "fetchWeather", + response = mapOf("result" to "sunny") + ) + val interopPart = mlKitPart.toInterop() + assertThat(interopPart.name).isEqualTo("fetchWeather") + assertThat(interopPart.id).isEqualTo("id-123") + assertThat(interopPart.response).isEqualTo(mapOf("result" to "sunny")) + } + @Test fun `CountTokensResponse toInterop should convert correctly`() { val mlKitCountTokensResponse = CountTokensResponse(42) @@ -72,6 +134,59 @@ internal class ConvertersTest { assertThat(interopCandidate.finishReason).isEqualTo(InteropFinishReason.OTHER) } + @Test + fun `Candidate toInterop should convert functionCalls correctly`() { + val mlKitFunctionCall = + FunctionCall( + name = "fetchWeather", + arguments = mapOf("city" to FunctionArgValue.StringValue("London")), + id = "id-123" + ) + val mlKitCandidate = + mock(Candidate::class.java).apply { + `when`(text).thenReturn("") + `when`(finishReason).thenReturn(Candidate.FinishReason.STOP) + `when`(functionCalls).thenReturn(listOf(mlKitFunctionCall)) + } + val interopCandidate = mlKitCandidate.toInterop() + assertThat(interopCandidate.functionCalls).hasSize(1) + val call = interopCandidate.functionCalls[0] + assertThat(call.name).isEqualTo("fetchWeather") + assertThat(call.args).isEqualTo(mapOf("city" to "London")) + assertThat(call.id).isEqualTo("id-123") + } + + @Test + fun `GenerateContentRequest toMlKit should convert tools correctly`() { + val schema = + InteropFunctionSchema( + type = "OBJECT", + description = "weather params", + properties = + mapOf("city" to InteropFunctionSchema(type = "STRING", description = "city name")), + required = listOf("city") + ) + val declaration = + InteropFunctionDeclaration( + name = "fetchWeather", + description = "Fetches the weather for a city", + parameters = schema + ) + val interopRequest = + InteropGenerateContentRequest( + parts = listOf(InteropTextPart("What is the weather?")), + tools = listOf(declaration) + ) + val mlKitRequest = interopRequest.toMlKit() + + assertThat(mlKitRequest.dynamicTools).isNotNull() + assertThat(mlKitRequest.dynamicTools).hasSize(1) + val mlKitTool = mlKitRequest.dynamicTools!!.first() + assertThat(mlKitTool.name).isEqualTo("fetchWeather") + assertThat(mlKitTool.description).isEqualTo("Fetches the weather for a city") + assertThat(mlKitTool.parameters?.type).isEqualTo(FunctionType.OBJECT) + } + @Test fun `GenerateContentRequest toMlKit should convert correctly and cover all fields`() { val interopRequest = diff --git a/ai-logic/firebase-ai-ondevice/src/test/kotlin/com/google/firebase/ai/ondevice/GenerativeModelImplTest.kt b/ai-logic/firebase-ai-ondevice/src/test/kotlin/com/google/firebase/ai/ondevice/GenerativeModelImplTest.kt index df843b72b80..9f5df7344ff 100644 --- a/ai-logic/firebase-ai-ondevice/src/test/kotlin/com/google/firebase/ai/ondevice/GenerativeModelImplTest.kt +++ b/ai-logic/firebase-ai-ondevice/src/test/kotlin/com/google/firebase/ai/ondevice/GenerativeModelImplTest.kt @@ -24,10 +24,13 @@ import com.google.firebase.ai.ondevice.interop.FirebaseAIOnDeviceUnknownExceptio import com.google.firebase.ai.ondevice.interop.GenerateContentRequest as InteropGenerateContentRequest import com.google.firebase.ai.ondevice.interop.TextPart as InteropTextPart import com.google.mlkit.genai.common.FeatureStatus +import com.google.mlkit.genai.common.FunctionArgValue +import com.google.mlkit.genai.common.FunctionCall import com.google.mlkit.genai.common.GenAiException import com.google.mlkit.genai.common.GenAiException.ErrorCode import com.google.mlkit.genai.prompt.Candidate as MlKitCandidate import com.google.mlkit.genai.prompt.CountTokensResponse as MlKitCountTokensResponse +import com.google.mlkit.genai.prompt.Feature import com.google.mlkit.genai.prompt.GenerateContentRequest as MlKitGenerateContentRequest import com.google.mlkit.genai.prompt.GenerativeModel as MlKitGenerativeModel import io.kotest.assertions.throwables.shouldThrow @@ -67,7 +70,9 @@ class GenerativeModelImplTest { @Test fun `isThinkingModeAvailable returns true when MLKit supports thinking mode`() = runTest { val mlkitModel = - mockk { coEvery { isThinkingModeAvailable() } returns true } + mockk { + coEvery { getAvailableFeatures() } returns listOf(Feature.THINKING_MODE) + } val model = GenerativeModelImpl(mlkitModel) model.isThinkingModeAvailable().shouldBeTrue() @@ -77,12 +82,33 @@ class GenerativeModelImplTest { fun `isThinkingModeAvailable returns false when MLKit does not support thinking mode`() = runTest { val mlkitModel = - mockk { coEvery { isThinkingModeAvailable() } returns false } + mockk { coEvery { getAvailableFeatures() } returns emptyList() } val model = GenerativeModelImpl(mlkitModel) model.isThinkingModeAvailable().shouldBeFalse() } + @Test + fun `isFunctionCallingAvailable returns true when MLKit supports function calling`() = runTest { + val mlkitModel = + mockk { + coEvery { getAvailableFeatures() } returns listOf(Feature.FUNCTION_CALLING) + } + val model = GenerativeModelImpl(mlkitModel) + + model.isFunctionCallingAvailable().shouldBeTrue() + } + + @Test + fun `isFunctionCallingAvailable returns false when MLKit does not support function calling`() = + runTest { + val mlkitModel = + mockk { coEvery { getAvailableFeatures() } returns emptyList() } + val model = GenerativeModelImpl(mlkitModel) + + model.isFunctionCallingAvailable().shouldBeFalse() + } + @Test fun `generateContent should return converted response`() = runTest { val mlkitModel = @@ -95,6 +121,7 @@ class GenerativeModelImplTest { mockk { every { text } returns "response text" every { finishReason } returns MlKitCandidate.FinishReason.MAX_TOKENS + every { functionCalls } returns emptyList() } ) every { thoughtProcess } returns @@ -102,6 +129,7 @@ class GenerativeModelImplTest { mockk { every { text } returns "thought text" every { finishReason } returns MlKitCandidate.FinishReason.STOP + every { functionCalls } returns emptyList() } ) } @@ -125,6 +153,47 @@ class GenerativeModelImplTest { } } + @Test + fun `generateContent should return converted function calls`() = runTest { + val mlkitModel = + mockk { + coEvery { getBaseModelName() } returns "gemini-3.1-flash-lite" + coEvery { generateContent(any()) } returns + mockk { + every { candidates } returns + listOf( + mockk { + every { text } returns "" + every { finishReason } returns MlKitCandidate.FinishReason.STOP + every { functionCalls } returns + listOf( + FunctionCall( + name = "fetchWeather", + arguments = mapOf("city" to FunctionArgValue.StringValue("London")), + id = "call-1" + ) + ) + } + ) + every { thoughtProcess } returns emptyList() + } + } + val model = GenerativeModelImpl(mlkitModel) + + val generateContentRequest = InteropGenerateContentRequest(listOf(InteropTextPart("prompt"))) + val response = model.generateContent(generateContentRequest) + + response.candidates shouldHaveSize 1 + response.candidates.first().apply { + functionCalls shouldHaveSize 1 + functionCalls.first().apply { + name shouldBe "fetchWeather" + args shouldBe mapOf("city" to "London") + id shouldBe "call-1" + } + } + } + @Test fun `countTokens should return converted response`() = runTest { val mlkitModel = @@ -152,6 +221,7 @@ class GenerativeModelImplTest { mockk { every { text } returns "partial response" every { finishReason } returns MlKitCandidate.FinishReason.STOP + every { functionCalls } returns emptyList() } ) every { thoughtProcess } returns emptyList() diff --git a/ai-logic/firebase-ai/CHANGELOG.md b/ai-logic/firebase-ai/CHANGELOG.md index 84bf83211f6..3fc3b5b84ea 100644 --- a/ai-logic/firebase-ai/CHANGELOG.md +++ b/ai-logic/firebase-ai/CHANGELOG.md @@ -1,5 +1,8 @@ # Unreleased +- [feature] Added support for function calling in on-device and hybrid + inference modes, with automatic cloud fallback when the on-device model does + not support function calling. (#8758) - [feature] Added support for multiple `TextPart` and `ImagePart` instances in on-device and hybrid inference modes. (#8754) diff --git a/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/GenerativeModel.kt b/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/GenerativeModel.kt index f7fe62b6718..7132edf9945 100644 --- a/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/GenerativeModel.kt +++ b/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/GenerativeModel.kt @@ -372,7 +372,8 @@ internal constructor( OnDeviceGenerativeModelProvider( it.newGenerativeModel(modelOption?.toInterop()), onDeviceConfig, - systemInstruction + systemInstruction, + tools ) } ?: MissingOnDeviceGenerativeModelProvider() diff --git a/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/generativemodel/OnDeviceGenerativeModelProvider.kt b/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/generativemodel/OnDeviceGenerativeModelProvider.kt index 357aaf7f083..97894118e5b 100644 --- a/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/generativemodel/OnDeviceGenerativeModelProvider.kt +++ b/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/generativemodel/OnDeviceGenerativeModelProvider.kt @@ -21,27 +21,46 @@ import com.google.firebase.ai.InferenceSource import com.google.firebase.ai.OnDeviceConfig import com.google.firebase.ai.ondevice.interop.FirebaseAIOnDeviceInvalidRequestException import com.google.firebase.ai.ondevice.interop.FirebaseAIOnDeviceNotAvailableException +import com.google.firebase.ai.ondevice.interop.FunctionCallPart as OnDeviceFunctionCallPart +import com.google.firebase.ai.ondevice.interop.FunctionDeclaration as OnDeviceFunctionDeclaration +import com.google.firebase.ai.ondevice.interop.FunctionResponsePart as OnDeviceFunctionResponsePart +import com.google.firebase.ai.ondevice.interop.FunctionSchema as OnDeviceFunctionSchema import com.google.firebase.ai.ondevice.interop.GenerateContentRequest as OnDeviceGenerateContentRequest import com.google.firebase.ai.ondevice.interop.GenerativeModel as OnDeviceGenerativeModel import com.google.firebase.ai.ondevice.interop.ImagePart as OnDeviceImagePart import com.google.firebase.ai.ondevice.interop.TextPart as OnDeviceTextPart +import com.google.firebase.ai.type.AutoFunctionDeclaration import com.google.firebase.ai.type.Candidate import com.google.firebase.ai.type.Content import com.google.firebase.ai.type.CountTokensResponse import com.google.firebase.ai.type.FinishReason import com.google.firebase.ai.type.FirebaseAIException +import com.google.firebase.ai.type.FunctionCallPart +import com.google.firebase.ai.type.FunctionDeclaration +import com.google.firebase.ai.type.FunctionResponsePart import com.google.firebase.ai.type.GenerateContentResponse import com.google.firebase.ai.type.GenerateObjectResponse import com.google.firebase.ai.type.ImagePart import com.google.firebase.ai.type.JsonSchema import com.google.firebase.ai.type.PublicPreviewAPI +import com.google.firebase.ai.type.Schema import com.google.firebase.ai.type.TextPart +import com.google.firebase.ai.type.Tool import com.google.firebase.ai.type.content import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.catch import kotlinx.coroutines.flow.emitAll import kotlinx.coroutines.flow.flow import kotlinx.coroutines.flow.map +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonNull +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.booleanOrNull +import kotlinx.serialization.json.doubleOrNull +import kotlinx.serialization.json.intOrNull +import kotlinx.serialization.json.longOrNull /** * Implementation of [GenerativeModelProvider] that delegates to an on-device generative model. @@ -52,12 +71,14 @@ import kotlinx.coroutines.flow.map * @property onDeviceModel The underlying on-device model to use for generation. * @property onDeviceConfig Configuration options for the on-device model. * @property systemInstruction Optional instructions that direct the model to behave a certain way. + * @property tools Optional list of tools provided to the model. */ @PublicPreviewAPI internal class OnDeviceGenerativeModelProvider( private val onDeviceModel: OnDeviceGenerativeModel, private val onDeviceConfig: OnDeviceConfig, - internal val systemInstruction: Content? = null + internal val systemInstruction: Content? = null, + internal val tools: List = emptyList() ) : GenerativeModelProvider { /** @@ -86,7 +107,7 @@ internal class OnDeviceGenerativeModelProvider( */ override suspend fun countTokens(prompt: List): CountTokensResponse = withFirebaseAIExceptionHandling { - ensureOnDeviceModelAvailable(checkThinkingMode = false) + ensureOnDeviceModelAvailable(checkThinkingMode = false, checkFunctionCalling = false) val request = buildOnDeviceGenerateContentRequest(prompt) @@ -129,7 +150,7 @@ internal class OnDeviceGenerativeModelProvider( jsonSchema: JsonSchema, prompt: List ): GenerateObjectResponse = withFirebaseAIExceptionHandling { - ensureOnDeviceModelAvailable() + ensureOnDeviceModelAvailable(checkFunctionCalling = false) val request = buildOnDeviceGenerateContentRequest(prompt) val interopResponse = onDeviceModel.generateObject(request, jsonSchema.clazz) @@ -159,7 +180,7 @@ internal class OnDeviceGenerativeModelProvider( */ override suspend fun warmUp() { withFirebaseAIExceptionHandling { - ensureOnDeviceModelAvailable(checkThinkingMode = false) + ensureOnDeviceModelAvailable(checkThinkingMode = false, checkFunctionCalling = false) onDeviceModel.warmup() } } @@ -172,7 +193,10 @@ internal class OnDeviceGenerativeModelProvider( } } - private suspend fun ensureOnDeviceModelAvailable(checkThinkingMode: Boolean = true) { + private suspend fun ensureOnDeviceModelAvailable( + checkThinkingMode: Boolean = true, + checkFunctionCalling: Boolean = true + ) { if (!onDeviceModel.isAvailable()) { throw FirebaseAIException.from( FirebaseAIOnDeviceNotAvailableException("On-device model is not available") @@ -189,6 +213,33 @@ internal class OnDeviceGenerativeModelProvider( ) ) } + if (checkFunctionCalling && tools.isNotEmpty()) { + val hasNonFunctionTools = + tools.any { + it.googleSearch != null || + it.codeExecution != null || + it.urlContext != null || + it.googleMaps != null + } + if (hasNonFunctionTools) { + throw FirebaseAIException.from( + FirebaseAIOnDeviceNotAvailableException( + "Tools other than function calling are not supported on the on-device model" + ) + ) + } + val hasFunctionDeclarations = + tools.any { + !it.functionDeclarations.isNullOrEmpty() || !it.autoFunctionDeclarations.isNullOrEmpty() + } + if (hasFunctionDeclarations && !onDeviceModel.isFunctionCallingAvailable()) { + throw FirebaseAIException.from( + FirebaseAIOnDeviceNotAvailableException( + "Function calling is not available on the on-device model" + ) + ) + } + } } private fun toPublicResponse( @@ -238,6 +289,18 @@ internal class OnDeviceGenerativeModelProvider( when (part) { is TextPart -> OnDeviceTextPart(part.text) is ImagePart -> OnDeviceImagePart(part.image) + is FunctionCallPart -> + OnDeviceFunctionCallPart( + name = part.name, + args = part.args.mapValues { it.value.toAny() }, + id = part.id + ) + is FunctionResponsePart -> + OnDeviceFunctionResponsePart( + name = part.name, + response = part.response.entries.associate { it.key to it.value.toAny() }, + id = part.id + ) else -> throw FirebaseAIException.from( FirebaseAIOnDeviceInvalidRequestException( @@ -270,6 +333,12 @@ internal class OnDeviceGenerativeModelProvider( if (systemTextParts.isEmpty()) null else OnDeviceTextPart(systemTextParts.joinToString("\n") { it.text }) } + val onDeviceTools = + tools.flatMap { tool -> + val normal = tool.functionDeclarations?.map { it.toInterop() } ?: emptyList() + val auto = tool.autoFunctionDeclarations?.map { it.toInterop() } ?: emptyList() + normal + auto + } return OnDeviceGenerateContentRequest( parts = onDeviceParts, temperature = onDeviceConfig.temperature, @@ -278,7 +347,8 @@ internal class OnDeviceGenerativeModelProvider( candidateCount = onDeviceConfig.candidateCount, maxOutputTokens = onDeviceConfig.maxOutputTokens, systemInstruction = onDeviceSystemInstruction, - enableThinking = onDeviceConfig.enableThinking + enableThinking = onDeviceConfig.enableThinking, + tools = onDeviceTools ) } @@ -286,3 +356,49 @@ internal class OnDeviceGenerativeModelProvider( private val TAG = OnDeviceGenerativeModelProvider::class.java.simpleName } } + +private fun JsonElement.toAny(): Any? = + when (this) { + is JsonNull -> null + is JsonPrimitive -> { + if (isString) content else booleanOrNull ?: intOrNull ?: longOrNull ?: doubleOrNull ?: content + } + is JsonArray -> map { it.toAny() } + is JsonObject -> entries.associate { it.key to it.value.toAny() } + } + +private fun Schema.toInterop(): OnDeviceFunctionSchema = + OnDeviceFunctionSchema( + type = type, + description = description, + properties = properties?.mapValues { it.value.toInterop() } ?: emptyMap(), + required = required ?: emptyList(), + enum = enum, + nullable = nullable, + items = items?.toInterop() + ) + +private fun FunctionDeclaration.toInterop(): OnDeviceFunctionDeclaration = + OnDeviceFunctionDeclaration( + name = name, + description = description, + parameters = schema.toInterop() + ) + +private fun JsonSchema<*>.toInterop(): OnDeviceFunctionSchema = + OnDeviceFunctionSchema( + type = type, + description = description, + properties = properties?.mapValues { it.value.toInterop() } ?: emptyMap(), + required = required ?: emptyList(), + enum = enum, + nullable = nullable, + items = items?.toInterop() + ) + +private fun AutoFunctionDeclaration<*, *>.toInterop(): OnDeviceFunctionDeclaration = + OnDeviceFunctionDeclaration( + name = name, + description = description, + parameters = inputSchema.toInterop() + ) diff --git a/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Candidate.kt b/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Candidate.kt index 18c1fb4c3db..3d87226e96c 100644 --- a/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Candidate.kt +++ b/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Candidate.kt @@ -24,7 +24,12 @@ import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.KSerializer import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonElement import kotlinx.serialization.json.JsonNames +import kotlinx.serialization.json.JsonNull +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive /** * A `Candidate` represents a single response generated by the model for a given request. @@ -94,9 +99,21 @@ internal constructor( nonEmptyThoughts.forEach { thought -> part(TextPart(text = thought.text, isThought = true, thoughtSignature = null)) } - if (candidate.text.isNotEmpty() || nonEmptyThoughts.isEmpty()) { + if ( + candidate.text.isNotEmpty() || + (nonEmptyThoughts.isEmpty() && candidate.functionCalls.isEmpty()) + ) { text(candidate.text) } + candidate.functionCalls.forEach { call -> + part( + FunctionCallPart( + name = call.name, + args = call.args.mapValues { it.value.toJsonElement() }, + id = call.id + ) + ) + } }, safetyRatings = emptyList(), citationMetadata = null, @@ -125,6 +142,17 @@ internal constructor( } } +private fun Any?.toJsonElement(): JsonElement = + when (this) { + null -> JsonNull + is Boolean -> JsonPrimitive(this) + is Number -> JsonPrimitive(this) + is String -> JsonPrimitive(this) + is List<*> -> JsonArray(map { it.toJsonElement() }) + is Map<*, *> -> JsonObject(entries.associate { it.key.toString() to it.value.toJsonElement() }) + else -> JsonPrimitive(toString()) + } + /** * An assessment of the potential harm of some generated content. * diff --git a/ai-logic/firebase-ai/src/test/java/com/google/firebase/ai/generativeModel/OnDeviceGenerativeModelProviderTests.kt b/ai-logic/firebase-ai/src/test/java/com/google/firebase/ai/generativeModel/OnDeviceGenerativeModelProviderTests.kt index 424a1763f8d..48c5b264cbf 100644 --- a/ai-logic/firebase-ai/src/test/java/com/google/firebase/ai/generativeModel/OnDeviceGenerativeModelProviderTests.kt +++ b/ai-logic/firebase-ai/src/test/java/com/google/firebase/ai/generativeModel/OnDeviceGenerativeModelProviderTests.kt @@ -25,15 +25,23 @@ import com.google.firebase.ai.ondevice.interop.CountTokensResponse as OnDeviceCo import com.google.firebase.ai.ondevice.interop.FinishReason as OnDeviceFinishReason import com.google.firebase.ai.ondevice.interop.FirebaseAIOnDeviceInvalidRequestException import com.google.firebase.ai.ondevice.interop.FirebaseAIOnDeviceNotAvailableException +import com.google.firebase.ai.ondevice.interop.FunctionCallPart as OnDeviceFunctionCallPart +import com.google.firebase.ai.ondevice.interop.FunctionResponsePart as OnDeviceFunctionResponsePart import com.google.firebase.ai.ondevice.interop.GenerateContentResponse as OnDeviceGenerateContentResponse import com.google.firebase.ai.ondevice.interop.GenerativeModel as OnDeviceGenerativeModel import com.google.firebase.ai.toInterop import com.google.firebase.ai.type.Content import com.google.firebase.ai.type.FirebaseAIException +import com.google.firebase.ai.type.FunctionCallPart +import com.google.firebase.ai.type.FunctionDeclaration +import com.google.firebase.ai.type.FunctionResponsePart import com.google.firebase.ai.type.JsonSchema import com.google.firebase.ai.type.PublicPreviewAPI +import com.google.firebase.ai.type.Schema import com.google.firebase.ai.type.TextPart +import com.google.firebase.ai.type.Tool import io.kotest.assertions.throwables.shouldThrow +import io.kotest.matchers.collections.shouldHaveSize import io.kotest.matchers.shouldBe import io.mockk.coEvery import io.mockk.every @@ -42,6 +50,9 @@ import io.mockk.slot import kotlinx.coroutines.flow.first import kotlinx.coroutines.flow.flowOf import kotlinx.coroutines.runBlocking +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.put import org.junit.Before import org.junit.Test @@ -388,4 +399,235 @@ internal class OnDeviceGenerativeModelProviderTests { emissions[1].thoughtSummary shouldBe "done thinking" emissions[1].text shouldBe "streamed answer" } + + @Test + fun `generateContent includes tools and maps functionCalls to response`(): Unit = runBlocking { + coEvery { onDeviceModel.isAvailable() } returns true + coEvery { onDeviceModel.isFunctionCallingAvailable() } returns true + val capturedRequest = slot() + coEvery { onDeviceModel.generateContent(capture(capturedRequest)) } returns + OnDeviceGenerateContentResponse( + candidates = + listOf( + OnDeviceCandidate( + text = "", + finishReason = OnDeviceFinishReason.STOP, + functionCalls = + listOf( + OnDeviceFunctionCallPart( + name = "fetchWeather", + args = mapOf("city" to "London"), + id = "call-123" + ) + ) + ) + ), + modelVersion = "gemini-nano" + ) + + val weatherTool = + Tool.functionDeclarations( + listOf( + FunctionDeclaration( + name = "fetchWeather", + description = "Fetches weather for a city", + parameters = mapOf("city" to Schema.string("City name")) + ) + ) + ) + + val functionProvider = + OnDeviceGenerativeModelProvider( + onDeviceModel = onDeviceModel, + onDeviceConfig = OnDeviceConfig(mode = InferenceMode.ONLY_ON_DEVICE), + tools = listOf(weatherTool) + ) + + val response = functionProvider.generateContent(prompt) + + capturedRequest.captured.tools.shouldHaveSize(1) + val tool = capturedRequest.captured.tools.first() + tool.name shouldBe "fetchWeather" + tool.description shouldBe "Fetches weather for a city" + tool.parameters?.type shouldBe "OBJECT" + + response.functionCalls shouldHaveSize 1 + val functionCall = response.functionCalls.first() + functionCall.name shouldBe "fetchWeather" + functionCall.args["city"] shouldBe JsonPrimitive("London") + functionCall.id shouldBe "call-123" + } + + @Test + fun `generateContent throws when function calling is not available`(): Unit = runBlocking { + coEvery { onDeviceModel.isAvailable() } returns true + coEvery { onDeviceModel.isFunctionCallingAvailable() } returns false + + val weatherTool = + Tool.functionDeclarations( + listOf( + FunctionDeclaration( + name = "fetchWeather", + description = "Fetches weather", + parameters = mapOf("city" to Schema.string("City name")) + ) + ) + ) + + val functionProvider = + OnDeviceGenerativeModelProvider( + onDeviceModel = onDeviceModel, + onDeviceConfig = OnDeviceConfig(mode = InferenceMode.ONLY_ON_DEVICE), + tools = listOf(weatherTool) + ) + + val exception = shouldThrow { functionProvider.generateContent(prompt) } + exception.cause!!::class shouldBe FirebaseAIOnDeviceNotAvailableException::class + } + + @Test + fun `generateContent throws when tools contain unsupported non-function tools`(): Unit = + runBlocking { + coEvery { onDeviceModel.isAvailable() } returns true + + val searchTool = Tool.googleSearch() + + val functionProvider = + OnDeviceGenerativeModelProvider( + onDeviceModel = onDeviceModel, + onDeviceConfig = OnDeviceConfig(mode = InferenceMode.ONLY_ON_DEVICE), + tools = listOf(searchTool) + ) + + val exception = shouldThrow { functionProvider.generateContent(prompt) } + exception.cause!!::class shouldBe FirebaseAIOnDeviceNotAvailableException::class + } + + @Test + fun `generateContent with multi-turn prompt containing FunctionCallPart and FunctionResponsePart`(): + Unit = runBlocking { + coEvery { onDeviceModel.isAvailable() } returns true + coEvery { onDeviceModel.isFunctionCallingAvailable() } returns true + val capturedRequest = slot() + coEvery { onDeviceModel.generateContent(capture(capturedRequest)) } returns + OnDeviceGenerateContentResponse( + candidates = listOf(OnDeviceCandidate("The weather is sunny", OnDeviceFinishReason.STOP)), + modelVersion = "gemini-nano" + ) + + val multiTurnPrompt = + listOf( + Content( + role = "model", + parts = + listOf( + FunctionCallPart( + name = "fetchWeather", + args = mapOf("city" to JsonPrimitive("London")), + id = "call-123" + ) + ) + ), + Content( + role = "user", + parts = + listOf( + FunctionResponsePart( + name = "fetchWeather", + response = buildJsonObject { put("condition", "sunny") }, + id = "call-123" + ) + ) + ) + ) + + val weatherTool = + Tool.functionDeclarations( + listOf( + FunctionDeclaration( + name = "fetchWeather", + description = "Fetches weather", + parameters = mapOf("city" to Schema.string("City name")) + ) + ) + ) + + val functionProvider = + OnDeviceGenerativeModelProvider( + onDeviceModel = onDeviceModel, + onDeviceConfig = OnDeviceConfig(mode = InferenceMode.ONLY_ON_DEVICE), + tools = listOf(weatherTool) + ) + + val response = functionProvider.generateContent(multiTurnPrompt) + + capturedRequest.captured.parts shouldHaveSize 2 + val callPart = capturedRequest.captured.parts[0] as OnDeviceFunctionCallPart + callPart.name shouldBe "fetchWeather" + callPart.args shouldBe mapOf("city" to "London") + callPart.id shouldBe "call-123" + + val responsePart = capturedRequest.captured.parts[1] as OnDeviceFunctionResponsePart + responsePart.name shouldBe "fetchWeather" + responsePart.response shouldBe mapOf("condition" to "sunny") + responsePart.id shouldBe "call-123" + + response.text shouldBe "The weather is sunny" + } + + @Test + fun `generateContentStream maps functionCalls in streamed response`(): Unit = runBlocking { + coEvery { onDeviceModel.isAvailable() } returns true + coEvery { onDeviceModel.isFunctionCallingAvailable() } returns true + + val functionCallChunk = + OnDeviceGenerateContentResponse( + candidates = + listOf( + OnDeviceCandidate( + text = "", + finishReason = OnDeviceFinishReason.STOP, + functionCalls = + listOf( + OnDeviceFunctionCallPart( + name = "fetchWeather", + args = mapOf("city" to "London"), + id = "call-123" + ) + ) + ) + ), + modelVersion = "gemini-nano" + ) + + every { onDeviceModel.generateContentStream(any()) } returns flowOf(functionCallChunk) + + val weatherTool = + Tool.functionDeclarations( + listOf( + FunctionDeclaration( + name = "fetchWeather", + description = "Fetches weather", + parameters = mapOf("city" to Schema.string("City name")) + ) + ) + ) + + val functionProvider = + OnDeviceGenerativeModelProvider( + onDeviceModel = onDeviceModel, + onDeviceConfig = OnDeviceConfig(mode = InferenceMode.ONLY_ON_DEVICE), + tools = listOf(weatherTool) + ) + + val emissions = mutableListOf() + functionProvider.generateContentStream(prompt).collect { emissions.add(it) } + + emissions shouldHaveSize 1 + emissions[0].functionCalls shouldHaveSize 1 + val call = emissions[0].functionCalls.first() + call.name shouldBe "fetchWeather" + call.args["city"] shouldBe JsonPrimitive("London") + call.id shouldBe "call-123" + } } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 6d27fdffb08..29f7bfc275e 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -29,8 +29,8 @@ firebaseAnnotations = "18.0.0" firebaseCommon = "23.0.0" firebaseComponents = "20.0.0" firebaseCrashlyticsGradle = "3.0.4" -genaiPrompt = "1.0.0-beta4" -genaiSchema = "1.0.0-alpha1" +genaiPrompt = "1.0.0-beta5" +genaiSchema = "1.0.0-alpha2" glide = "5.0.5" googleApiClient = "2.8.1" googleServices = "4.3.15"