feat(ai-components): native AIAgent module for install id and tool-call sign-in

examples/ExpoChatGPTSample/app.json

@@ -2,7 +2,7 @@
   "expo": {
     "name": "ExpoChatGPTSample",
     "slug": "ExpoChatGPTSample",
-    "scheme": "expochatgptsample",
+    "scheme": ["expochatgptsample", "io.getstream.expochatgptsample"],
     "ios": {
       "bundleIdentifier": "io.getstream.expochatgptsample",
       "appleTeamId": "EHV7XZLAHA"

examples/ExpoChatGPTSample/app/+native-intent.tsx

@@ -0,0 +1,5 @@
+// A sign-in opened in a Custom Tab comes back to the app on its scheme (`io.getstream.expochatgptsample://
+// sign-in?…`), which the SDK's sign-in reads itself. Stay on the screen it was opened from rather
+// than routing to a page that doesn't exist.
+export const redirectSystemPath = ({ path }: { initial: boolean; path: string | null }) =>
+  path?.includes('sign-in') ? null : path;

examples/ExpoChatGPTSample/contexts/AppContext.tsx

@@ -9,7 +9,11 @@ import {
 } from '@stream-io/calypso';
 import type { Channel, Event, StreamChat } from 'stream-chat';
 import { useStableCallback } from 'stream-chat-expo';
-import type { AIToolApprover } from 'stream-chat-expo/ai-components';
+import {
+  AIClientIdentity,
+  type AIToolApprover,
+  createAuthSessionAuthorizer,
+} from 'stream-chat-expo/ai-components';
 
 import { agents, agentSessions, streamToken } from '../agent/agents';
 import { AgentConfig } from '../agent/config';
@@ -18,8 +22,20 @@ type Conversation = { channel: Channel; reasoning: LiveReasoning; session: Sessi
 
 const IDLE: ConversationSnapshot = { kind: 'idle', skills: [] };
 
-// This install, for the tool calls addressed to it. Kept for the app's lifetime.
-const clientId = `rn-sample-${(globalThis as { crypto: { randomUUID: () => string } }).crypto.randomUUID()}`;
+// This install, for the tool calls addressed to it.
+const clientId = AIClientIdentity.installId();
+
+// Where a sign-in comes back to: the app's own scheme, which the web auth session hands back.
+// The router takes only https or a native app's own scheme in reverse domain form (RFC 8252).
+const SIGN_IN_SCHEME = 'io.getstream.expochatgptsample';
+const SIGN_IN_RETURN_URL = `${SIGN_IN_SCHEME}://sign-in`;
+
+// The person's "not now" on a sign-in. calypso 0.2.3 has no `declineLogin` yet, so this sends
+// the frame Vision-Agents' JS SDK sends for it over the session's socket.
+const declineLogin = (session: Session, toolCallId: string) =>
+  (session as unknown as { held(): { send(frame: Record<string, unknown>): void } })
+    .held()
+    .send({ allowed: false, tool_call_id: toolCallId, type: 'tool_approval' });
 
 export type AppContextValue = {
   /** What the agent is doing in the open conversation, from the session's socket. */
@@ -95,7 +111,12 @@ export const AppProvider = ({ client, children }: PropsWithChildren<{ client: St
   });
 
   const startConversation = useStableCallback(() => {
-    void open(() => agentSessions().create({ modelOverwrites: { llm: AgentConfig.chatModel } }));
+    void open(() =>
+      agentSessions().create({
+        modelOverwrites: { llm: AgentConfig.chatModel },
+        return_url: SIGN_IN_RETURN_URL,
+      }),
+    );
   });
 
   const openConversation = useStableCallback((sessionId: string) => {
@@ -160,6 +181,14 @@ export const AppProvider = ({ client, children }: PropsWithChildren<{ client: St
     () =>
       conversation
         ? {
+            authorizer: createAuthSessionAuthorizer({
+              callbackScheme: SIGN_IN_SCHEME,
+              decline: (call) =>
+                new Promise<void>((resolve) => {
+                  declineLogin(conversation.session, call.id);
+                  resolve();
+                }),
+            }),
             clientId,
             // `decide` sends over the session's socket and throws when it can't.
             decide: (call, allowed) =>

examples/ExpoChatGPTSample/screens/ChatContent.tsx

@@ -94,19 +94,27 @@ const CustomMessage = (props: MessageProps) => {
 type UnsupportedAttachmentComponent = ResolvedComponents['UnsupportedAttachment'];
 
 // A reply's steps (`ai_reasoning`, `ai_tool_call`) are attachments the chat SDK doesn't know. They
-// show in the reply's own view, so they're left out here; any other unknown attachment keeps the
-// SDK's own view.
+// show in the reply's own view, so they're left out here, as is the router's `connector_authorization`,
+// a second copy of a sign-in its tool call's step already asks for. Any other unknown attachment
+// keeps the SDK's own view.
 const hidingAISteps = (Fallback: UnsupportedAttachmentComponent) => {
   const HideAISteps: UnsupportedAttachmentComponent = (props) =>
-    props.attachment.type?.startsWith(AI_MESSAGE_PART_TYPE_PREFIX) ? null : <Fallback {...props} />;
+    props.attachment.type?.startsWith(AI_MESSAGE_PART_TYPE_PREFIX) ||
+    props.attachment.type === 'connector_authorization' ? null : (
+      <Fallback {...props} />
+    );
   return HideAISteps;
 };
 
 // A reply: its steps (reasoning, tool calls and their questions), then its answer.
 const CustomStreamingMessageView = () => {
-  const { message } = useMessageContext();
-  // Read so the steps render again as more of their reasoning arrives.
-  const { reasoningText, reasoningVersion } = useAppContext();
+  const { message: listed } = useMessageContext();
+  // Read so the steps render again as each live update arrives.
+  const { channel, reasoningText, reasoningVersion } = useAppContext();
+  // The list passes a row its message again only when it changes in ways it compares, which a
+  // step changing on its own, such as a tool call starting to wait for a sign-in, isn't. The
+  // channel's state has every update.
+  const message = channel?.state.findMessage(listed.id) ?? listed;
   const parts = useMemo(() => parseMessageParts(message.attachments), [message.attachments]);
   const stepText = useMemo(
     () => (stepId: string) => reasoningText(message.id, stepId),

package/bin/validate-entry-isolation.js

@@ -208,6 +208,10 @@ const copies = [
     path.join(AI_SRC, 'native-specs/NativeAIDictation.ts'),
     ...WRAPPERS.map((w) => path.join(w, 'src/native/NativeAIDictation.ts')),
   ],
+  [
+    path.join(AI_SRC, 'native-specs/NativeAIAgent.ts'),
+    ...WRAPPERS.map((w) => path.join(w, 'src/native/NativeAIAgent.ts')),
+  ],
   WRAPPERS.map((w) => path.join(w, 'src/native/PerfTextNativeComponent.ts')),
 ];
 for (const [source, ...targets] of copies) {

package/expo-package/.gitignore

@@ -1,6 +1,7 @@
 # android
 android/build
 android/src/main/java/com/streamchatreactnative/shared
+android/src/agent-sign-in
 
 # ios
 ios/build

package/expo-package/android/build.gradle

@@ -42,6 +42,26 @@ def hasNativeSources = { File dir ->
   dir.exists() && !fileTree(dir).matching { include "**/*.kt"; include "**/*.java" }.files.isEmpty()
 }
 
+// The agent components' sign-ins open in the browser's Auth Tab, which needs androidx.browser
+// 1.9.0, and that needs the app to build with compileSdk 36 and AGP 8.9.1. An app below either
+// gets 1.8.0 and opens them in a Custom Tab instead, so the SDK asks neither of anyone.
+def versionAtLeast = { String version, List<Integer> wanted ->
+  def parts = version.tokenize('.-').collect { it.replaceAll(/\D.*$/, '') }.findAll { it }.collect { it.toInteger() }
+  for (int i = 0; i < wanted.size(); i++) {
+    def part = i < parts.size() ? parts[i] : 0
+    if (part != wanted[i]) return part > wanted[i]
+  }
+  return true
+}
+def agentSignInWithAuthTab =
+  getExtOrIntegerDefault("compileSdkVersion").toString().toInteger() >= 36 &&
+    versionAtLeast(com.android.Version.ANDROID_GRADLE_PLUGIN_VERSION, [8, 9, 1])
+def agentSignInVariant = agentSignInWithAuthTab ? "authtab" : "customtabs"
+def sharedAgentSignInDir = new File(canonicalProjectDir, "../../shared-native/android-${agentSignInVariant}")
+def agentSignInDir = hasNativeSources(sharedAgentSignInDir)
+  ? sharedAgentSignInDir
+  : new File(projectDir, "src/agent-sign-in/${agentSignInVariant}")
+
 android {
   compileSdkVersion getExtOrIntegerDefault("compileSdkVersion")
   def agpVersion = com.android.Version.ANDROID_GRADLE_PLUGIN_VERSION
@@ -82,6 +102,7 @@ android {
         java.srcDirs += [
           "src/newarch",
           "src/main/java/com/streamchatreactnative/shared",
+          agentSignInDir,
           "${project.buildDir}/generated/source/codegen/java"
         ]
       }
@@ -140,6 +161,9 @@ repositories {
 
 dependencies {
   implementation "com.facebook.react:react-native:+"
+  // The agent components' tool-call sign-ins: an Auth Tab where the app's build allows 1.9.0,
+  // else a Custom Tab (see `agentSignInWithAuthTab`).
+  implementation "androidx.browser:browser:${agentSignInWithAuthTab ? '1.9.0' : '1.8.0'}"
 }
 
 if (isNewArchitectureEnabled()) {

package/expo-package/android/src/main/AndroidManifest.xml

@@ -1,4 +1,12 @@
 <manifest xmlns:android="http://schemas.android.com/apk/res/android"
   package="com.streamchatexpo">
 
+  <!-- Lets the agent components' sign-ins see which browser has Custom Tabs and Auth Tabs on
+       Android 11 and later. No permission. -->
+  <queries>
+    <intent>
+      <action android:name="android.support.customtabs.action.CustomTabsService" />
+    </intent>
+  </queries>
+
 </manifest>

package/expo-package/android/src/main/java/com/streamchatexpo/StreamChatExpoPackage.java

@@ -18,6 +18,7 @@ public class StreamChatExpoPackage extends TurboReactPackage {
   private static final String STREAM_MULTIPART_UPLOADER_MODULE = "StreamMultipartUploader";
   private static final String STREAM_VIDEO_THUMBNAIL_MODULE = "StreamVideoThumbnail";
   private static final String AI_DICTATION_MODULE = "AIDictation";
+  private static final String AI_AGENT_MODULE = "AIAgent";
 
   @Nullable
   @Override
@@ -34,6 +35,10 @@ public NativeModule getModule(String name, ReactApplicationContext reactContext)
       return createNewArchModule("com.streamchatexpo.AIDictationModule", reactContext);
     }
 
+    if (name.equals(AI_AGENT_MODULE)) {
+      return createNewArchModule("com.streamchatexpo.AIAgentModule", reactContext);
+    }
+
     return null;
   }
 
@@ -75,6 +80,17 @@ public ReactModuleInfoProvider getReactModuleInfoProvider() {
                       false, // isCxxModule
                       true // isTurboModule
               ));
+      moduleInfos.put(
+              AI_AGENT_MODULE,
+              new ReactModuleInfo(
+                      AI_AGENT_MODULE,
+                      AI_AGENT_MODULE,
+                      false, // canOverrideExistingModule
+                      false, // needsEagerInit
+                      false, // hasConstants
+                      false, // isCxxModule
+                      true // isTurboModule
+              ));
       return moduleInfos;
     };
   }

package/expo-package/android/src/newarch/com/streamchatexpo/AIAgentModule.kt

@@ -0,0 +1,37 @@
+package com.streamchatexpo
+
+import com.facebook.react.bridge.Promise
+import com.facebook.react.bridge.ReactApplicationContext
+import com.streamchatreactnative.shared.agent.AIAgentController
+
+/**
+ * The `AIAgent` TurboModule used by the `ai-components` agent views: this install's identifier
+ * and the sign-ins of tool calls. The implementation is shared with the other SDK package; see
+ * [AIAgentController].
+ */
+class AIAgentModule(
+  reactContext: ReactApplicationContext,
+) : NativeAIAgentSpec(reactContext) {
+  private val controller = AIAgentController(reactContext)
+
+  override fun getName(): String = NAME
+
+  override fun initialize() {
+    super.initialize()
+    controller.initialize()
+  }
+
+  override fun invalidate() {
+    super.invalidate()
+    controller.invalidate()
+  }
+
+  override fun installId(): String = controller.installId()
+
+  override fun openAuthSession(url: String, callbackScheme: String, promise: Promise) =
+    controller.openAuthSession(url, callbackScheme, promise)
+
+  companion object {
+    const val NAME = AIAgentController.NAME
+  }
+}

package/expo-package/package.json

@@ -111,6 +111,7 @@
     },
     "ios": {
       "modulesProvider": {
+        "AIAgent": "AIAgent",
         "StreamMultipartUploader": "StreamMultipartUploader",
         "StreamVideoThumbnail": "StreamVideoThumbnail"
       },

package/expo-package/src/native/NativeAIAgent.ts

@@ -0,0 +1,25 @@
+import type { TurboModule } from 'react-native';
+import { TurboModuleRegistry } from 'react-native';
+
+/**
+ * Codegen spec of the native module behind the agent components: this install's identifier and
+ * the sign-ins of tool calls.
+ *
+ * This file is duplicated (byte for byte) in `package/native-package/src/native/NativeAIAgent.ts`
+ * and `package/expo-package/src/native/NativeAIAgent.ts`, which are the copies codegen reads.
+ * `yarn validate-entry-isolation` fails if they drift apart.
+ */
+export interface Spec extends TurboModule {
+  /** This install's identifier, created on first use and kept in storage that isn't backed up. */
+  installId(): string;
+  /**
+   * Opens `url` in the platform's web authentication session and resolves with the URL the page
+   * handed back to `callbackScheme`, or an empty string when the person closed it. Rejects when
+   * the session couldn't be opened.
+   */
+  openAuthSession(url: string, callbackScheme: string): Promise<string>;
+}
+
+// `get` rather than `getEnforcing`, so importing the AI entry point never throws in an app built
+// without the module, such as one on an older version of the SDK's native code.
+export default TurboModuleRegistry.get<Spec>('AIAgent');

package/expo-package/stream-chat-expo.podspec

@@ -42,9 +42,12 @@ Pod::Spec.new do |s|
       false
     end
 
+  # `AIAgent` (the agent components' install id and tool-call sign-ins) opens sign-ins in
+  # `ASWebAuthenticationSession`, which needs no usage description, so it is always compiled.
   if dictation_enabled
-    s.frameworks = "Speech"
+    s.frameworks = ["AuthenticationServices", "Speech"]
   else
+    s.frameworks = "AuthenticationServices"
     s.exclude_files = ["ios/shared/AIDictation.h", "ios/shared/AIDictation.mm", "ios/shared/AIDictationModule.swift"]
   end
 

package/native-package/.gitignore

@@ -1,6 +1,7 @@
 # android
 android/build
 android/src/main/java/com/streamchatreactnative/shared
+android/src/agent-sign-in
 
 # ios
 ios/build

package/native-package/android/build.gradle

@@ -49,6 +49,26 @@ def hasNativeSources = { File dir ->
   dir.exists() && !fileTree(dir).matching { include "**/*.kt"; include "**/*.java" }.files.isEmpty()
 }
 
+// The agent components' sign-ins open in the browser's Auth Tab, which needs androidx.browser
+// 1.9.0, and that needs the app to build with compileSdk 36 and AGP 8.9.1. An app below either
+// gets 1.8.0 and opens them in a Custom Tab instead, so the SDK asks neither of anyone.
+def versionAtLeast = { String version, List<Integer> wanted ->
+  def parts = version.tokenize('.-').collect { it.replaceAll(/\D.*$/, '') }.findAll { it }.collect { it.toInteger() }
+  for (int i = 0; i < wanted.size(); i++) {
+    def part = i < parts.size() ? parts[i] : 0
+    if (part != wanted[i]) return part > wanted[i]
+  }
+  return true
+}
+def agentSignInWithAuthTab =
+  getExtOrIntegerDefault("compileSdkVersion").toString().toInteger() >= 36 &&
+    versionAtLeast(com.android.Version.ANDROID_GRADLE_PLUGIN_VERSION, [8, 9, 1])
+def agentSignInVariant = agentSignInWithAuthTab ? "authtab" : "customtabs"
+def sharedAgentSignInDir = new File(canonicalProjectDir, "../../shared-native/android-${agentSignInVariant}")
+def agentSignInDir = hasNativeSources(sharedAgentSignInDir)
+  ? sharedAgentSignInDir
+  : new File(projectDir, "src/agent-sign-in/${agentSignInVariant}")
+
 android {
   compileSdkVersion getExtOrIntegerDefault("compileSdkVersion")
   def agpVersion = com.android.Version.ANDROID_GRADLE_PLUGIN_VERSION
@@ -92,6 +112,7 @@ android {
         java.srcDirs += [
           "src/newarch",
           "src/main/java/com/streamchatreactnative/shared",
+          agentSignInDir,
           // This is needed to build Kotlin project with NewArch enabled
           "${project.buildDir}/generated/source/codegen/java"
         ]
@@ -158,6 +179,9 @@ dependencies {
   //noinspection GradleDynamicVersion
   implementation "com.facebook.react:react-native:+"
   implementation "androidx.exifinterface:exifinterface:1.3.2"
+  // The agent components' tool-call sign-ins: an Auth Tab where the app's build allows 1.9.0,
+  // else a Custom Tab (see `agentSignInWithAuthTab`).
+  implementation "androidx.browser:browser:${agentSignInWithAuthTab ? '1.9.0' : '1.8.0'}"
 }
 
 if (isNewArchitectureEnabled()) {

package/native-package/android/src/main/AndroidManifest.xml

@@ -1,4 +1,12 @@
 <manifest xmlns:android="http://schemas.android.com/apk/res/android"
   package="com.streamchatreactnative">
 
-</manifest>
+  <!-- Lets the agent components' sign-ins see which browser has Custom Tabs and Auth Tabs on
+       Android 11 and later. No permission. -->
+  <queries>
+    <intent>
+      <action android:name="android.support.customtabs.action.CustomTabsService" />
+    </intent>
+  </queries>
+
+</manifest>