diff --git a/BotSharp.sln b/BotSharp.sln index 20bfeb54e..ac88b815a 100644 --- a/BotSharp.sln +++ b/BotSharp.sln @@ -125,6 +125,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Core.Rules", "src\ EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.DeepSeekAI", "src\Plugins\BotSharp.Plugin.DeepSeekAI\BotSharp.Plugin.DeepSeekAI.csproj", "{AF329442-B48E-4B48-A18A-1C869D1BA6F5}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.LiteLLM", "src\Plugins\BotSharp.Plugin.LiteLLM\BotSharp.Plugin.LiteLLM.csproj", "{0016D0D4-BE65-4626-8C38-3464A7906F10}" +EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.MiniMaxAI", "src\Plugins\BotSharp.Plugin.MiniMaxAI\BotSharp.Plugin.MiniMaxAI.csproj", "{D92C1A49-3F64-4D0C-91C0-B14EA47D2503}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Core.Realtime", "src\Infrastructure\BotSharp.Core.Realtime\BotSharp.Core.Realtime.csproj", "{781F1465-365C-0F22-1775-25025DAFA4C7}" @@ -739,6 +741,18 @@ Global {AF329442-B48E-4B48-A18A-1C869D1BA6F5}.Release|x64.Build.0 = Release|Any CPU {AF329442-B48E-4B48-A18A-1C869D1BA6F5}.Release|x86.ActiveCfg = Release|Any CPU {AF329442-B48E-4B48-A18A-1C869D1BA6F5}.Release|x86.Build.0 = Release|Any CPU + {0016D0D4-BE65-4626-8C38-3464A7906F10}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0016D0D4-BE65-4626-8C38-3464A7906F10}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0016D0D4-BE65-4626-8C38-3464A7906F10}.Debug|x64.ActiveCfg = Debug|Any CPU + {0016D0D4-BE65-4626-8C38-3464A7906F10}.Debug|x64.Build.0 = Debug|Any CPU + {0016D0D4-BE65-4626-8C38-3464A7906F10}.Debug|x86.ActiveCfg = Debug|Any CPU + {0016D0D4-BE65-4626-8C38-3464A7906F10}.Debug|x86.Build.0 = Debug|Any CPU + {0016D0D4-BE65-4626-8C38-3464A7906F10}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0016D0D4-BE65-4626-8C38-3464A7906F10}.Release|Any CPU.Build.0 = Release|Any CPU + {0016D0D4-BE65-4626-8C38-3464A7906F10}.Release|x64.ActiveCfg = Release|Any CPU + {0016D0D4-BE65-4626-8C38-3464A7906F10}.Release|x64.Build.0 = Release|Any CPU + {0016D0D4-BE65-4626-8C38-3464A7906F10}.Release|x86.ActiveCfg = Release|Any CPU + {0016D0D4-BE65-4626-8C38-3464A7906F10}.Release|x86.Build.0 = Release|Any CPU {D92C1A49-3F64-4D0C-91C0-B14EA47D2503}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {D92C1A49-3F64-4D0C-91C0-B14EA47D2503}.Debug|Any CPU.Build.0 = Debug|Any CPU {D92C1A49-3F64-4D0C-91C0-B14EA47D2503}.Debug|x64.ActiveCfg = Debug|Any CPU @@ -1015,6 +1029,7 @@ Global {F812BAAE-5A7D-4DF7-8E71-70696B51C61F} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749} {AFD64412-4D6A-452E-82A2-79E5D8842E29} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749} {AF329442-B48E-4B48-A18A-1C869D1BA6F5} = {D5293208-2BEF-42FC-A64C-5954F61720BA} + {0016D0D4-BE65-4626-8C38-3464A7906F10} = {D5293208-2BEF-42FC-A64C-5954F61720BA} {D92C1A49-3F64-4D0C-91C0-B14EA47D2503} = {D5293208-2BEF-42FC-A64C-5954F61720BA} {781F1465-365C-0F22-1775-25025DAFA4C7} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749} {C19D9AC1-97DD-8E65-E8DB-D295A095AA2D} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC} diff --git a/src/Plugins/BotSharp.Plugin.LiteLLM/BotSharp.Plugin.LiteLLM.csproj b/src/Plugins/BotSharp.Plugin.LiteLLM/BotSharp.Plugin.LiteLLM.csproj new file mode 100644 index 000000000..3f9a26ce0 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.LiteLLM/BotSharp.Plugin.LiteLLM.csproj @@ -0,0 +1,21 @@ + + + + $(TargetFramework) + enable + $(LangVersion) + $(BotSharpVersion) + $(GeneratePackageOnBuild) + $(GenerateDocumentationFile) + $(SolutionDir)packages + + + + + + + + + + + diff --git a/src/Plugins/BotSharp.Plugin.LiteLLM/LiteLLMPlugin.cs b/src/Plugins/BotSharp.Plugin.LiteLLM/LiteLLMPlugin.cs new file mode 100644 index 000000000..cf3d37c8d --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.LiteLLM/LiteLLMPlugin.cs @@ -0,0 +1,18 @@ +using BotSharp.Abstraction.Plugins; +using BotSharp.Plugin.LiteLLM.Providers.Text; +using BotSharp.Plugin.LiteLLM.Providers.Chat; + +namespace BotSharp.Plugin.LiteLLM; + +public class LiteLLMPlugin : IBotSharpPlugin +{ + public string Id => "b3c1f0d2-6a4e-4d1b-9c8a-2f7e5a9d4b60"; + public string Name => "LiteLLM"; + public string Description => "LiteLLM AI gateway. Call 100+ LLM providers through a single OpenAI-compatible endpoint."; + public string IconUrl => "https://litellm.ai/favicon.ico"; + public void RegisterDI(IServiceCollection services, IConfiguration config) + { + services.AddScoped(); + services.AddScoped(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.LiteLLM/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.LiteLLM/Providers/Chat/ChatCompletionProvider.cs new file mode 100644 index 000000000..0592db8db --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.LiteLLM/Providers/Chat/ChatCompletionProvider.cs @@ -0,0 +1,657 @@ +#pragma warning disable OPENAI001 +using BotSharp.Abstraction.Conversations.Enums; +using BotSharp.Abstraction.Utilities; +using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.Files.Models; +using BotSharp.Abstraction.Files.Utilities; +using BotSharp.Abstraction.Hooks; +using BotSharp.Abstraction.MessageHub.Models; +using BotSharp.Core.Infrastructures.Streams; +using BotSharp.Core.MessageHub; +using BotSharp.Plugin.LiteLLM.Providers; +using Microsoft.Extensions.Logging; +using OpenAI.Chat; + +namespace BotSharp.Plugin.LiteLLM.Providers.Chat; + +public class ChatCompletionProvider : IChatCompletion +{ + protected readonly IServiceProvider _services; + protected readonly ILogger _logger; + private List renderedInstructions = []; + + protected string _model; + public virtual string Provider => "litellm"; + public string Model => _model; + + public ChatCompletionProvider( + IServiceProvider services, + ILogger logger) + { + _services = services; + _logger = logger; + } + + public async Task GetChatCompletions(Agent agent, List conversations) + { + var contentHooks = _services.GetHooks(agent.Id); + + // Before chat completion hook + foreach (var hook in contentHooks) + { + await hook.BeforeGenerating(agent, conversations); + } + + var client = ProviderHelper.GetClient(Provider, _model, _services); + var chatClient = client.GetChatClient(_model); + var (prompt, messages, options) = PrepareOptions(agent, conversations); + + var response = chatClient.CompleteChat(messages, options); + var value = response.Value; + var reason = value.FinishReason; + var content = value.Content; + var text = content.FirstOrDefault()?.Text ?? string.Empty; + + RoleDialogModel responseMessage; + if (reason == ChatFinishReason.FunctionCall || reason == ChatFinishReason.ToolCalls) + { + var toolCall = value.ToolCalls.FirstOrDefault(); + responseMessage = new RoleDialogModel(AgentRole.Function, text) + { + CurrentAgentId = agent.Id, + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + ToolCallId = toolCall?.Id, + FunctionName = toolCall?.FunctionName, + FunctionArgs = toolCall?.FunctionArguments?.ToString(), + RenderedInstruction = string.Join("\r\n", renderedInstructions) + }; + + // Somethings LLM will generate a function name with agent name. + responseMessage.FunctionName = responseMessage.FunctionName.NormalizeFunctionName(); + } + else if (reason == ChatFinishReason.Length) + { + _logger.LogWarning($"Action: {nameof(GetChatCompletions)}, Reason: {reason}, Agent: {agent.Name}, MaxOutputTokens: {options.MaxOutputTokenCount}, Content:{text}"); + + responseMessage = new RoleDialogModel(AgentRole.Assistant, $"AI response exceeded max output length") + { + CurrentAgentId = agent.Id, + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + StopCompletion = true + }; + } + else + { + responseMessage = new RoleDialogModel(AgentRole.Assistant, text) + { + CurrentAgentId = agent.Id, + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + RenderedInstruction = string.Join("\r\n", renderedInstructions), + Annotations = value.Annotations?.Select(x => new ChatAnnotation + { + Title = x.WebResourceTitle, + Url = x.WebResourceUri.AbsoluteUri, + StartIndex = x.StartIndex, + EndIndex = x.EndIndex + })?.ToList() + }; + } + + var tokenUsage = response?.Value?.Usage; + var inputTokenDetails = response?.Value?.Usage?.InputTokenDetails; + + // After chat completion hook + foreach (var hook in contentHooks) + { + await hook.AfterGenerated(responseMessage, new TokenStatsModel + { + Prompt = prompt, + Provider = Provider, + Model = _model, + TextInputTokens = (tokenUsage?.InputTokenCount ?? 0) - (inputTokenDetails?.CachedTokenCount ?? 0), + CachedTextInputTokens = inputTokenDetails?.CachedTokenCount ?? 0, + TextOutputTokens = tokenUsage?.OutputTokenCount ?? 0 + }); + } + + return responseMessage; + } + + public async Task GetChatCompletionsAsync(Agent agent, List conversations, Func onMessageReceived, Func onFunctionExecuting) + { + var hooks = _services.GetHooks(agent.Id); + + // Before chat completion hook + foreach (var hook in hooks) + { + await hook.BeforeGenerating(agent, conversations); + } + + var client = ProviderHelper.GetClient(Provider, _model, _services); + var chatClient = client.GetChatClient(_model); + var (prompt, messages, options) = PrepareOptions(agent, conversations); + + var response = await chatClient.CompleteChatAsync(messages, options); + var value = response.Value; + var reason = value.FinishReason; + var content = value.Content; + var text = content.FirstOrDefault()?.Text ?? string.Empty; + + var msg = new RoleDialogModel(AgentRole.Assistant, text) + { + CurrentAgentId = agent.Id, + RenderedInstruction = string.Join("\r\n", renderedInstructions) + }; + + var tokenUsage = response?.Value?.Usage; + var inputTokenDetails = response?.Value?.Usage?.InputTokenDetails; + + // After chat completion hook + foreach (var hook in hooks) + { + await hook.AfterGenerated(msg, new TokenStatsModel + { + Prompt = prompt, + Provider = Provider, + Model = _model, + TextInputTokens = (tokenUsage?.InputTokenCount ?? 0) - (inputTokenDetails?.CachedTokenCount ?? 0), + CachedTextInputTokens = inputTokenDetails?.CachedTokenCount ?? 0, + TextOutputTokens = tokenUsage?.OutputTokenCount ?? 0 + }); + } + + if (reason == ChatFinishReason.FunctionCall || reason == ChatFinishReason.ToolCalls) + { + var toolCall = value.ToolCalls?.FirstOrDefault(); + _logger.LogInformation($"[{agent.Name}]: {toolCall?.FunctionName}({toolCall?.FunctionArguments})"); + + var funcContextIn = new RoleDialogModel(AgentRole.Function, text) + { + CurrentAgentId = agent.Id, + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + ToolCallId = toolCall?.Id, + FunctionName = toolCall?.FunctionName, + FunctionArgs = toolCall?.FunctionArguments?.ToString(), + RenderedInstruction = string.Join("\r\n", renderedInstructions) + }; + + // Somethings LLM will generate a function name with agent name. + funcContextIn.FunctionName = funcContextIn.FunctionName.NormalizeFunctionName(); + + // Execute functions + await onFunctionExecuting(funcContextIn); + } + else if (reason == ChatFinishReason.Length) + { + _logger.LogWarning($"Action: {nameof(GetChatCompletionsAsync)}, Reason: {reason}, Agent: {agent.Name}, MaxOutputTokens: {options.MaxOutputTokenCount}, Content:{text}"); + + msg = new RoleDialogModel(AgentRole.Assistant, $"AI response exceeded max output length") + { + CurrentAgentId = agent.Id, + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + StopCompletion = true, + RenderedInstruction = string.Join("\r\n", renderedInstructions) + }; + await onMessageReceived(msg); + } + else + { + // Text response received + msg = new RoleDialogModel(AgentRole.Assistant, text) + { + CurrentAgentId = agent.Id, + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + RenderedInstruction = string.Join("\r\n", renderedInstructions), + Annotations = value.Annotations?.Select(x => new ChatAnnotation + { + Title = x.WebResourceTitle, + Url = x.WebResourceUri.AbsoluteUri, + StartIndex = x.StartIndex, + EndIndex = x.EndIndex + })?.ToList() + }; + await onMessageReceived(msg); + } + + return true; + } + + public async Task GetChatCompletionsStreamingAsync(Agent agent, List conversations) + { + var client = ProviderHelper.GetClient(Provider, _model, _services); + var chatClient = client.GetChatClient(_model); + var (prompt, messages, options) = PrepareOptions(agent, conversations); + + var hub = _services.GetRequiredService>>(); + var conv = _services.GetRequiredService(); + var messageId = conversations.LastOrDefault()?.MessageId ?? string.Empty; + + var contentHooks = _services.GetHooks(agent.Id); + // Before chat completion hook + foreach (var hook in contentHooks) + { + await hook.BeforeGenerating(agent, conversations); + } + + hub.Push(new() + { + EventName = ChatEvent.BeforeReceiveLlmStreamMessage, + RefId = conv.ConversationId, + Data = new RoleDialogModel(AgentRole.Assistant, string.Empty) + { + CurrentAgentId = agent.Id, + MessageId = messageId + } + }); + + using var textStream = new RealtimeTextStream(); + var toolCalls = new List(); + ChatTokenUsage? tokenUsage = null; + + var responseMessage = new RoleDialogModel(AgentRole.Assistant, string.Empty) + { + CurrentAgentId = agent.Id, + MessageId = messageId + }; + + var streamingCancellation = _services.GetRequiredService(); + var cancellationToken = streamingCancellation.GetToken(conv.ConversationId); + + try + { + await foreach (var choice in chatClient.CompleteChatStreamingAsync(messages, options, cancellationToken)) + { + tokenUsage = choice.Usage; + + if (!choice.ToolCallUpdates.IsNullOrEmpty()) + { + toolCalls.AddRange(choice.ToolCallUpdates); + } + + if (!choice.ContentUpdate.IsNullOrEmpty()) + { + var text = choice.ContentUpdate[0]?.Text ?? string.Empty; + textStream.Collect(text); + +#if DEBUG + _logger.LogCritical($"Content update: {text}"); +#endif + + var content = new RoleDialogModel(AgentRole.Assistant, text) + { + CurrentAgentId = agent.Id, + MessageId = messageId + }; + hub.Push(new() + { + EventName = ChatEvent.OnReceiveLlmStreamMessage, + RefId = conv.ConversationId, + Data = content + }); + } + + if (choice.FinishReason == ChatFinishReason.ToolCalls || choice.FinishReason == ChatFinishReason.FunctionCall) + { + var meta = toolCalls.FirstOrDefault(x => !string.IsNullOrEmpty(x.FunctionName)); + var functionName = meta?.FunctionName; + var toolCallId = meta?.ToolCallId; + var args = toolCalls.Where(x => x.FunctionArgumentsUpdate != null).Select(x => x.FunctionArgumentsUpdate.ToString()).ToList(); + var functionArgument = string.Join(string.Empty, args); + +#if DEBUG + _logger.LogCritical($"Tool Call (id: {toolCallId}) => {functionName}({functionArgument})"); +#endif + + responseMessage = new RoleDialogModel(AgentRole.Function, string.Empty) + { + CurrentAgentId = agent.Id, + MessageId = messageId, + ToolCallId = toolCallId, + FunctionName = functionName, + FunctionArgs = functionArgument + }; + } + else if (choice.FinishReason.HasValue) + { + var allText = textStream.GetText(); + _logger.LogCritical($"Text Content: {allText}"); + + responseMessage = new RoleDialogModel(AgentRole.Assistant, allText) + { + CurrentAgentId = agent.Id, + MessageId = messageId, + IsStreaming = true + }; + } + } + } + catch (OperationCanceledException) + { + _logger.LogWarning("Streaming was cancelled for conversation {ConversationId}", conv.ConversationId); + } + + // Build responseMessage from collected text when cancelled before FinishReason + if (cancellationToken.IsCancellationRequested && string.IsNullOrEmpty(responseMessage.Content)) + { + var allText = textStream.GetText(); + responseMessage = new RoleDialogModel(AgentRole.Assistant, allText) + { + CurrentAgentId = agent.Id, + MessageId = messageId, + IsStreaming = true + }; + } + + hub.Push(new() + { + EventName = ChatEvent.AfterReceiveLlmStreamMessage, + RefId = conv.ConversationId, + Data = responseMessage + }); + + + var inputTokenDetails = tokenUsage?.InputTokenDetails; + // After chat completion hook + foreach (var hook in contentHooks) + { + await hook.AfterGenerated(responseMessage, new TokenStatsModel + { + Prompt = prompt, + Provider = Provider, + Model = _model, + TextInputTokens = (tokenUsage?.InputTokenCount ?? 0) - (inputTokenDetails?.CachedTokenCount ?? 0), + CachedTextInputTokens = inputTokenDetails?.CachedTokenCount ?? 0, + TextOutputTokens = tokenUsage?.OutputTokenCount ?? 0 + }); + } + + return responseMessage; + } + + public void SetModelName(string model) + { + _model = model; + } + + protected (string, IEnumerable, ChatCompletionOptions) PrepareOptions(Agent agent, List conversations) + { + var agentService = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + var settingsService = _services.GetRequiredService(); + var settings = settingsService.GetSetting(Provider, _model); + var allowMultiModal = settings != null && settings.MultiModal; + renderedInstructions = []; + + var messages = new List(); + var options = InitChatCompletionOption(agent); + + // Prepare instruction and functions + var renderData = agentService.CollectRenderData(agent); + var (instruction, functions) = agentService.PrepareInstructionAndFunctions(agent, renderData); + if (!string.IsNullOrWhiteSpace(instruction)) + { + renderedInstructions.Add(instruction); + messages.Add(new SystemChatMessage(instruction)); + } + + // Render functions + if (options.WebSearchOptions == null) + { + foreach (var function in functions) + { + if (!agentService.RenderFunction(agent, function, renderData)) + { + continue; + } + + var property = agentService.RenderFunctionProperty(agent, function, renderData); + + options.Tools.Add(ChatTool.CreateFunctionTool( + functionName: function.Name, + functionDescription: function.Description, + functionParameters: BinaryData.FromObjectAsJson(property))); + } + } + + if (!string.IsNullOrEmpty(agent.Knowledges)) + { + messages.Add(new SystemChatMessage(agent.Knowledges)); + } + + var filteredMessages = conversations.Select(x => x).ToList(); + var firstUserMsgIdx = filteredMessages.FindIndex(x => x.Role == AgentRole.User); + if (firstUserMsgIdx > 0) + { + filteredMessages = filteredMessages.Where((_, idx) => idx >= firstUserMsgIdx).ToList(); + } + + var imageDetailLevel = ChatImageDetailLevel.Auto; + if (allowMultiModal) + { + imageDetailLevel = ParseChatImageDetailLevel(state.GetState("chat_image_detail_level")); + } + + foreach (var message in filteredMessages) + { + if (message.Role == AgentRole.Function) + { + messages.Add(new AssistantChatMessage(new List + { + ChatToolCall.CreateFunctionToolCall(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? "{}")) + })); + + messages.Add(new ToolChatMessage(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.LlmContent)); + } + else if (message.Role == AgentRole.User) + { + var text = message.LlmContent; + var textPart = ChatMessageContentPart.CreateTextPart(text); + var contentParts = new List { textPart }; + + if (allowMultiModal && !message.Files.IsNullOrEmpty()) + { + CollectMessageContentParts(contentParts, message.Files, imageDetailLevel); + } + messages.Add(new UserChatMessage(contentParts) { ParticipantName = message.FunctionName }); + } + else if (message.Role == AgentRole.Assistant) + { + var text = message.LlmContent; + var textPart = ChatMessageContentPart.CreateTextPart(text); + var contentParts = new List { textPart }; + + if (allowMultiModal && !message.Files.IsNullOrEmpty()) + { + CollectMessageContentParts(contentParts, message.Files, imageDetailLevel); + } + messages.Add(new AssistantChatMessage(contentParts)); + } + } + + var prompt = GetPrompt(messages, options); + return (prompt, messages, options); + } + + private void CollectMessageContentParts(List contentParts, List files, ChatImageDetailLevel imageDetailLevel) + { + foreach (var file in files) + { + if (!string.IsNullOrEmpty(file.FileData)) + { + var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData); + var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileStorageUrl)) + { + var fileStorage = _services.GetRequiredService(); + var binary = fileStorage.GetFileBytes(file.FileStorageUrl); + var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); + var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType.IfNullOrEmptyAs(file.ContentType), imageDetailLevel); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileUrl)) + { + var uri = new Uri(file.FileUrl); + var contentPart = ChatMessageContentPart.CreateImagePart(uri, imageDetailLevel); + contentParts.Add(contentPart); + } + } + } + + private string GetPrompt(IEnumerable messages, ChatCompletionOptions options) + { + var prompt = string.Empty; + + if (!messages.IsNullOrEmpty()) + { + // System instruction + var verbose = string.Join("\r\n", messages + .Select(x => x as SystemChatMessage) + .Where(x => x != null) + .Select(x => + { + if (!string.IsNullOrEmpty(x.ParticipantName)) + { + // To display Agent name in log + return $"[{x.ParticipantName}]: {x.Content.FirstOrDefault()?.Text ?? string.Empty}"; + } + return $"{AgentRole.System}: {x.Content.FirstOrDefault()?.Text ?? string.Empty}"; + })); + prompt += $"{verbose}\r\n"; + + prompt += "\r\n[CONVERSATION]"; + verbose = string.Join("\r\n", messages + .Where(x => x as SystemChatMessage == null) + .Select(x => + { + var fnMessage = x as ToolChatMessage; + if (fnMessage != null) + { + return $"{AgentRole.Function}: {fnMessage.Content.FirstOrDefault()?.Text ?? string.Empty}"; + } + + var userMessage = x as UserChatMessage; + if (userMessage != null) + { + var content = x.Content.FirstOrDefault()?.Text ?? string.Empty; + return !string.IsNullOrEmpty(userMessage.ParticipantName) && userMessage.ParticipantName != "route_to_agent" ? + $"{userMessage.ParticipantName}: {content}" : + $"{AgentRole.User}: {content}"; + } + + var assistMessage = x as AssistantChatMessage; + if (assistMessage != null) + { + var toolCall = assistMessage.ToolCalls?.FirstOrDefault(); + return toolCall != null ? + $"{AgentRole.Assistant}: Call function {toolCall?.FunctionName}({toolCall?.FunctionArguments})" : + $"{AgentRole.Assistant}: {assistMessage.Content.FirstOrDefault()?.Text ?? string.Empty}"; + } + + return string.Empty; + })); + prompt += $"\r\n{verbose}\r\n"; + } + + if (!options.Tools.IsNullOrEmpty()) + { + var functions = string.Join("\r\n", options.Tools.Select(fn => + { + return $"\r\n{fn.FunctionName}: {fn.FunctionDescription}\r\n{fn.FunctionParameters}"; + })); + prompt += $"\r\n[FUNCTIONS]{functions}\r\n"; + } + + return prompt; + } + + private ChatCompletionOptions InitChatCompletionOption(Agent agent) + { + var state = _services.GetRequiredService(); + var settingsService = _services.GetRequiredService(); + var settings = settingsService.GetSetting(Provider, _model); + + // Reasoning effort + ChatReasoningEffortLevel? reasoningEffortLevel = null; + float? temperature = float.Parse(state.GetState("temperature", "0.0")); + if (settings?.Reasoning != null) + { + temperature = settings.Reasoning.Temperature; + var level = state.GetState("reasoning_effort_level") + .IfNullOrEmptyAs(agent?.LlmConfig?.ReasoningEffortLevel) + .IfNullOrEmptyAs(settings?.Reasoning?.EffortLevel); + reasoningEffortLevel = ParseReasoningEffortLevel(level); + } + + // Web search + ChatWebSearchOptions? webSearchOptions = null; + if (settings?.WebSearch != null) + { + temperature = null; + reasoningEffortLevel = null; + webSearchOptions = new(); + } + + var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens) + ? tokens + : agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN; + + return new ChatCompletionOptions() + { + Temperature = temperature, + MaxOutputTokenCount = maxTokens, + ReasoningEffortLevel = reasoningEffortLevel, + WebSearchOptions = webSearchOptions + }; + } + + private ChatReasoningEffortLevel? ParseReasoningEffortLevel(string? level) + { + if (string.IsNullOrWhiteSpace(level)) + { + return null; + } + + var effortLevel = new ChatReasoningEffortLevel("minimal"); + switch (level.ToLower()) + { + case "low": + effortLevel = ChatReasoningEffortLevel.Low; + break; + case "medium": + effortLevel = ChatReasoningEffortLevel.Medium; + break; + case "high": + effortLevel = ChatReasoningEffortLevel.High; + break; + default: + break; + } + + return effortLevel; + } + + private ChatImageDetailLevel ParseChatImageDetailLevel(string? level) + { + if (string.IsNullOrWhiteSpace(level)) + { + return ChatImageDetailLevel.Auto; + } + + var imageLevel = ChatImageDetailLevel.Auto; + switch (level.ToLower()) + { + case "low": + imageLevel = ChatImageDetailLevel.Low; + break; + case "high": + imageLevel = ChatImageDetailLevel.High; + break; + default: + break; + } + + return imageLevel; + } +} diff --git a/src/Plugins/BotSharp.Plugin.LiteLLM/Providers/ProviderHelper.cs b/src/Plugins/BotSharp.Plugin.LiteLLM/Providers/ProviderHelper.cs new file mode 100644 index 000000000..6a1cef5f8 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.LiteLLM/Providers/ProviderHelper.cs @@ -0,0 +1,16 @@ +using OpenAI; +using System.ClientModel; + +namespace BotSharp.Plugin.LiteLLM.Providers; + +public static class ProviderHelper +{ + public static OpenAIClient GetClient(string provider, string model, IServiceProvider services) + { + var settingsService = services.GetRequiredService(); + var settings = settingsService.GetSetting(provider, model); + var options = !string.IsNullOrEmpty(settings.Endpoint) ? + new OpenAIClientOptions { Endpoint = new Uri(settings.Endpoint) } : null; + return new OpenAIClient(new ApiKeyCredential(settings.ApiKey), options); + } +} diff --git a/src/Plugins/BotSharp.Plugin.LiteLLM/Providers/Text/TextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.LiteLLM/Providers/Text/TextCompletionProvider.cs new file mode 100644 index 000000000..50ae2321a --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.LiteLLM/Providers/Text/TextCompletionProvider.cs @@ -0,0 +1,101 @@ +using BotSharp.Abstraction.Hooks; +using Microsoft.Extensions.Logging; +using OpenAI.Chat; + +namespace BotSharp.Plugin.LiteLLM.Providers.Text; + +public class TextCompletionProvider : ITextCompletion +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + protected string _model; + + public string Provider => "litellm"; + public string Model => _model; + + public TextCompletionProvider( + IServiceProvider services, + ILogger logger) + { + _services = services; + _logger = logger; + } + + public async Task GetCompletion(string text, string agentId, string messageId) + { + var contentHooks = _services.GetHooks(agentId); + var state = _services.GetRequiredService(); + + // Before chat completion hook + var agent = new Agent() + { + Id = agentId, + }; + var message = new RoleDialogModel(AgentRole.User, text) + { + CurrentAgentId = agentId, + MessageId = messageId + }; + + foreach (var hook in contentHooks) + { + await hook.BeforeGenerating(agent, new List { message }); + } + + var client = ProviderHelper.GetClient(Provider, _model, _services); + var chatClient = client.GetChatClient(_model); + var options = PrepareOptions(); + var response = chatClient.CompleteChat([ new UserChatMessage(text) ], options); + + // AI response + var content = response.Value?.Content ?? []; + var completion = string.Empty; + foreach (var t in content) + { + completion += t?.Text ?? string.Empty; + }; + + // After chat completion hook + var responseMessage = new RoleDialogModel(AgentRole.Assistant, completion) + { + CurrentAgentId = agentId, + MessageId = messageId + }; + + var tokenUsage = response?.Value?.Usage; + var inputTokenDetails = response?.Value?.Usage?.InputTokenDetails; + + foreach (var hook in contentHooks) + { + await hook.AfterGenerated(responseMessage, new TokenStatsModel + { + Prompt = text, + Provider = Provider, + Model = _model, + TextInputTokens = (tokenUsage?.InputTokenCount ?? 0) - (inputTokenDetails?.CachedTokenCount ?? 0), + CachedTextInputTokens = inputTokenDetails?.CachedTokenCount ?? 0, + TextOutputTokens = tokenUsage?.OutputTokenCount ?? 0 + }); + } + + return completion.Trim(); + } + + public void SetModelName(string model) + { + _model = model; + } + + private ChatCompletionOptions PrepareOptions() + { + var state = _services.GetRequiredService(); + var temperature = float.Parse(state.GetState("temperature", "0.0")); + var maxTokens = int.Parse(state.GetState("max_tokens", "1024")); + + return new ChatCompletionOptions + { + Temperature = temperature, + MaxOutputTokenCount = maxTokens + }; + } +} diff --git a/src/Plugins/BotSharp.Plugin.LiteLLM/Using.cs b/src/Plugins/BotSharp.Plugin.LiteLLM/Using.cs new file mode 100644 index 000000000..751f1aec8 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.LiteLLM/Using.cs @@ -0,0 +1,18 @@ +global using System; +global using System.Collections.Generic; +global using System.Text; +global using System.Threading.Tasks; +global using System.Linq; +global using System.Text.Json; +global using Microsoft.Extensions.Configuration; +global using Microsoft.Extensions.DependencyInjection; +global using BotSharp.Abstraction.Conversations.Models; +global using BotSharp.Abstraction.Agents.Models; +global using BotSharp.Abstraction.MLTasks; +global using BotSharp.Abstraction.Agents; +global using BotSharp.Abstraction.Agents.Constants; +global using BotSharp.Abstraction.Agents.Enums; +global using BotSharp.Abstraction.Conversations; +global using BotSharp.Abstraction.Loggers; +global using BotSharp.Abstraction.Functions.Models; +global using BotSharp.Abstraction.Utilities; \ No newline at end of file diff --git a/src/WebStarter/WebStarter.csproj b/src/WebStarter/WebStarter.csproj index 6f7129e45..7af0d2ce5 100644 --- a/src/WebStarter/WebStarter.csproj +++ b/src/WebStarter/WebStarter.csproj @@ -62,6 +62,7 @@ + diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 34dd1afb8..59c4186ab 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -594,6 +594,29 @@ } ] }, + { + "Provider": "litellm", + "Models": [ + { + "Name": "gpt-4o", + "ApiKey": "", + "Endpoint": "http://localhost:4000/v1/", + "Type": "chat", + "Capabilities": [ + "Chat", + "ImageReading" + ], + "Cost": { + "TextInputCost": 0, + "CachedTextInputCost": 0, + "AudioInputCost": 0, + "CachedAudioInputCost": 0, + "TextOutputCost": 0, + "AudioOutputCost": 0 + } + } + ] + }, { "Provider": "minimax", "Models": [ @@ -1512,6 +1535,7 @@ "BotSharp.Plugin.GoogleAI", "BotSharp.Plugin.MetaAI", "BotSharp.Plugin.DeepSeekAI", + "BotSharp.Plugin.LiteLLM", "BotSharp.Plugin.MiniMaxAI", "BotSharp.Plugin.MetaMessenger", "BotSharp.Plugin.HuggingFace", diff --git a/tests/BotSharp.LLM.Tests/BotSharp.LLM.Tests.csproj b/tests/BotSharp.LLM.Tests/BotSharp.LLM.Tests.csproj index 0223aac31..d0181fadd 100644 --- a/tests/BotSharp.LLM.Tests/BotSharp.LLM.Tests.csproj +++ b/tests/BotSharp.LLM.Tests/BotSharp.LLM.Tests.csproj @@ -26,6 +26,7 @@ + diff --git a/tests/BotSharp.LLM.Tests/ChatCompletionTests.cs b/tests/BotSharp.LLM.Tests/ChatCompletionTests.cs index f6c06dc46..f96ed1453 100644 --- a/tests/BotSharp.LLM.Tests/ChatCompletionTests.cs +++ b/tests/BotSharp.LLM.Tests/ChatCompletionTests.cs @@ -55,6 +55,13 @@ public static IEnumerable CreateTestLLMProviders() (services, configuration, modelName) = LLMProvider.CreateAnthropic(); yield return new object[] { services.BuildServiceProvider().GetService() ?? throw new Exception("Error while initializing"), agent, modelName }; } + + if (LLMProvider.CanRunLiteLLM) + { + //LiteLLM + (services, configuration, modelName) = LLMProvider.CreateLiteLLM(); + yield return new object[] { services.BuildServiceProvider().GetService() ?? throw new Exception("Error while initializing"), agent, modelName }; + } } public ChatCompletionTests() { @@ -97,7 +104,6 @@ public async Task GetChatCompletionsStreamingAsync_Test(IChatCompletion chatComp { chatCompletion.SetModelName(modelName); - RoleDialogModel reply = null; var messages = new List { new RoleDialogModel(AgentRole.User, "write a poem about stars") @@ -105,8 +111,7 @@ public async Task GetChatCompletionsStreamingAsync_Test(IChatCompletion chatComp var result = await chatCompletion.GetChatCompletionsStreamingAsync(agent, messages); result.ShouldNotBeNull(); - reply.ShouldNotBeNull(); - reply.Content.ShouldNotBeNullOrEmpty(); + result.Content.ShouldNotBeNullOrEmpty(); } } } \ No newline at end of file diff --git a/tests/BotSharp.LLM.Tests/Core/LLMProvider.cs b/tests/BotSharp.LLM.Tests/Core/LLMProvider.cs index 7b4fd6523..b71014794 100644 --- a/tests/BotSharp.LLM.Tests/Core/LLMProvider.cs +++ b/tests/BotSharp.LLM.Tests/Core/LLMProvider.cs @@ -7,12 +7,18 @@ using BotSharp.Abstraction.Settings; using BotSharp.Core; using BotSharp.Core.Infrastructures; +using BotSharp.Core.MessageHub; +using BotSharp.Core.Conversations.Services; +using BotSharp.Abstraction.MessageHub.Models; +using BotSharp.Abstraction.Conversations; +using BotSharp.Abstraction.Conversations.Models; using BotSharp.Plugin.AnthropicAI; using BotSharp.Plugin.AnthropicAI.Settings; using BotSharp.Plugin.GoogleAi; using BotSharp.Plugin.GoogleAi.Settings; using BotSharp.Plugin.OpenAI; using BotSharp.Plugin.OpenAI.Settings; +using BotSharp.Plugin.LiteLLM; using GenerativeAI; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; @@ -25,7 +31,8 @@ public static class LLMProvider public static bool CanRunGemini => !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("GOOGLE_API_KEY")); public static bool CanRunOpenAI => !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("OPEN_AI_APIKEY")); public static bool CanRunAnthropic => !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("ANTHROPIC_API_KEY")); - + public static bool CanRunLiteLLM => !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("LITELLM_API_BASE")); + private static ILoggerFactory _loggerFactory = LoggerFactory.Create((builder) => builder.AddConsole()); public static (IServiceCollection services, IConfiguration config, string modelName) CreateGemini() { @@ -120,6 +127,10 @@ private static void AddCommonServices(ServiceCollection services, IConfiguration services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + // Services required by the streaming chat completion path. + services.AddSingleton>>(); + services.AddSingleton(); + services.AddSingleton(); services.AddLogging(s=>s.AddConsole()); } @@ -157,5 +168,39 @@ public static (IServiceCollection services, IConfiguration configuration, string new AnthropicPlugin().RegisterDI(services, configuration); return (services, configuration, modelName); } + + public static (IServiceCollection services, IConfiguration configuration, string modelName) CreateLiteLLM() + { + // Model name / alias configured on your LiteLLM proxy (defaults to "gpt-4o"). + string modelName = Environment.GetEnvironmentVariable("LITELLM_MODEL") ?? "gpt-4o"; + + // Base URL of the LiteLLM proxy, e.g. http://localhost:4000/v1/ + var endpoint = Environment.GetEnvironmentVariable("LITELLM_API_BASE") ?? + throw new Exception("LITELLM_API_BASE is not set"); + // Proxy master/virtual key. Optional - only required if the proxy enforces auth. + var apiKey = Environment.GetEnvironmentVariable("LITELLM_API_KEY") ?? "sk-1234"; + + var services = new ServiceCollection(); + + var configuration = new ConfigurationBuilder().AddJsonFile("appsettings.json").Build(); + + LlmProviderSetting setting = new LlmProviderSetting(); + setting.Provider = "litellm"; + setting.Models = new List([ + new LlmModelSetting() + { + Name = modelName, + ApiKey = apiKey, + Endpoint = endpoint + } + ]); + + services.AddSingleton>(new List([ setting])); + + AddCommonServices(services, configuration); + + new LiteLLMPlugin().RegisterDI(services, configuration); + return (services, configuration, modelName); + } } } \ No newline at end of file diff --git a/tests/BotSharp.LLM.Tests/Core/NullConversationService.cs b/tests/BotSharp.LLM.Tests/Core/NullConversationService.cs new file mode 100644 index 000000000..7ec9cd2f3 --- /dev/null +++ b/tests/BotSharp.LLM.Tests/Core/NullConversationService.cs @@ -0,0 +1,47 @@ +using BotSharp.Abstraction.Conversations; +using BotSharp.Abstraction.Conversations.Models; +using BotSharp.Abstraction.Models; +using BotSharp.Abstraction.Repositories.Filters; +using BotSharp.Abstraction.Utilities; + +namespace BotSharp.Plugin.Google.Core +{ + /// + /// Minimal IConversationService test stub. Only ConversationId is exercised by the + /// streaming code path; the rest satisfy the interface and are not called in tests. + /// + public class NullConversationService : IConversationService + { + private readonly IConversationStateService _states; + + public NullConversationService(IConversationStateService states) + { + _states = states; + } + + public IConversationStateService States => _states; + public string ConversationId => "test-conversation-id"; + + public Task NewConversation(Conversation conversation) => Task.FromResult(conversation); + public Task SetConversationId(string conversationId, List states, bool isReadOnly = false) => Task.CompletedTask; + public Task GetConversation(string id, bool isLoadStates = false) => Task.FromResult(new Conversation()); + public Task> GetConversations(ConversationFilter filter) => Task.FromResult(new PagedItems()); + public Task UpdateConversationTitle(string id, string title) => Task.FromResult(true); + public Task UpdateConversationTitleAlias(string id, string titleAlias) => Task.FromResult(true); + public Task UpdateConversationTags(string conversationId, List toAddTags, List toDeleteTags) => Task.FromResult(true); + public Task UpdateConversationMessage(string conversationId, UpdateMessageRequest request) => Task.FromResult(true); + public Task> GetLastConversations() => Task.FromResult(new List()); + public Task> GetIdleConversations(int batchSize, int messageLimit, int bufferHours, IEnumerable excludeAgentIds) => Task.FromResult(new List()); + public Task DeleteConversations(IEnumerable ids) => Task.FromResult(true); + public Task TruncateConversation(string conversationId, string messageId, string? newMessageId = null) => Task.FromResult(true); + public Task SendMessage(string agentId, RoleDialogModel message, PostbackMessageModel? replyMessage, Func onResponseReceived) => Task.FromResult(true); + public Task> GetDialogHistory(int lastCount = 100, bool fromBreakpoint = true, IEnumerable? includeMessageTypes = null, ConversationDialogFilter? filter = null) => Task.FromResult(new List()); + public Task CleanHistory(string agentId) => Task.CompletedTask; + public Task UpdateBreakpoint(bool resetStates = false, string? reason = null, params string[] excludedStates) => Task.CompletedTask; + public Task GetConversationSummary(ConversationSummaryModel model) => Task.FromResult(string.Empty); + public Task GetConversationRecordOrCreateNew(string agentId) => Task.FromResult(new Conversation()); + public bool IsConversationMode() => false; + public Task SaveStates() => Task.CompletedTask; + public Task> GetConversationStateSearhKeys(ConversationStateKeysFilter filter) => Task.FromResult(new List()); + } +}