Skip to content

Commit

Permalink
.Net: Add support for Base64 encoded images in MistralAI (#10180)
Browse files Browse the repository at this point in the history
### Motivation and Context

Closes #10166 

### Description

<!-- Describe your changes, the overall approach, the underlying design.
These notes will help understanding how your code works. Thanks! -->

### Contribution Checklist

<!-- Before submitting this PR, please make sure: -->

- [ ] The code builds clean without any errors or warnings
- [ ] The PR follows the [SK Contribution
Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
- [ ] All unit tests pass, and I have added new tests where possible
- [ ] I didn't break anyone 😄
  • Loading branch information
markwallace-microsoft authored Jan 21, 2025
1 parent 7a61c9c commit 704fbdf
Show file tree
Hide file tree
Showing 6 changed files with 92 additions and 5 deletions.
57 changes: 57 additions & 0 deletions dotnet/samples/Concepts/ChatCompletion/MistralAI_ChatCompletion.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
// Copyright (c) Microsoft. All rights reserved.

using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.ChatCompletion;
using Microsoft.SemanticKernel.Connectors.MistralAI;

namespace ChatCompletion;

// The following example shows how to use Semantic Kernel with MistralAI API
public class MistralAI_ChatCompletion(ITestOutputHelper output) : BaseTest(output)
{
[Fact]
public async Task GetChatMessageContentAsync()
{
Assert.NotNull(TestConfiguration.MistralAI.ChatModelId);
Assert.NotNull(TestConfiguration.MistralAI.ApiKey);

MistralAIChatCompletionService chatService = new(
modelId: TestConfiguration.MistralAI.ChatModelId,
apiKey: TestConfiguration.MistralAI.ApiKey);

var chatHistory = new ChatHistory("You are a librarian, expert about books");

chatHistory.AddUserMessage("Hi, I'm looking for book suggestions");
this.OutputLastMessage(chatHistory);

var reply = await chatService.GetChatMessageContentAsync(chatHistory, new MistralAIPromptExecutionSettings { MaxTokens = 200 });
Console.WriteLine(reply);
}

[Fact]
public async Task GetChatMessageContentUsingImageContentAsync()
{
Assert.NotNull(TestConfiguration.MistralAI.ImageModelId);
Assert.NotNull(TestConfiguration.MistralAI.ApiKey);

// Create a logging handler to output HTTP requests and responses
var handler = new LoggingHandler(new HttpClientHandler(), this.Output);
var httpClient = new HttpClient(handler);

MistralAIChatCompletionService chatService = new(
modelId: TestConfiguration.MistralAI.ImageModelId,
apiKey: TestConfiguration.MistralAI.ApiKey,
httpClient: httpClient);

var chatHistory = new ChatHistory();

var chatMessage = new ChatMessageContent(AuthorRole.User, "What's in this image?");
chatMessage.Items.Add(new ImageContent("data:image/jpeg;base64,/9j/4AAQSkZJRgABAQEA2ADYAAD/2wBDAAIBAQIBAQICAgICAgICAwUDAwMDAwYEBAMFBwYHBwcGBwcICQsJCAgKCAcHCg0KCgsMDAwMBwkODw0MDgsMDAz/2wBDAQICAgMDAwYDAwYMCAcIDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAz/wAARCAAQABADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD5rooor8DP9oD/2Q=="));

chatHistory.Add(chatMessage);
this.OutputLastMessage(chatHistory);

var reply = await chatService.GetChatMessageContentAsync(chatHistory, new MistralAIPromptExecutionSettings { MaxTokens = 200 });
Console.WriteLine(reply);
}
}
1 change: 1 addition & 0 deletions dotnet/samples/Concepts/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ dotnet test -l "console;verbosity=detailed" --filter "FullyQualifiedName=ChatCom
- [OpenAI_FunctionCalling](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/OpenAI_FunctionCalling.cs)
- [OpenAI_ReasonedFunctionCalling](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/OpenAI_ReasonedFunctionCalling.cs)
- [MultipleProviders_ChatHistoryReducer](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/MuiltipleProviders_ChatHistoryReducer.cs)
- [MistralAI_ChatCompletion](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/MistralAI_ChatCompletion.cs)
- [MistralAI_ChatPrompt](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/MistralAI_ChatPrompt.cs)
- [MistralAI_FunctionCalling](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/MistralAI_FunctionCalling.cs)
- [MistralAI_StreamingFunctionCalling](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/MistralAI_StreamingFunctionCalling.cs)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
// Copyright (c) Microsoft. All rights reserved.

using System;
using System.Text.Json.Serialization;

namespace Microsoft.SemanticKernel.Connectors.MistralAI.Client;
internal class ImageUrlChunk(Uri imageUrl) : ContentChunk(ContentChunkType.ImageUrl)
internal class ImageUrlChunk(string imageUrl) : ContentChunk(ContentChunkType.ImageUrl)
{
[JsonPropertyName("image_url")]
public string ImageUrl { get; set; } = imageUrl.ToString();
public string ImageUrl { get; set; } = imageUrl;
}
Original file line number Diff line number Diff line change
Expand Up @@ -784,9 +784,16 @@ internal List<MistralChatMessage> ToMistralChatMessages(ChatMessageContent chatM
{
content.Add(new TextChunk(textContent.Text!));
}
else if (item is ImageContent imageContent && imageContent.Uri is not null)
else if (item is ImageContent imageContent)
{
content.Add(new ImageUrlChunk(imageContent.Uri));
if (imageContent.Uri is not null)
{
content.Add(new ImageUrlChunk(imageContent.Uri.ToString()));
}
else if (imageContent.DataUri is not null)
{
content.Add(new ImageUrlChunk(imageContent.DataUri));
}
}
else
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,28 @@ public async Task ValidateGetChatMessageContentsWithImageAsync()
Assert.Contains("Snow", response[0].Content, System.StringComparison.InvariantCultureIgnoreCase);
}

[Fact(Skip = "This test is for manual verification.")]
public async Task ValidateGetChatMessageContentsWithImageDataUriAsync()
{
// Arrange
var model = this._configuration["MistralAI:ImageModelId"];
var apiKey = this._configuration["MistralAI:ApiKey"];
var service = new MistralAIChatCompletionService(model!, apiKey!, httpClient: this._httpClient);

// Act
var chatHistory = new ChatHistory
{
new ChatMessageContent(AuthorRole.User, "What's in this image?"),
new ChatMessageContent(AuthorRole.User, [new ImageContent("data:image/jpeg;base64,/9j/4AAQSkZJRgABAQEA2ADYAAD/2wBDAAIBAQIBAQICAgICAgICAwUDAwMDAwYEBAMFBwYHBwcGBwcICQsJCAgKCAcHCg0KCgsMDAwMBwkODw0MDgsMDAz/2wBDAQICAgMDAwYDAwYMCAcIDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAz/wAARCAAQABADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD5rooor8DP9oD/2Q==")])
};
var response = await service.GetChatMessageContentsAsync(chatHistory, this._executionSettings);

// Assert
Assert.NotNull(response);
Assert.Single(response);
Assert.Contains("square", response[0].Content, System.StringComparison.InvariantCultureIgnoreCase);
}

[Fact(Skip = "This test is for manual verification.")]
public async Task ValidateGetChatMessageContentsWithImageAndJsonFormatAsync()
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,7 @@ public class MistralAIConfig
public string ApiKey { get; set; }
public string ChatModelId { get; set; }
public string EmbeddingModelId { get; set; }
public string ImageModelId { get; set; }
}

public class GoogleAIConfig
Expand Down

0 comments on commit 704fbdf

Please sign in to comment.