.Net: Support Support BinaryContent in Gemini Connector (#13140)

Millmer · SergeyMenshykh · web-flow · commit b03c3d030c93 · 2025-11-03T13:22:35.000Z
### Motivation and Context Resolves: #13131 ### Description The Gemini connector actually already supports arbitrary base64 PDF content by prioritising binary data in the `CreateGeminiPartFromImage` function in the `GeminiRequest.cs`. I kept the current behaviour with the `ImageContent` and `AudioContent` but also added explicit support for the `BinaryContent` KernelContent type in the `GetGeminiPartFromKernelContent` function to bring it inline with how the other connectors work. I applied the same Chat Completion Unit test as those I found in the OpenAI Connector for BinaryContent. ### Contribution Checklist  - [x] The code builds clean without any errors or warnings - [x] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [x] All unit tests pass, and I have added new tests where possible - [x] I didn't break anyone (I think...) 😄 --------- Co-authored-by: SergeyMenshykh <68852919+SergeyMenshykh@users.noreply.github.com>
diff --git a/dotnet/samples/Concepts/ChatCompletion/Google_GeminiChatCompletionWithFile.cs b/dotnet/samples/Concepts/ChatCompletion/Google_GeminiChatCompletionWithFile.cs
@@ -0,0 +1,145 @@
+﻿// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.SemanticKernel;
+using Microsoft.SemanticKernel.ChatCompletion;
+using Resources;
+
+namespace ChatCompletion;
+
+/// <summary>
+/// This sample shows how to use binary file and inline Base64 inputs, like PDFs, with Google Gemini's chat completion.
+/// </summary>
+public class Google_GeminiChatCompletionWithFile(ITestOutputHelper output) : BaseTest(output)
+{
+    [Fact]
+    public async Task GoogleAIChatCompletionWithLocalFile()
+    {
+        Console.WriteLine("============= Google AI - Gemini Chat Completion With Local File =============");
+
+        Assert.NotNull(TestConfiguration.GoogleAI.ApiKey);
+        Assert.NotNull(TestConfiguration.GoogleAI.Gemini.ModelId);
+
+        Kernel kernel = Kernel.CreateBuilder()
+            .AddGoogleAIGeminiChatCompletion(TestConfiguration.GoogleAI.Gemini.ModelId, TestConfiguration.GoogleAI.ApiKey)
+            .Build();
+
+        var fileBytes = await EmbeddedResource.ReadAllAsync("employees.pdf");
+
+        var chatHistory = new ChatHistory("You are a friendly assistant.");
+        chatHistory.AddUserMessage(
+        [
+            new TextContent("What's in this file?"),
+            new BinaryContent(fileBytes, "application/pdf")
+        ]);
+
+        var chatCompletionService = kernel.GetRequiredService<IChatCompletionService>();
+
+        var reply = await chatCompletionService.GetChatMessageContentAsync(chatHistory);
+
+        Console.WriteLine(reply.Content);
+    }
+
+    [Fact]
+    public async Task VertexAIChatCompletionWithLocalFile()
+    {
+        Console.WriteLine("============= Vertex AI - Gemini Chat Completion With Local File =============");
+
+        Assert.NotNull(TestConfiguration.VertexAI.BearerKey);
+        Assert.NotNull(TestConfiguration.VertexAI.Location);
+        Assert.NotNull(TestConfiguration.VertexAI.ProjectId);
+        Assert.NotNull(TestConfiguration.VertexAI.Gemini.ModelId);
+
+        Kernel kernel = Kernel.CreateBuilder()
+            .AddVertexAIGeminiChatCompletion(
+                modelId: TestConfiguration.VertexAI.Gemini.ModelId,
+                bearerKey: TestConfiguration.VertexAI.BearerKey,
+                location: TestConfiguration.VertexAI.Location,
+                projectId: TestConfiguration.VertexAI.ProjectId)
+            .Build();
+
+        var fileBytes = await EmbeddedResource.ReadAllAsync("employees.pdf");
+
+        var chatHistory = new ChatHistory("You are a friendly assistant.");
+        chatHistory.AddUserMessage(
+        [
+            new TextContent("What's in this file?"),
+            new BinaryContent(fileBytes, "application/pdf"),
+        ]);
+
+        var chatCompletionService = kernel.GetRequiredService<IChatCompletionService>();
+
+        var reply = await chatCompletionService.GetChatMessageContentAsync(chatHistory);
+
+        Console.WriteLine(reply.Content);
+    }
+
+    [Fact]
+    public async Task GoogleAIChatCompletionWithBase64DataUri()
+    {
+        Console.WriteLine("============= Google AI - Gemini Chat Completion With Base64 Data Uri =============");
+
+        Assert.NotNull(TestConfiguration.GoogleAI.ApiKey);
+        Assert.NotNull(TestConfiguration.GoogleAI.Gemini.ModelId);
+
+        Kernel kernel = Kernel.CreateBuilder()
+            .AddGoogleAIGeminiChatCompletion(TestConfiguration.GoogleAI.Gemini.ModelId, TestConfiguration.GoogleAI.ApiKey)
+            .Build();
+
+        var fileBytes = await EmbeddedResource.ReadAllAsync("employees.pdf");
+        var fileBase64 = Convert.ToBase64String(fileBytes.ToArray());
+        var dataUri = $"data:application/pdf;base64,{fileBase64}";
+
+        var chatHistory = new ChatHistory("You are a friendly assistant.");
+        chatHistory.AddUserMessage(
+        [
+            new TextContent("What's in this file?"),
+            new BinaryContent(dataUri)
+            // Google AI Gemini AI does not support arbitrary URIs but we can convert a Base64 URI into InlineData with the correct mimeType.
+        ]);
+
+        var chatCompletionService = kernel.GetRequiredService<IChatCompletionService>();
+
+        var reply = await chatCompletionService.GetChatMessageContentAsync(chatHistory);
+
+        Console.WriteLine(reply.Content);
+    }
+
+    [Fact]
+    public async Task VertexAIChatCompletionWithBase64DataUri()
+    {
+        Console.WriteLine("============= Vertex AI - Gemini Chat Completion With Base64 Data Uri =============");
+
+        Assert.NotNull(TestConfiguration.VertexAI.BearerKey);
+        Assert.NotNull(TestConfiguration.VertexAI.Location);
+        Assert.NotNull(TestConfiguration.VertexAI.ProjectId);
+        Assert.NotNull(TestConfiguration.VertexAI.Gemini.ModelId);
+
+        Kernel kernel = Kernel.CreateBuilder()
+            .AddVertexAIGeminiChatCompletion(
+                modelId: TestConfiguration.VertexAI.Gemini.ModelId,
+                bearerKey: TestConfiguration.VertexAI.BearerKey,
+                location: TestConfiguration.VertexAI.Location,
+                projectId: TestConfiguration.VertexAI.ProjectId)
+            .Build();
+
+        var fileBytes = await EmbeddedResource.ReadAllAsync("employees.pdf");
+        var fileBase64 = Convert.ToBase64String(fileBytes.ToArray());
+        var dataUri = $"data:application/pdf;base64,{fileBase64}";
+
+        var chatHistory = new ChatHistory("You are a friendly assistant.");
+        chatHistory.AddUserMessage(
+        [
+            new TextContent("What's in this file?"),
+            new BinaryContent(dataUri)
+            // Vertex AI API does not support URIs outside of inline Base64 or GCS buckets within the same project. The bucket that stores the file must be in the same Google Cloud project that's sending the request. You must always provide the mimeType via the metadata property.
+            // var content = new BinaryContent(gs://generativeai-downloads/files/employees.pdf);
+            // content.Metadata = new Dictionary<string, object?> { { "mimeType", "application/pdf" } };
+        ]);
+
+        var chatCompletionService = kernel.GetRequiredService<IChatCompletionService>();
+
+        var reply = await chatCompletionService.GetChatMessageContentAsync(chatHistory);
+
+        Console.WriteLine(reply.Content);
+    }
+}
diff --git a/dotnet/samples/Concepts/README.md b/dotnet/samples/Concepts/README.md
@@ -64,6 +64,7 @@ dotnet test -l "console;verbosity=detailed" --filter "FullyQualifiedName=ChatCom
 - [Google_GeminiChatCompletion](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/Google_GeminiChatCompletion.cs)
 - [Google_GeminiChatCompletionStreaming](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/Google_GeminiChatCompletionStreaming.cs)
 - [Google_GeminiChatCompletionWithThinkingBudget](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/Google_GeminiChatCompletionWithThinkingBudget.cs)
+- [Google_GeminiChatCompletionWithFile.cs](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/Google_GeminiChatCompletionWithFile.cs)
 - [Google_GeminiGetModelResult](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/Google_GeminiGetModelResult.cs)
 - [Google_GeminiStructuredOutputs](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/Google_GeminiStructuredOutputs.cs)
 - [Google_GeminiVision](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/Google_GeminiVision.cs)
diff --git a/dotnet/src/Connectors/Connectors.Google.UnitTests/Core/Gemini/GeminiRequestTests.cs b/dotnet/src/Connectors/Connectors.Google.UnitTests/Core/Gemini/GeminiRequestTests.cs
@@ -371,6 +371,33 @@ public void FromChatHistoryAudioAsAudioContentItReturnsWithChatHistory()
                 .SequenceEqual(Convert.FromBase64String(c.Parts![0].InlineData!.InlineData))));
     }
 
+    [Fact]
+    public void FromChatHistoryPdfAsBinaryContentItReturnsWithChatHistory()
+    {
+        // Arrange
+        ReadOnlyMemory<byte> pdfAsBytes = new byte[] { 0x00, 0x01, 0x02, 0x03 };
+        ChatHistory chatHistory = [];
+        chatHistory.AddUserMessage("user-message");
+        chatHistory.AddAssistantMessage("assist-message");
+        chatHistory.AddUserMessage(contentItems:
+            [new BinaryContent(new Uri("https://example-file.com/file.pdf")) { MimeType = "application/pdf" }]);
+        chatHistory.AddUserMessage(contentItems:
+            [new BinaryContent(pdfAsBytes, "application/pdf")]);
+        var executionSettings = new GeminiPromptExecutionSettings();
+
+        // Act
+        var request = GeminiRequest.FromChatHistoryAndExecutionSettings(chatHistory, executionSettings);
+
+        // Assert
+        Assert.Collection(request.Contents,
+            c => Assert.Equal(chatHistory[0].Content, c.Parts![0].Text),
+            c => Assert.Equal(chatHistory[1].Content, c.Parts![0].Text),
+            c => Assert.Equal(chatHistory[2].Items.Cast<BinaryContent>().Single().Uri,
+                c.Parts![0].FileData!.FileUri),
+            c => Assert.True(pdfAsBytes.ToArray()
+                .SequenceEqual(Convert.FromBase64String(c.Parts![0].InlineData!.InlineData))));
+    }
+
     [Fact]
     public void FromChatHistoryUnsupportedContentItThrowsNotSupportedException()
     {
diff --git a/dotnet/src/Connectors/Connectors.Google.UnitTests/Services/GoogleAIGeminiChatCompletionServiceTests.cs b/dotnet/src/Connectors/Connectors.Google.UnitTests/Services/GoogleAIGeminiChatCompletionServiceTests.cs
@@ -5,7 +5,9 @@
 using System.IO;
 using System.Net.Http;
 using System.Text;
+using System.Text.Json;
 using System.Threading.Tasks;
+using Microsoft.SemanticKernel;
 using Microsoft.SemanticKernel.ChatCompletion;
 using Microsoft.SemanticKernel.Connectors.Google;
 using Microsoft.SemanticKernel.Services;
@@ -144,6 +146,78 @@ public async Task RequestBodyIncludesThinkingConfigWhenSetAsync(int? thinkingBud
         }
     }
 
+    [Fact]
+    public async Task GetChatMessageContentsAsyncThrowsExceptionWithEmptyBinaryContentAsync()
+    {
+        // Arrange
+        var sut = new GoogleAIGeminiChatCompletionService("gemini-2.5-pro", "key");
+
+        var chatHistory = new ChatHistory();
+        chatHistory.AddUserMessage([new BinaryContent()]);
+
+        // Act & Assert
+        await Assert.ThrowsAsync<InvalidOperationException>(() => sut.GetChatMessageContentsAsync(chatHistory));
+    }
+
+    [Fact]
+    public async Task GetChatMessageContentsThrowsExceptionUriOnlyReferenceBinaryContentAsync()
+    {
+        // Arrange
+        var sut = new GoogleAIGeminiChatCompletionService("gemini-2.5-pro", "key");
+
+        var chatHistory = new ChatHistory();
+        chatHistory.AddUserMessage([new BinaryContent(new Uri("file://testfile.pdf"))]);
+
+        // Act & Assert
+        await Assert.ThrowsAsync<InvalidOperationException>(() => sut.GetChatMessageContentsAsync(chatHistory));
+    }
+
+    [Theory]
+    [InlineData(true)]
+    [InlineData(false)]
+    public async Task ItSendsBinaryContentCorrectlyAsync(bool useUriData)
+    {
+        // Arrange
+        var sut = new GoogleAIGeminiChatCompletionService("gemini-2.5-pro", "key", httpClient: this._httpClient);
+
+        var mimeType = "application/pdf";
+        var chatHistory = new ChatHistory();
+        chatHistory.AddUserMessage([
+            new TextContent("What's in this file?"),
+            useUriData
+                ? new BinaryContent($"data:{mimeType};base64,{PdfBase64Data}")
+                : new BinaryContent(Convert.FromBase64String(PdfBase64Data), mimeType)
+        ]);
+
+        // Act
+        await sut.GetChatMessageContentsAsync(chatHistory);
+
+        // Assert
+        var actualRequestContent = Encoding.UTF8.GetString(this._messageHandlerStub.RequestContent!);
+        Assert.NotNull(actualRequestContent);
+        var optionsJson = JsonSerializer.Deserialize<JsonElement>(actualRequestContent);
+
+        var contents = optionsJson.GetProperty("contents");
+        Assert.Equal(1, contents.GetArrayLength());
+
+        var parts = contents[0].GetProperty("parts");
+        Assert.Equal(2, parts.GetArrayLength());
+
+        Assert.True(parts[0].TryGetProperty("text", out var prompt));
+        Assert.Equal("What's in this file?", prompt.ToString());
+
+        // Check for the file data
+        Assert.True(parts[1].TryGetProperty("inlineData", out var inlineData));
+        Assert.Equal(JsonValueKind.Object, inlineData.ValueKind);
+        Assert.Equal(mimeType, inlineData.GetProperty("mimeType").GetString());
+        Assert.Equal(PdfBase64Data, inlineData.GetProperty("data").ToString());
+    }
+
+    /// <summary>
+    /// Sample PDF data URI for testing.
+    /// </summary>
+    private const string PdfBase64Data = "JVBERi0xLjQKMSAwIG9iago8PC9UeXBlIC9DYXRhbG9nCi9QYWdlcyAyIDAgUgo+PgplbmRvYmoKMiAwIG9iago8PC9UeXBlIC9QYWdlcwovS2lkcyBbMyAwIFJdCi9Db3VudCAxCj4+CmVuZG9iagozIDAgb2JqCjw8L1R5cGUgL1BhZ2UKL1BhcmVudCAyIDAgUgovTWVkaWFCb3ggWzAgMCA1OTUgODQyXQovQ29udGVudHMgNSAwIFIKL1Jlc291cmNlcyA8PC9Qcm9jU2V0IFsvUERGIC9UZXh0XQovRm9udCA8PC9GMSA0IDAgUj4+Cj4+Cj4+CmVuZG9iago0IDAgb2JqCjw8L1R5cGUgL0ZvbnQKL1N1YnR5cGUgL1R5cGUxCi9OYW1lIC9GMQovQmFzZUZvbnQgL0hlbHZldGljYQovRW5jb2RpbmcgL01hY1JvbWFuRW5jb2RpbmcKPj4KZW5kb2JqCjUgMCBvYmoKPDwvTGVuZ3RoIDUzCj4+CnN0cmVhbQpCVAovRjEgMjAgVGYKMjIwIDQwMCBUZAooRHVtbXkgUERGKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZgowMDAwMDAwMDA5IDAwMDAwIG4KMDAwMDAwMDA2MyAwMDAwMCBuCjAwMDAwMDAxMjQgMDAwMDAgbgowMDAwMDAwMjc3IDAwMDAwIG4KMDAwMDAwMDM5MiAwMDAwMCBuCnRyYWlsZXIKPDwvU2l6ZSA2Ci9Sb290IDEgMCBSCj4+CnN0YXJ0eHJlZgo0OTUKJSVFT0YK";
+
     public void Dispose()
     {
         this._httpClient.Dispose();
diff --git a/dotnet/src/Connectors/Connectors.Google.UnitTests/Services/VertexAIGeminiChatCompletionServiceTests.cs b/dotnet/src/Connectors/Connectors.Google.UnitTests/Services/VertexAIGeminiChatCompletionServiceTests.cs
@@ -5,7 +5,9 @@
 using System.IO;
 using System.Net.Http;
 using System.Text;
+using System.Text.Json;
 using System.Threading.Tasks;
+using Microsoft.SemanticKernel;
 using Microsoft.SemanticKernel.ChatCompletion;
 using Microsoft.SemanticKernel.Connectors.Google;
 using Microsoft.SemanticKernel.Services;
@@ -156,6 +158,78 @@ public async Task RequestBodyIncludesThinkingConfigWhenSetAsync(int? thinkingBud
         }
     }
 
+    [Fact]
+    public async Task GetChatMessageContentsAsyncThrowsExceptionWithEmptyBinaryContentAsync()
+    {
+        // Arrange
+        var sut = new VertexAIGeminiChatCompletionService("gemini-2.5-pro", "key", "location", "project");
+
+        var chatHistory = new ChatHistory();
+        chatHistory.AddUserMessage([new BinaryContent()]);
+
+        // Act & Assert
+        await Assert.ThrowsAsync<InvalidOperationException>(() => sut.GetChatMessageContentsAsync(chatHistory));
+    }
+
+    [Fact]
+    public async Task GetChatMessageContentsThrowsExceptionUriOnlyReferenceBinaryContentAsync()
+    {
+        // Arrange
+        var sut = new VertexAIGeminiChatCompletionService("gemini-2.5-pro", "key", "location", "project");
+
+        var chatHistory = new ChatHistory();
+        chatHistory.AddUserMessage([new BinaryContent(new Uri("file://testfile.pdf"))]);
+
+        // Act & Assert
+        await Assert.ThrowsAsync<InvalidOperationException>(() => sut.GetChatMessageContentsAsync(chatHistory));
+    }
+
+    [Theory]
+    [InlineData(true)]
+    [InlineData(false)]
+    public async Task ItSendsBinaryContentCorrectlyAsync(bool useUriData)
+    {
+        // Arrange
+        var sut = new VertexAIGeminiChatCompletionService("gemini-2.5-pro", "key", "location", "project", httpClient: this._httpClient);
+
+        var mimeType = "application/pdf";
+        var chatHistory = new ChatHistory();
+        chatHistory.AddUserMessage([
+            new TextContent("What's in this file?"),
+            useUriData
+                ? new BinaryContent($"data:{mimeType};base64,{PdfBase64Data}")
+                : new BinaryContent(Convert.FromBase64String(PdfBase64Data), mimeType)
+        ]);
+
+        // Act
+        await sut.GetChatMessageContentsAsync(chatHistory);
+
+        // Assert
+        var actualRequestContent = Encoding.UTF8.GetString(this._messageHandlerStub.RequestContent!);
+        Assert.NotNull(actualRequestContent);
+        var optionsJson = JsonSerializer.Deserialize<JsonElement>(actualRequestContent);
+
+        var contents = optionsJson.GetProperty("contents");
+        Assert.Equal(1, contents.GetArrayLength());
+
+        var parts = contents[0].GetProperty("parts");
+        Assert.Equal(2, parts.GetArrayLength());
+
+        Assert.True(parts[0].TryGetProperty("text", out var prompt));
+        Assert.Equal("What's in this file?", prompt.ToString());
+
+        // Check for the file data
+        Assert.True(parts[1].TryGetProperty("inlineData", out var inlineData));
+        Assert.Equal(JsonValueKind.Object, inlineData.ValueKind);
+        Assert.Equal(mimeType, inlineData.GetProperty("mimeType").GetString());
+        Assert.Equal(PdfBase64Data, inlineData.GetProperty("data").ToString());
+    }
+
+    /// <summary>
+    /// Sample PDF data URI for testing.
+    /// </summary>
+    private const string PdfBase64Data = "JVBERi0xLjQKMSAwIG9iago8PC9UeXBlIC9DYXRhbG9nCi9QYWdlcyAyIDAgUgo+PgplbmRvYmoKMiAwIG9iago8PC9UeXBlIC9QYWdlcwovS2lkcyBbMyAwIFJdCi9Db3VudCAxCj4+CmVuZG9iagozIDAgb2JqCjw8L1R5cGUgL1BhZ2UKL1BhcmVudCAyIDAgUgovTWVkaWFCb3ggWzAgMCA1OTUgODQyXQovQ29udGVudHMgNSAwIFIKL1Jlc291cmNlcyA8PC9Qcm9jU2V0IFsvUERGIC9UZXh0XQovRm9udCA8PC9GMSA0IDAgUj4+Cj4+Cj4+CmVuZG9iago0IDAgb2JqCjw8L1R5cGUgL0ZvbnQKL1N1YnR5cGUgL1R5cGUxCi9OYW1lIC9GMQovQmFzZUZvbnQgL0hlbHZldGljYQovRW5jb2RpbmcgL01hY1JvbWFuRW5jb2RpbmcKPj4KZW5kb2JqCjUgMCBvYmoKPDwvTGVuZ3RoIDUzCj4+CnN0cmVhbQpCVAovRjEgMjAgVGYKMjIwIDQwMCBUZAooRHVtbXkgUERGKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZgowMDAwMDAwMDA5IDAwMDAwIG4KMDAwMDAwMDA2MyAwMDAwMCBuCjAwMDAwMDAxMjQgMDAwMDAgbgowMDAwMDAwMjc3IDAwMDAwIG4KMDAwMDAwMDM5MiAwMDAwMCBuCnRyYWlsZXIKPDwvU2l6ZSA2Ci9Sb290IDEgMCBSCj4+CnN0YXJ0eHJlZgo0OTUKJSVFT0YK";
+
     public void Dispose()
     {
         this._httpClient.Dispose();
diff --git a/dotnet/src/Connectors/Connectors.Google/Core/Gemini/Models/GeminiRequest.cs b/dotnet/src/Connectors/Connectors.Google/Core/Gemini/Models/GeminiRequest.cs
diff --git a/dotnet/src/IntegrationTests/Connectors/Google/Gemini/GeminiChatCompletionTests.cs b/dotnet/src/IntegrationTests/Connectors/Google/Gemini/GeminiChatCompletionTests.cs
