feat: Support multimodal embedding for Gemini Embedding 2.0 and MaaS models in Models.embed_content() (Vertex AI API)

README.md

@@ -586,12 +586,13 @@ public class ComputeTokens {
 #### Embed Content
 
 The `embedContent` method allows you to generate embeddings for words, phrases,
-sentences, and code. Note that only text embedding is supported in this method.
+sentences, and code, as well as multimodal content like images or videos via Vertex AI.
 
 ```java
 package <your package name>;
 
 import com.google.genai.Client;
+import com.google.genai.types.EmbedContentConfig;
 import com.google.genai.types.EmbedContentResponse;
 
 public class EmbedContent {
@@ -602,6 +603,24 @@ public class EmbedContent {
         client.models.embedContent("gemini-embedding-001", "why is the sky blue?", null);
 
     System.out.println("Embedding response: " + response);
+
+    // Multimodal embedding with Vertex AI
+    Client vertexClient = Client.builder().vertexAI(true).build();
+    EmbedContentConfig config =
+        EmbedContentConfig.builder()
+            .outputDimensionality(10)
+            .title("test_title")
+            .taskType("RETRIEVAL_DOCUMENT")
+            .build();
+
+    EmbedContentResponse mmResponse =
+        vertexClient.models.embedContent(
+            "gemini-embedding-2-exp-11-2025",
+            Content.fromParts(
+                Part.fromText("Hello"),
+                Part.fromUri("gs://cloud-samples-data/generative-ai/image/a-man-and-a-dog.png", "image/png")),
+            config);
+    System.out.println("Multimodal embedding response: " + mmResponse);
   }
 }
 ```

examples/src/main/java/com/google/genai/examples/Constants.java

@@ -61,6 +61,10 @@ private Constants() {}
   /** The name of the embedding model to be used in the examples. */
   public static final String EMBEDDING_MODEL_NAME = "text-embedding-004";
 
+  /** The name of the vertex multimodal embedding model to be used in the examples. */
+  public static final String VERTEX_MULTIMODAL_EMBEDDING_MODEL_NAME =
+      "gemini-embedding-2-exp-11-2025";
+
   /** The file path to be used in the files operations examples. */
   public static final String UPLOAD_FILE_PATH = "./resources/test.txt";
 }

examples/src/main/java/com/google/genai/examples/EmbedContent.java

@@ -41,7 +41,11 @@
 package com.google.genai.examples;
 
 import com.google.genai.Client;
+import com.google.genai.types.Content;
 import com.google.genai.types.EmbedContentResponse;
+import com.google.genai.types.FileData;
+import com.google.genai.types.Part;
+import java.util.Arrays;
 
 /** An example of using the Unified Gen AI Java SDK to embed content. */
 public final class EmbedContent {
@@ -73,6 +77,24 @@ public static void main(String[] args) {
         client.models.embedContent(modelId, "why is the sky blue?", null);
 
     System.out.println("Embedding response: " + response);
+
+    if (client.vertexAI()) {
+      System.out.println("Embed content with GCS image example.");
+      Part textPart = Part.builder().text("What is in this image?").build();
+      Part imagePart =
+          Part.builder()
+              .fileData(
+                  FileData.builder()
+                      .fileUri("gs://cloud-samples-data/generative-ai/image/a-man-and-a-dog.png")
+                      .mimeType("image/png")
+                      .build())
+              .build();
+      Content content = Content.builder().parts(Arrays.asList(textPart, imagePart)).build();
+      response =
+          client.models.embedContent(
+              Constants.VERTEX_MULTIMODAL_EMBEDDING_MODEL_NAME, content, null);
+      System.out.println("Embedding response with GCS image: " + response);
+    }
   }
 
   private EmbedContent() {}

examples/src/main/java/com/google/genai/examples/EmbedContentAsync.java

@@ -41,7 +41,11 @@
 package com.google.genai.examples;
 
 import com.google.genai.Client;
+import com.google.genai.types.Content;
 import com.google.genai.types.EmbedContentResponse;
+import com.google.genai.types.FileData;
+import com.google.genai.types.Part;
+import java.util.Arrays;
 import java.util.concurrent.CompletableFuture;
 
 /** An example of using the Unified Gen AI Java SDK to embed content asynchronously. */
@@ -79,6 +83,28 @@ public static void main(String[] args) {
               System.out.println("Async embedding response: " + response);
             })
         .join();
+
+    // Vertex Multimodal embedding.
+    if (client.vertexAI()) {
+      System.out.println("Embed content with GCS image example.");
+      Part textPart = Part.builder().text("What is in this image?").build();
+      Part imagePart =
+          Part.builder()
+              .fileData(
+                  FileData.builder()
+                      .fileUri("gs://cloud-samples-data/generative-ai/image/a-man-and-a-dog.png")
+                      .mimeType("image/png")
+                      .build())
+              .build();
+      Content content = Content.builder().parts(Arrays.asList(textPart, imagePart)).build();
+      responseFuture =
+          client.async.models.embedContent(Constants.VERTEX_MULTIMODAL_EMBEDDING_MODEL_NAME, content, null);
+      responseFuture
+          .thenAccept(
+              response ->
+                  System.out.println("Async embedding response with GCS image: " + response))
+          .join();
+    }
   }
 
   private EmbedContentAsync() {}

src/main/java/com/google/genai/AsyncModels.java

@@ -38,8 +38,9 @@
 import com.google.genai.types.EditImageParameters;
 import com.google.genai.types.EditImageResponse;
 import com.google.genai.types.EmbedContentConfig;
-import com.google.genai.types.EmbedContentParameters;
+import com.google.genai.types.EmbedContentParametersPrivate;
 import com.google.genai.types.EmbedContentResponse;
+import com.google.genai.types.EmbeddingApiType;
 import com.google.genai.types.GenerateContentConfig;
 import com.google.genai.types.GenerateContentParameters;
 import com.google.genai.types.GenerateContentResponse;
@@ -145,20 +146,33 @@ CompletableFuture<ResponseStream<GenerateContentResponse>> privateGenerateConten
   }
 
   CompletableFuture<EmbedContentResponse> privateEmbedContent(
-      String model, List<Content> contents, EmbedContentConfig config) {
-    EmbedContentParameters.Builder parameterBuilder = EmbedContentParameters.builder();
+      String model,
+      List<Content> contents,
+      Content content,
+      EmbeddingApiType embeddingApiType,
+      EmbedContentConfig config) {
+    EmbedContentParametersPrivate.Builder parameterBuilder =
+        EmbedContentParametersPrivate.builder();
 
     if (!Common.isZero(model)) {
       parameterBuilder.model(model);
     }
     if (!Common.isZero(contents)) {
       parameterBuilder.contents(contents);
     }
+    if (!Common.isZero(content)) {
+      parameterBuilder.content(content);
+    }
+    if (!Common.isZero(embeddingApiType)) {
+      parameterBuilder.embeddingApiType(embeddingApiType);
+    }
     if (!Common.isZero(config)) {
       parameterBuilder.config(config);
     }
     JsonNode parameterNode = JsonSerializable.toJsonNode(parameterBuilder.build());
-    BuiltRequest builtRequest = models.buildRequestForPrivateEmbedContent(model, contents, config);
+    BuiltRequest builtRequest =
+        models.buildRequestForPrivateEmbedContent(
+            model, contents, content, embeddingApiType, config);
     return this.apiClient
         .asyncRequest("post", builtRequest.path(), builtRequest.body(), builtRequest.httpOptions())
         .thenApplyAsync(
@@ -956,7 +970,46 @@ public CompletableFuture<EmbedContentResponse> embedContent(
     for (String text : texts) {
       contents.add(Content.fromParts(Part.fromText(text)));
     }
-    return privateEmbedContent(model, contents, config);
+    Content content = null;
+    if (!contents.isEmpty()) {
+      content = contents.get(0);
+    }
+    boolean isVertexEmbedContentModel =
+        this.apiClient.vertexAI() && Transformers.tIsVertexEmbedContentModel(model);
+    if (isVertexEmbedContentModel && contents.size() > 1) {
+      throw new IllegalArgumentException(
+          "The embedContent API for this model only supports one content at a time.");
+    }
+    EmbeddingApiType apiType =
+        isVertexEmbedContentModel
+            ? new EmbeddingApiType("EMBED_CONTENT")
+            : new EmbeddingApiType("PREDICT");
+    return privateEmbedContent(model, contents, content, apiType, config);
+  }
+
+  /**
+   * Asynchronously embeds content given a GenAI model and a content object.
+   *
+   * @param model the name of the GenAI model to use for embedding
+   * @param content a {@link com.google.genai.types.Content} to send to the embedding model
+   * @return a {@link com.google.genai.types.EmbedContentResponse} instance that contains the
+   *     embedding.
+   */
+  public CompletableFuture<EmbedContentResponse> embedContent(
+      String model, Content content, EmbedContentConfig config) {
+    List<Content> contents = new ArrayList<>();
+    contents.add(content);
+    boolean isVertexEmbedContentModel =
+        this.apiClient.vertexAI() && Transformers.tIsVertexEmbedContentModel(model);
+    if (isVertexEmbedContentModel && contents.size() > 1) {
+      throw new IllegalArgumentException(
+          "The embedContent API for this model only supports one content at a time.");
+    }
+    EmbeddingApiType apiType =
+        isVertexEmbedContentModel
+            ? new EmbeddingApiType("EMBED_CONTENT")
+            : new EmbeddingApiType("PREDICT");
+    return privateEmbedContent(model, contents, content, apiType, config);
   }
 
   /**