Backport: feat(provider/perplexity): Add PDF support (#10130)

This is an automated backport of #10089 to the release-v5.0 branch. FYI
@goyalshivansh2805

---------

Co-authored-by: Shivansh Goyal <[email protected]>
Co-authored-by: josh <[email protected]>

Author: 3 people

.changeset/chilly-bears-study.md

@@ -0,0 +1,5 @@
+---
+'@ai-sdk/perplexity': patch
+---
+
+Add PDF support to Perplexity provider

content/providers/01-ai-sdk-providers/70-perplexity.mdx

@@ -133,6 +133,48 @@ The metadata includes:
 
 You can enable image responses by setting `return_images: true` in the provider options. This feature is only available to Perplexity Tier-2 users and above.
 
+### PDF Support
+
+The Perplexity provider supports reading PDF files.
+You can pass PDF files as part of the message content using the `file` type:
+
+```ts
+const result = await generateText({
+  model: perplexity('sonar-pro'),
+  messages: [
+    {
+      role: 'user',
+      content: [
+        {
+          type: 'text',
+          text: 'What is this document about?',
+        },
+        {
+          type: 'file',
+          data: fs.readFileSync('./data/ai.pdf'),
+          mediaType: 'application/pdf',
+          filename: 'ai.pdf', // optional
+        },
+      ],
+    },
+  ],
+});
+```
+
+You can also pass the URL of a PDF:
+
+```ts
+{
+  type: 'file',
+  data: new URL('https://example.com/document.pdf'),
+  mediaType: 'application/pdf',
+  filename: 'document.pdf', // optional
+}
+```
+
+The model will have access to the contents of the PDF file and
+respond to questions about it.
+
 <Note>
   For more details about Perplexity's capabilities, see the [Perplexity chat
   completion docs](https://docs.perplexity.ai/api-reference/chat-completions).
@@ -152,3 +194,4 @@ You can enable image responses by setting `return_images: true` in the provider
   Please see the [Perplexity docs](https://docs.perplexity.ai) for detailed API
   documentation and the latest updates.
 </Note>
+```

examples/ai-core/src/generate-text/perplexity-pdf-url.ts

@@ -0,0 +1,30 @@
+import { perplexity } from '@ai-sdk/perplexity';
+import { generateText } from 'ai';
+import 'dotenv/config';
+
+async function main() {
+  const result = await generateText({
+    model: perplexity('sonar-pro'),
+    messages: [
+      {
+        role: 'user',
+        content: [
+          {
+            type: 'text',
+            text: 'What is this document about? Provide a brief summary.',
+          },
+          {
+            type: 'file',
+            data: new URL('https://example.com/path/to/document.pdf'),
+            mediaType: 'application/pdf',
+            filename: 'document.pdf',
+          },
+        ],
+      },
+    ],
+  });
+
+  console.log(result.text);
+}
+
+main().catch(console.error);

examples/ai-core/src/generate-text/perplexity-pdf.ts

@@ -0,0 +1,31 @@
+import { perplexity } from '@ai-sdk/perplexity';
+import { generateText } from 'ai';
+import 'dotenv/config';
+import fs from 'fs';
+
+async function main() {
+  const result = await generateText({
+    model: perplexity('sonar-pro'),
+    messages: [
+      {
+        role: 'user',
+        content: [
+          {
+            type: 'text',
+            text: 'What is this document about? Provide a brief summary.',
+          },
+          {
+            type: 'file',
+            data: fs.readFileSync('./data/ai.pdf'),
+            mediaType: 'application/pdf',
+            filename: 'ai.pdf',
+          },
+        ],
+      },
+    ],
+  });
+
+  console.log(result.text);
+}
+
+main().catch(console.error);

examples/ai-core/src/stream-text/perplexity-pdf.ts

@@ -0,0 +1,33 @@
+import { perplexity } from '@ai-sdk/perplexity';
+import { streamText } from 'ai';
+import 'dotenv/config';
+import fs from 'fs';
+
+async function main() {
+  const result = streamText({
+    model: perplexity('sonar-pro'),
+    messages: [
+      {
+        role: 'user',
+        content: [
+          {
+            type: 'text',
+            text: 'What is this document about? Provide a brief summary.',
+          },
+          {
+            type: 'file',
+            data: fs.readFileSync('./data/ai.pdf'),
+            mediaType: 'application/pdf',
+            filename: 'ai.pdf',
+          },
+        ],
+      },
+    ],
+  });
+
+  for await (const textPart of result.textStream) {
+    process.stdout.write(textPart);
+  }
+}
+
+main().catch(console.error);

packages/perplexity/src/convert-to-perplexity-messages.ts

@@ -22,12 +22,14 @@ export function convertToPerplexityMessages(
 
       case 'user':
       case 'assistant': {
-        const hasImage = content.some(
-          part => part.type === 'file' && part.mediaType.startsWith('image/'),
+        const hasMultipartContent = content.some(
+          part =>
+            (part.type === 'file' && part.mediaType.startsWith('image/')) ||
+            (part.type === 'file' && part.mediaType === 'application/pdf'),
         );
 
         const messageContent = content
-          .map(part => {
+          .map((part, index) => {
             switch (part.type) {
               case 'text': {
                 return {
@@ -36,30 +38,51 @@ export function convertToPerplexityMessages(
                 };
               }
               case 'file': {
-                return part.data instanceof URL
-                  ? {
-                      type: 'image_url',
-                      image_url: {
-                        url: part.data.toString(),
-                      },
-                    }
-                  : {
-                      type: 'image_url',
-                      image_url: {
-                        url: `data:${part.mediaType ?? 'image/jpeg'};base64,${
-                          typeof part.data === 'string'
-                            ? part.data
-                            : convertUint8ArrayToBase64(part.data)
-                        }`,
-                      },
-                    };
+                if (part.mediaType === 'application/pdf') {
+                  return part.data instanceof URL
+                    ? {
+                        type: 'file_url',
+                        file_url: {
+                          url: part.data.toString(),
+                        },
+                        file_name: part.filename,
+                      }
+                    : {
+                        type: 'file_url',
+                        file_url: {
+                          url:
+                            typeof part.data === 'string'
+                              ? part.data
+                              : convertUint8ArrayToBase64(part.data),
+                        },
+                        file_name: part.filename || `document-${index}.pdf`,
+                      };
+                } else if (part.mediaType.startsWith('image/')) {
+                  return part.data instanceof URL
+                    ? {
+                        type: 'image_url',
+                        image_url: {
+                          url: part.data.toString(),
+                        },
+                      }
+                    : {
+                        type: 'image_url',
+                        image_url: {
+                          url: `data:${part.mediaType ?? 'image/jpeg'};base64,${
+                            typeof part.data === 'string'
+                              ? part.data
+                              : convertUint8ArrayToBase64(part.data)
+                          }`,
+                        },
+                      };
+                }
               }
             }
           })
           .filter(Boolean) as PerplexityMessageContent[];
         messages.push({
           role,
-          content: hasImage
+          content: hasMultipartContent
             ? messageContent
             : messageContent
                 .filter(part => part.type === 'text')

packages/perplexity/src/perplexity-language-model-prompt.ts

@@ -15,4 +15,11 @@ export type PerplexityMessageContent =
       image_url: {
         url: string;
       };
+    }
+  | {
+      type: 'file_url';
+      file_url: {
+        url: string;
+      };
+      file_name?: string;
     };

packages/perplexity/src/perplexity-language-model.test.ts

@@ -149,6 +149,104 @@ describe('PerplexityLanguageModel', () => {
       });
     });
 
+    it('should handle PDF files with base64 encoding', async () => {
+      const mockPdfData = 'mock-pdf-data';
+      const prompt: LanguageModelV2Prompt = [
+        {
+          role: 'user',
+          content: [
+            { type: 'text', text: 'Analyze this PDF' },
+            {
+              type: 'file',
+              mediaType: 'application/pdf',
+              data: mockPdfData,
+              filename: 'test.pdf',
+            },
+          ],
+        },
+      ];
+
+      prepareJsonResponse({
+        content: 'This is an analysis of the PDF',
+      });
+
+      const result = await perplexityModel.doGenerate({ prompt });
+
+      // Verify the request contains the correct PDF format
+      const requestBody =
+        await jsonServer.calls[jsonServer.calls.length - 1].requestBodyJson;
+      expect(requestBody.messages[0].content).toEqual([
+        {
+          type: 'text',
+          text: 'Analyze this PDF',
+        },
+        {
+          type: 'file_url',
+          file_url: {
+            url: expect.stringContaining(mockPdfData),
+          },
+          file_name: 'test.pdf',
+        },
+      ]);
+
+      // Verify the response is processed correctly
+      expect(result.content).toEqual([
+        {
+          type: 'text',
+          text: 'This is an analysis of the PDF',
+        },
+      ]);
+    });
+
+    it('should handle PDF files with URLs', async () => {
+      const pdfUrl = 'https://example.com/test.pdf';
+      const prompt: LanguageModelV2Prompt = [
+        {
+          role: 'user',
+          content: [
+            { type: 'text', text: 'Analyze this PDF' },
+            {
+              type: 'file',
+              mediaType: 'application/pdf',
+              data: new URL(pdfUrl),
+              filename: 'test.pdf',
+            },
+          ],
+        },
+      ];
+
+      prepareJsonResponse({
+        content: 'This is an analysis of the PDF from URL',
+      });
+
+      const result = await perplexityModel.doGenerate({ prompt });
+
+      // Verify the request contains the correct PDF URL format
+      const requestBody =
+        await jsonServer.calls[jsonServer.calls.length - 1].requestBodyJson;
+      expect(requestBody.messages[0].content).toEqual([
+        {
+          type: 'text',
+          text: 'Analyze this PDF',
+        },
+        {
+          type: 'file_url',
+          file_url: {
+            url: pdfUrl,
+          },
+          file_name: 'test.pdf',
+        },
+      ]);
+
+      // Verify the response is processed correctly
+      expect(result.content).toEqual([
+        {
+          type: 'text',
+          text: 'This is an analysis of the PDF from URL',
+        },
+      ]);
+    });
+
     it('should extract citations as sources', async () => {
       prepareJsonResponse({
         citations: ['http://example.com/123', 'https://example.com/456'],