feat: enhance content extraction settings with scraper selection

Shishir435 · Shishir435 · commit d5c5c15cee49 · 2025-11-04T23:00:07.000+05:30
- Introduce content scraper options (auto, defuddle, readability) for improved article parsing.
- Update content extraction logic to respect user preferences for scraper selection.
- Ensure backward compatibility by defaulting to "auto" if no scraper is set.
diff --git a/src/contents/index.ts b/src/contents/index.ts
@@ -312,41 +312,63 @@ browser.runtime.onMessage.addListener(
             }
           }
 
-          // Parse with defuddle first (better GitHub support, more tokens)
-          console.log("[Content Script] Parsing article with defuddle...")
+          // Parse content based on user's scraper preference
+          const scraper = effectiveConfig.contentScraper || "auto"
+          console.log(`[Content Script] Using scraper: ${scraper}`)
+
           let readableText = ""
           let pageTitle = ""
           let defuddleResult: ReturnType<Defuddle["parse"]> | null = null
 
-          try {
-            const defuddle = new Defuddle(document, {
-              markdown: true,
-              separateMarkdown: false,
-              removeExactSelectors: true // Remove ads and social buttons
-            })
-            defuddleResult = defuddle.parse()
+          // Try Defuddle if user selected "auto" or "defuddle"
+          if (scraper === "auto" || scraper === "defuddle") {
+            console.log("[Content Script] Parsing article with Defuddle...")
+            try {
+              const defuddle = new Defuddle(document, {
+                markdown: true,
+                separateMarkdown: false,
+                removeExactSelectors: true // Remove ads and social buttons
+              })
+              defuddleResult = defuddle.parse()
 
-            // Prefer markdown if available, otherwise use HTML content
-            readableText =
-              defuddleResult?.contentMarkdown || defuddleResult?.content || ""
-            readableText = normalizeWhitespaceForLLM(readableText)
-            pageTitle = defuddleResult?.title || ""
+              // Prefer markdown if available, otherwise use HTML content
+              readableText =
+                defuddleResult?.contentMarkdown || defuddleResult?.content || ""
+              readableText = normalizeWhitespaceForLLM(readableText)
+              pageTitle = defuddleResult?.title || ""
 
-            console.log(
-              `[Content Script] defuddle extracted ${readableText.length} chars (${defuddleResult?.contentMarkdown ? "markdown" : "HTML"})`
-            )
-          } catch (error) {
-            console.warn(
-              "[Content Script] defuddle failed, falling back to Readability:",
-              error
-            )
+              console.log(
+                `[Content Script] Defuddle extracted ${readableText.length} chars (${defuddleResult?.contentMarkdown ? "markdown" : "HTML"})`
+              )
+            } catch (error) {
+              console.warn("[Content Script] Defuddle failed:", error)
+              // If user selected "defuddle" only, don't fallback
+              if (scraper === "defuddle") {
+                console.error(
+                  "[Content Script] Defuddle-only mode failed, no fallback available"
+                )
+              }
+            }
           }
 
-          // Fallback to Readability if defuddle failed or returned minimal content
-          if (!readableText || readableText.trim().length < 100) {
-            console.log(
-              "[Content Script] defuddle returned minimal content, trying Readability..."
-            )
+          // Try Readability if:
+          // 1. User selected "readability" only
+          // 2. User selected "auto" and defuddle failed or returned minimal content
+          if (
+            scraper === "readability" ||
+            (scraper === "auto" &&
+              (!readableText || readableText.trim().length < 100))
+          ) {
+            if (scraper === "auto") {
+              console.log(
+                "[Content Script] Defuddle returned minimal content, trying Readability..."
+              )
+            } else {
+              console.log(
+                "[Content Script] Parsing article with Readability..."
+              )
+            }
+
             try {
               const article = new Readability(
                 document.cloneNode(true) as Document
@@ -356,8 +378,11 @@ browser.runtime.onMessage.addListener(
               const normalizedReadability =
                 normalizeWhitespaceForLLM(readabilityText)
 
-              // Use Readability result if it's better than defuddle
+              // Use Readability result if:
+              // - User selected "readability" only, OR
+              // - Auto mode and Readability is better than defuddle
               if (
+                scraper === "readability" ||
                 normalizedReadability.length > readableText.length ||
                 readableText.trim().length < 50
               ) {
@@ -367,15 +392,12 @@ browser.runtime.onMessage.addListener(
                 )
               }
 
-              // Use Readability title if defuddle didn't provide one
+              // Use Readability title if not already set
               if (!pageTitle && article?.title) {
                 pageTitle = article.title
               }
             } catch (error) {
-              console.error(
-                "[Content Script] Readability fallback failed:",
-                error
-              )
+              console.error("[Content Script] Readability failed:", error)
             }
           }
 
diff --git a/src/features/model/components/content-extraction-settings.tsx b/src/features/model/components/content-extraction-settings.tsx
@@ -30,6 +30,13 @@ const useContentExtractionConfig = () => {
     [setConfig]
   )
 
+  // Ensure contentScraper is set (for backward compatibility)
+  useEffect(() => {
+    if (!config.contentScraper) {
+      updateConfig({ contentScraper: "auto" })
+    }
+  }, [config.contentScraper, updateConfig])
+
   return [config, updateConfig] as const
 }
 
diff --git a/src/features/model/components/global-settings.tsx b/src/features/model/components/global-settings.tsx
@@ -19,11 +19,24 @@ import { Separator } from "@/components/ui/separator"
 import { Slider } from "@/components/ui/slider"
 import { Switch } from "@/components/ui/switch"
 import {
+  CONTENT_SCRAPER_OPTIONS,
   SCROLL_STRATEGY_DESCRIPTIONS,
   SCROLL_STRATEGY_OPTIONS
 } from "@/lib/constants-ui"
-import { Settings, Target, Zap } from "@/lib/lucide-icon"
-import type { ContentExtractionConfig, ScrollStrategy } from "@/types"
+import {
+  BookOpen,
+  Code,
+  FileText,
+  Settings,
+  Sparkles,
+  Target,
+  Zap
+} from "@/lib/lucide-icon"
+import type {
+  ContentExtractionConfig,
+  ContentScraper,
+  ScrollStrategy
+} from "@/types"
 import { TIMEOUT_FIELDS } from "./content-extraction-constants"
 
 interface GlobalSettingsProps {
@@ -60,6 +73,85 @@ export const GlobalSettings = ({ config, onUpdate }: GlobalSettingsProps) => {
     </div>
   )
 
+  // Get icon for scraper type
+  const getScraperIcon = (scraper: ContentScraper) => {
+    switch (scraper) {
+      case "auto":
+        return Sparkles
+      case "defuddle":
+        return Code
+      case "readability":
+        return BookOpen
+    }
+  }
+
+  // Render content scraper selection
+  const renderContentScraperSelect = (
+    value: ContentScraper,
+    onValueChange: (value: ContentScraper) => void
+  ) => (
+    <div className="space-y-3">
+      <Label className="flex items-center gap-2 text-sm font-medium">
+        <FileText className="h-4 w-4" />
+        Content Scraper
+      </Label>
+      <div className="grid gap-3">
+        {CONTENT_SCRAPER_OPTIONS.map((option) => {
+          const Icon = getScraperIcon(option.value)
+          const isSelected = value === option.value
+          return (
+            <button
+              key={option.value}
+              type="button"
+              onClick={() => onValueChange(option.value)}
+              className={`
+                group relative flex items-start gap-3 rounded-lg border p-4 text-left transition-all
+                hover:bg-accent/50 hover:border-accent-foreground/20
+                ${
+                  isSelected
+                    ? "border-primary bg-accent/30 ring-1 ring-primary shadow-sm"
+                    : "border-border"
+                }
+              `}>
+              <div
+                className={`
+                  flex h-10 w-10 shrink-0 items-center justify-center rounded-md transition-colors
+                  ${isSelected ? "bg-primary text-primary-foreground" : "bg-muted group-hover:bg-muted/80"}
+                `}>
+                <Icon className="h-5 w-5" />
+              </div>
+              <div className="flex-1 space-y-1.5 min-w-0">
+                <div className="flex items-center gap-2 flex-wrap">
+                  <span className="font-semibold text-sm">{option.label}</span>
+                  {option.recommended && (
+                    <Badge
+                      variant="default"
+                      className="text-[10px] h-5 px-1.5 font-medium">
+                      Recommended
+                    </Badge>
+                  )}
+                  {isSelected && (
+                    <Badge
+                      variant="secondary"
+                      className="text-[10px] h-5 px-1.5">
+                      Active
+                    </Badge>
+                  )}
+                </div>
+                <p className="text-xs font-medium text-foreground/80">
+                  {option.description}
+                </p>
+                <p className="text-xs text-muted-foreground leading-relaxed">
+                  {option.detail}
+                </p>
+              </div>
+            </button>
+          )
+        })}
+      </div>
+    </div>
+  )
+
   // Render scroll strategy select
   const renderScrollStrategySelect = (
     value: ScrollStrategy,
@@ -162,6 +254,13 @@ export const GlobalSettings = ({ config, onUpdate }: GlobalSettingsProps) => {
 
         <Separator />
 
+        {/* Content Scraper Selection */}
+        {renderContentScraperSelect(config.contentScraper, (value) =>
+          onUpdate({ contentScraper: value })
+        )}
+
+        <Separator />
+
         {/* Scroll Strategy */}
         {renderScrollStrategySelect(config.scrollStrategy, (value) =>
           onUpdate({ scrollStrategy: value })
diff --git a/src/lib/constants-ui.ts b/src/lib/constants-ui.ts
@@ -10,7 +10,7 @@ import {
   type LucideIcon,
   Twitter
 } from "@/lib/lucide-icon"
-import type { ScrollStrategy } from "@/types"
+import type { ContentScraper, ScrollStrategy } from "@/types"
 
 export interface SocialLink {
   label: string
@@ -79,6 +79,34 @@ export const SCROLL_STRATEGY_DESCRIPTIONS: Record<ScrollStrategy, string> = {
     "Intelligent scrolling with content detection, best for dynamic content"
 }
 
+// Content scraper options
+export const CONTENT_SCRAPER_OPTIONS = [
+  {
+    value: "auto" as ContentScraper,
+    label: "Auto",
+    description: "Smart fallback: Defuddle → Readability",
+    detail:
+      "Best for most websites. Tries Defuddle first for better markdown and code formatting, falls back to Readability if needed.",
+    recommended: true
+  },
+  {
+    value: "defuddle" as ContentScraper,
+    label: "Defuddle",
+    description: "Technical & code-heavy content",
+    detail:
+      "Best for GitHub, documentation, Stack Overflow, and developer blogs. Preserves code blocks and markdown structure.",
+    recommended: false
+  },
+  {
+    value: "readability" as ContentScraper,
+    label: "Readability",
+    description: "Articles & blog posts",
+    detail:
+      "Best for news articles, Medium posts, and traditional blog content. Mozilla's proven algorithm for clean text extraction.",
+    recommended: false
+  }
+]
+
 export const GUIDES = [
   {
     label: "Ollama Client Setup Guide",
diff --git a/src/lib/constants.ts b/src/lib/constants.ts
@@ -164,6 +164,7 @@ export const DEFAULT_MODEL_CONFIG: ModelConfig = {
 
 export const DEFAULT_CONTENT_EXTRACTION_CONFIG: ContentExtractionConfig = {
   enabled: true,
+  contentScraper: "auto", // Try defuddle first, then readability
   excludedUrlPatterns: DEFAULT_EXCLUDE_URLS,
   scrollStrategy: "smart",
   scrollDepth: 0.8, // 80% of page
diff --git a/src/lib/lucide-icon.ts b/src/lib/lucide-icon.ts
@@ -15,6 +15,7 @@ import ChevronsUpDown from "lucide-react/dist/esm/icons/chevrons-up-down"
 import Circle from "lucide-react/dist/esm/icons/circle"
 import CircleCheck from "lucide-react/dist/esm/icons/circle-check"
 import Clock from "lucide-react/dist/esm/icons/clock"
+import Code from "lucide-react/dist/esm/icons/code"
 import Copy from "lucide-react/dist/esm/icons/copy"
 import CopyCheck from "lucide-react/dist/esm/icons/copy-check"
 import Cpu from "lucide-react/dist/esm/icons/cpu"
@@ -134,6 +135,7 @@ export {
   Sun,
   Copy,
   Check,
+  Code,
   ChevronDown,
   RefreshCcw,
   Circle,
diff --git a/src/options/components/ollama-options.tsx b/src/options/components/ollama-options.tsx
@@ -36,7 +36,7 @@ export const OllamaOptions = () => {
       content: <ModelPullPanel />
     },
     contentExtraction: {
-      label: "Content Extraction",
+      label: "Extraction",
       content: <ContentExtractionSettings />
     },
     voices: {
diff --git a/src/types/index.ts b/src/types/index.ts
@@ -317,9 +317,11 @@ export interface ChatInput {
 }
 
 export type ScrollStrategy = "none" | "gradual" | "instant" | "smart"
+export type ContentScraper = "auto" | "defuddle" | "readability"
 
 export interface ContentExtractionConfig {
   enabled: boolean
+  contentScraper: ContentScraper // Which scraper to use: auto (try defuddle then readability), defuddle, or readability
   excludedUrlPatterns: string[] // URL patterns to exclude from extraction
   scrollStrategy: ScrollStrategy
   scrollDepth: number // 0-1 (percentage of page)

Original file line number	Diff line number	Diff line change
`@@ -30,6 +30,13 @@ const useContentExtractionConfig = () => {`
`30`	`30`	`[setConfig]`
`31`	`31`	`)`
`32`	`32`
	`33`	`+ // Ensure contentScraper is set (for backward compatibility)`
	`34`	`+ useEffect(() => {`
	`35`	`+ if (!config.contentScraper) {`
	`36`	`+ updateConfig({ contentScraper: "auto" })`
	`37`	`+ }`
	`38`	`+ }, [config.contentScraper, updateConfig])`
	`39`	`+`
`33`	`40`	`return [config, updateConfig] as const`
`34`	`41`	`}`
`35`	`42`