From 15b5015e55d7be9453264729b23d8436ecc23c1d Mon Sep 17 00:00:00 2001 From: Kai Date: Sat, 21 Feb 2026 09:48:49 -0800 Subject: [PATCH] =?UTF-8?q?feat:=20dual-mode=20root=20=E2=80=94=20serve=20?= =?UTF-8?q?markdown=20to=20agents,=20HTML=20to=20browsers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Content negotiation at / via middleware: - curl, wget, httpie, python-requests → markdown - Accept: text/markdown → markdown - ?format=md query param → markdown - Known AI agent UAs (GPTBot, OpenClaw, etc.) → markdown - Browsers (Accept: text/html) → HTML landing page New /api/index.md route serves rich markdown summary with: - Live stats (skills, MCP servers, agents count) - Full API endpoint table - Top skills and MCP servers - Registration/submission docs - Content negotiation docs Middleware moved from project root to src/middleware.ts (required for src/ directory structure in Next.js 16). Closes task-1771437023881-nc1um0a7a --- middleware.ts | 92 ------------------------ src/app/api/index.md/route.ts | 129 ++++++++++++++++++++++++++++++++++ src/middleware.ts | 120 +++++++++++++++++++++++++++++++ 3 files changed, 249 insertions(+), 92 deletions(-) delete mode 100644 middleware.ts create mode 100644 src/app/api/index.md/route.ts create mode 100644 src/middleware.ts diff --git a/middleware.ts b/middleware.ts deleted file mode 100644 index b574e8de..00000000 --- a/middleware.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { NextRequest, NextResponse } from "next/server"; - -/** - * Agent Detection Middleware - * - * Detects agent visitors via: - * 1. Accept: text/markdown header - * 2. Known agent User-Agent strings - * - * When an agent hits `/`, they get redirected to /llms.txt content - * instead of HTML. Other pages serve normally. - */ - -const AGENT_USER_AGENTS = [ - "GPTBot", - "ChatGPT-User", - "Claude-Web", - "Anthropic", - "CCBot", - "Bytespider", - "cohere-ai", - "PerplexityBot", - "YouBot", - "Google-Extended", - "Applebot-Extended", - "openclaw", - "OpenClaw", - "langchain", - "LangChain", - "autogpt", - "AutoGPT", - "BabyAGI", - "AgentGPT", - "CrewAI", - "crewai", - "phind", - "Phind", -]; - -function isAgentRequest(request: NextRequest): boolean { - // Check Accept header for markdown preference - const accept = request.headers.get("accept") || ""; - if ( - accept.includes("text/markdown") || - accept.includes("text/plain") && !accept.includes("text/html") - ) { - return true; - } - - // Check User-Agent - const ua = request.headers.get("user-agent") || ""; - for (const agent of AGENT_USER_AGENTS) { - if (ua.toLowerCase().includes(agent.toLowerCase())) { - return true; - } - } - - // Check custom header - if (request.headers.get("x-agent") === "true") { - return true; - } - - return false; -} - -export function middleware(request: NextRequest) { - const { pathname } = request.nextUrl; - - let response: NextResponse; - - // Only intercept the homepage for agents - if (pathname === "/" && isAgentRequest(request)) { - // Rewrite to the llms.txt route (serves plain text) - const url = request.nextUrl.clone(); - url.pathname = "/llms.txt"; - response = NextResponse.rewrite(url); - } else { - response = NextResponse.next(); - } - - // Add security headers to all responses - response.headers.set("X-Content-Type-Options", "nosniff"); - response.headers.set("X-Frame-Options", "DENY"); - response.headers.set("Referrer-Policy", "strict-origin-when-cross-origin"); - response.headers.set("X-XSS-Protection", "1; mode=block"); - - return response; -} - -export const config = { - matcher: ["/"], -}; diff --git a/src/app/api/index.md/route.ts b/src/app/api/index.md/route.ts new file mode 100644 index 00000000..a6a94049 --- /dev/null +++ b/src/app/api/index.md/route.ts @@ -0,0 +1,129 @@ +import { NextResponse } from "next/server"; +import { + getSkills, + getMcpServers, + getAgents, + getLlmsTxtEntries, + getAcpAgents, +} from "@/lib/data"; + +export const revalidate = 300; + +export async function GET() { + const skills = getSkills(); + const mcpServers = getMcpServers(); + const agents = getAgents(); + const llmsTxtEntries = getLlmsTxtEntries(); + const acpAgents = getAcpAgents(); + + const md = `# forAgents.dev + +> The MCP server registry for AI agents. Skills. Servers. Agents. Signal. + +Built by [Team Reflectt](https://reflectt.ai). Every endpoint available as markdown and JSON — no HTML parsing required. + +## Quick Stats + +- ${skills.length} Skills +- ${mcpServers.length} MCP Servers +- ${agents.length} Registered Agents +- ${acpAgents.length} ACP Agents +- ${llmsTxtEntries.length} llms.txt Sites + +## API Endpoints + +All endpoints support \`.md\` (markdown) and \`.json\` (structured data). + +| Resource | Markdown | JSON | +|----------|----------|------| +| News Feed | \`GET /api/feed.md\` | \`GET /api/feed.json\` | +| Skills | \`GET /api/skills.md\` | \`GET /api/skills.json\` | +| MCP Servers | \`GET /api/mcp.md\` | \`GET /api/mcp.json\` | +| Agents | \`GET /api/agents.md\` | \`GET /api/agents.json\` | +| Search | \`GET /api/search.md?q={query}\` | \`GET /api/search?q={query}\` | +| llms.txt Directory | \`GET /api/llms-directory.md\` | — | +| Submissions | \`GET /api/submissions?format=md\` | \`GET /api/submissions\` | + +## Featured: agent-team-kit + +Multi-agent coordination that actually works — clear roles, intake loops, and self-service queues. + +\`\`\`bash +curl -fsSL https://forAgents.dev/api/team-kit.sh | bash +\`\`\` + +- [GitHub](https://github.com/reflectt/agent-team-kit) +- [Docs](/skills/agent-team-kit) + +## Top Skills + +${skills + .slice(0, 6) + .map( + (s) => + `- **${s.name}** — ${s.description}\n Install: \`${s.install_cmd}\`\n Details: [/skills/${s.slug}](/skills/${s.slug})` + ) + .join("\n")} + +## Top MCP Servers + +${mcpServers + .slice(0, 6) + .map( + (s) => + `- **${s.name}** (${s.category}) — ${s.description}\n Install: \`${s.install_cmd}\`\n [GitHub](${s.github})` + ) + .join("\n")} + +## Guides + +- [Getting Started](/api/getting-started.md) +- [Kit Integration Guide](/api/guides/integration.md) — Memory, Autonomy, and Team kits +- [How to Submit](/api/how-to-submit.md) + +## Registration & Submission + +Register your agent: + +\`\`\` +POST /api/register +{ "name": "...", "platform": "...", "ownerUrl": "..." } +\`\`\` + +Submit skills, MCP servers, or agents: + +\`\`\` +POST /api/submit +{ "type": "skill|mcp|agent", "name": "...", "description": "...", "url": "...", "author": "...", "tags": [...] } +\`\`\` + +## Machine-Readable Discovery + +- \`GET /llms.txt\` — Full site map for LLMs +- \`GET /.well-known/agent.json\` — Agent card (A2A compatible) + +## Content Negotiation + +This root URL (\`/\`) serves **markdown** to agents and **HTML** to browsers. + +Detection rules: +1. \`?format=md\` query parameter +2. \`Accept: text/markdown\` header +3. \`Accept: text/plain\` (without \`text/html\`) +4. Known agent/CLI User-Agent (curl, wget, python-requests, etc.) + +To force HTML: request with \`Accept: text/html\`. +To force markdown: add \`?format=md\` or set \`Accept: text/markdown\`. + +--- + +Source: https://forAgents.dev | GitHub: https://github.com/reflectt | Contact: https://reflectt.ai +`; + + return new NextResponse(md, { + headers: { + "Content-Type": "text/markdown; charset=utf-8", + "Cache-Control": "public, max-age=300, stale-while-revalidate=600", + }, + }); +} diff --git a/src/middleware.ts b/src/middleware.ts new file mode 100644 index 00000000..a3e9a024 --- /dev/null +++ b/src/middleware.ts @@ -0,0 +1,120 @@ +import { NextRequest, NextResponse } from "next/server"; + +/** + * Content Negotiation Middleware for forAgents.dev + * + * Detects agent/CLI requests and serves markdown instead of HTML at `/`. + * + * Detection rules (any match → markdown): + * 1. Query param: ?format=md + * 2. Accept header includes text/markdown + * 3. Accept header includes text/plain but NOT text/html + * 4. Custom header: x-agent: true + * 5. Known AI agent User-Agent strings (GPTBot, Claude, OpenClaw, etc.) + * 6. Known CLI User-Agent strings (curl, wget, httpie, etc.) + * combined with no explicit text/html in Accept + * + * Agents get /api/index.md (rich markdown summary with API docs + stats). + * Browsers get the HTML landing page. + */ + +const AGENT_USER_AGENTS = [ + "GPTBot", + "ChatGPT-User", + "Claude-Web", + "Anthropic", + "CCBot", + "Bytespider", + "cohere-ai", + "PerplexityBot", + "YouBot", + "Google-Extended", + "Applebot-Extended", + "openclaw", + "OpenClaw", + "langchain", + "LangChain", + "autogpt", + "AutoGPT", + "BabyAGI", + "AgentGPT", + "CrewAI", + "crewai", + "phind", + "Phind", +]; + +/** + * CLI tools that default to Accept: *\/* and are almost certainly + * non-browser programmatic requests. + */ +const CLI_UA_PATTERN = + /\b(curl|wget|httpie|python-requests|node-fetch|got\/|axios\/|undici|httpx|aiohttp|Go-http-client|Ruby|Faraday|libcurl|okhttp)\b/i; + +function isAgentRequest(request: NextRequest): boolean { + const accept = request.headers.get("accept") || ""; + const ua = request.headers.get("user-agent") || ""; + + // 1. Explicit query param — always wins + if (request.nextUrl.searchParams.get("format") === "md") { + return true; + } + + // 2. Accept header prefers markdown + if (accept.includes("text/markdown")) { + return true; + } + + // 3. Accept header prefers plain text but not HTML (typical CLI default) + if (accept.includes("text/plain") && !accept.includes("text/html")) { + return true; + } + + // 4. Custom header + if (request.headers.get("x-agent") === "true") { + return true; + } + + // 5. Known AI agent UA — always serve markdown + const uaLower = ua.toLowerCase(); + for (const agent of AGENT_USER_AGENTS) { + if (uaLower.includes(agent.toLowerCase())) { + return true; + } + } + + // 6. CLI tool UA when Accept doesn't explicitly request text/html + // (curl sends Accept: */* by default — not a browser) + if (!accept.includes("text/html") && CLI_UA_PATTERN.test(ua)) { + return true; + } + + return false; +} + +export function middleware(request: NextRequest) { + const { pathname } = request.nextUrl; + + let response: NextResponse; + + // Only intercept the homepage for agents + if (pathname === "/" && isAgentRequest(request)) { + const url = request.nextUrl.clone(); + url.pathname = "/api/index.md"; + response = NextResponse.rewrite(url); + } else { + response = NextResponse.next(); + } + + // Add security headers to all responses + response.headers.set("X-Content-Type-Options", "nosniff"); + response.headers.set("X-Frame-Options", "DENY"); + response.headers.set("Referrer-Policy", "strict-origin-when-cross-origin"); + response.headers.set("X-XSS-Protection", "1; mode=block"); + + return response; +} + +export const config = { + matcher: ["/"], +};