feat: dual-mode root — markdown for agents, HTML for browsers

src/app/api/index.md/route.ts

@@ -0,0 +1,129 @@
+import { NextResponse } from "next/server";
+import {
+  getSkills,
+  getMcpServers,
+  getAgents,
+  getLlmsTxtEntries,
+  getAcpAgents,
+} from "@/lib/data";
+
+export const revalidate = 300;
+
+export async function GET() {
+  const skills = getSkills();
+  const mcpServers = getMcpServers();
+  const agents = getAgents();
+  const llmsTxtEntries = getLlmsTxtEntries();
+  const acpAgents = getAcpAgents();
+
+  const md = `# forAgents.dev
+
+> The MCP server registry for AI agents. Skills. Servers. Agents. Signal.
+
+Built by [Team Reflectt](https://reflectt.ai). Every endpoint available as markdown and JSON — no HTML parsing required.
+
+## Quick Stats
+
+- ${skills.length} Skills
+- ${mcpServers.length} MCP Servers
+- ${agents.length} Registered Agents
+- ${acpAgents.length} ACP Agents
+- ${llmsTxtEntries.length} llms.txt Sites
+
+## API Endpoints
+
+All endpoints support \`.md\` (markdown) and \`.json\` (structured data).
+
+| Resource | Markdown | JSON |
+|----------|----------|------|
+| News Feed | \`GET /api/feed.md\` | \`GET /api/feed.json\` |
+| Skills | \`GET /api/skills.md\` | \`GET /api/skills.json\` |
+| MCP Servers | \`GET /api/mcp.md\` | \`GET /api/mcp.json\` |
+| Agents | \`GET /api/agents.md\` | \`GET /api/agents.json\` |
+| Search | \`GET /api/search.md?q={query}\` | \`GET /api/search?q={query}\` |
+| llms.txt Directory | \`GET /api/llms-directory.md\` | — |
+| Submissions | \`GET /api/submissions?format=md\` | \`GET /api/submissions\` |
+
+## Featured: agent-team-kit
+
+Multi-agent coordination that actually works — clear roles, intake loops, and self-service queues.
+
+\`\`\`bash
+curl -fsSL https://forAgents.dev/api/team-kit.sh | bash
+\`\`\`
+
+- [GitHub](https://github.com/reflectt/agent-team-kit)
+- [Docs](/skills/agent-team-kit)
+
+## Top Skills
+
+${skills
+  .slice(0, 6)
+  .map(
+    (s) =>
+      `- **${s.name}** — ${s.description}\n  Install: \`${s.install_cmd}\`\n  Details: [/skills/${s.slug}](/skills/${s.slug})`
+  )
+  .join("\n")}
+
+## Top MCP Servers
+
+${mcpServers
+  .slice(0, 6)
+  .map(
+    (s) =>
+      `- **${s.name}** (${s.category}) — ${s.description}\n  Install: \`${s.install_cmd}\`\n  [GitHub](${s.github})`
+  )
+  .join("\n")}
+
+## Guides
+
+- [Getting Started](/api/getting-started.md)
+- [Kit Integration Guide](/api/guides/integration.md) — Memory, Autonomy, and Team kits
+- [How to Submit](/api/how-to-submit.md)
+
+## Registration & Submission
+
+Register your agent:
+
+\`\`\`
+POST /api/register
+{ "name": "...", "platform": "...", "ownerUrl": "..." }
+\`\`\`
+
+Submit skills, MCP servers, or agents:
+
+\`\`\`
+POST /api/submit
+{ "type": "skill|mcp|agent", "name": "...", "description": "...", "url": "...", "author": "...", "tags": [...] }
+\`\`\`
+
+## Machine-Readable Discovery
+
+- \`GET /llms.txt\` — Full site map for LLMs
+- \`GET /.well-known/agent.json\` — Agent card (A2A compatible)
+
+## Content Negotiation
+
+This root URL (\`/\`) serves **markdown** to agents and **HTML** to browsers.
+
+Detection rules:
+1. \`?format=md\` query parameter
+2. \`Accept: text/markdown\` header
+3. \`Accept: text/plain\` (without \`text/html\`)
+4. Known agent/CLI User-Agent (curl, wget, python-requests, etc.)
+
+To force HTML: request with \`Accept: text/html\`.
+To force markdown: add \`?format=md\` or set \`Accept: text/markdown\`.
+
+---
+
+Source: https://forAgents.dev | GitHub: https://github.com/reflectt | Contact: https://reflectt.ai
+`;
+
+  return new NextResponse(md, {
+    headers: {
+      "Content-Type": "text/markdown; charset=utf-8",
+      "Cache-Control": "public, max-age=300, stale-while-revalidate=600",
+    },
+  });
+}

src/middleware.ts

@@ -0,0 +1,120 @@
+import { NextRequest, NextResponse } from "next/server";
+
+/**
+ * Content Negotiation Middleware for forAgents.dev
+ *
+ * Detects agent/CLI requests and serves markdown instead of HTML at `/`.
+ *
+ * Detection rules (any match → markdown):
+ *   1. Query param: ?format=md
+ *   2. Accept header includes text/markdown
+ *   3. Accept header includes text/plain but NOT text/html
+ *   4. Custom header: x-agent: true
+ *   5. Known AI agent User-Agent strings (GPTBot, Claude, OpenClaw, etc.)
+ *   6. Known CLI User-Agent strings (curl, wget, httpie, etc.)
+ *      combined with no explicit text/html in Accept
+ *
+ * Agents get /api/index.md (rich markdown summary with API docs + stats).
+ * Browsers get the HTML landing page.
+ */
+
+const AGENT_USER_AGENTS = [
+  "GPTBot",
+  "ChatGPT-User",
+  "Claude-Web",
+  "Anthropic",
+  "CCBot",
+  "Bytespider",
+  "cohere-ai",
+  "PerplexityBot",
+  "YouBot",
+  "Google-Extended",
+  "Applebot-Extended",
+  "openclaw",
+  "OpenClaw",
+  "langchain",
+  "LangChain",
+  "autogpt",
+  "AutoGPT",
+  "BabyAGI",
+  "AgentGPT",
+  "CrewAI",
+  "crewai",
+  "phind",
+  "Phind",
+];
+
+/**
+ * CLI tools that default to Accept: *\/* and are almost certainly
+ * non-browser programmatic requests.
+ */
+const CLI_UA_PATTERN =
+  /\b(curl|wget|httpie|python-requests|node-fetch|got\/|axios\/|undici|httpx|aiohttp|Go-http-client|Ruby|Faraday|libcurl|okhttp)\b/i;
+
+function isAgentRequest(request: NextRequest): boolean {
+  const accept = request.headers.get("accept") || "";
+  const ua = request.headers.get("user-agent") || "";
+
+  // 1. Explicit query param — always wins
+  if (request.nextUrl.searchParams.get("format") === "md") {
+    return true;
+  }
+
+  // 2. Accept header prefers markdown
+  if (accept.includes("text/markdown")) {
+    return true;
+  }
+
+  // 3. Accept header prefers plain text but not HTML (typical CLI default)
+  if (accept.includes("text/plain") && !accept.includes("text/html")) {
+    return true;
+  }
+
+  // 4. Custom header
+  if (request.headers.get("x-agent") === "true") {
+    return true;
+  }
+
+  // 5. Known AI agent UA — always serve markdown
+  const uaLower = ua.toLowerCase();
+  for (const agent of AGENT_USER_AGENTS) {
+    if (uaLower.includes(agent.toLowerCase())) {
+      return true;
+    }
+  }
+
+  // 6. CLI tool UA when Accept doesn't explicitly request text/html
+  //    (curl sends Accept: */* by default — not a browser)
+  if (!accept.includes("text/html") && CLI_UA_PATTERN.test(ua)) {
+    return true;
+  }
+
+  return false;
+}
+
+export function middleware(request: NextRequest) {
+  const { pathname } = request.nextUrl;
+
+  let response: NextResponse;
+
+  // Only intercept the homepage for agents
+  if (pathname === "/" && isAgentRequest(request)) {
+    const url = request.nextUrl.clone();
+    url.pathname = "/api/index.md";
+    response = NextResponse.rewrite(url);
+  } else {
+    response = NextResponse.next();
+  }
+
+  // Add security headers to all responses
+  response.headers.set("X-Content-Type-Options", "nosniff");
+  response.headers.set("X-Frame-Options", "DENY");
+  response.headers.set("Referrer-Policy", "strict-origin-when-cross-origin");
+  response.headers.set("X-XSS-Protection", "1; mode=block");
+
+  return response;
+}
+
+export const config = {
+  matcher: ["/"],
+};