BC5 readiness: TypeScript live wire-capture canary

CONTRIBUTING.md

@@ -247,6 +247,54 @@ beyond the language-specific drift checks. These run as part of `make check`:
   endpoint or append a one-line entry to the allowlist with a justification
   comment.
 
+## Live canary
+
+The TypeScript runner also drives a *live canary* against a real Basecamp
+backend. It dispatches every operation in
+[`conformance/tests/live-my-surface.json`](conformance/tests/live-my-surface.json)
+through the SDK's typed surface, captures the raw wire response (bytes +
+headers), and validates each response body against the OpenAPI response
+schema. Forward-compat additions on the wire surface as "extras observed"
+in the run summary — never as failures — so new BC5 fields don't break
+the canary while still being visible.
+
+The canary is **opt-in** and **does not run as part of `make check`**:
+
+```bash
+BASECAMP_LIVE=1 \
+BASECAMP_TOKEN=<your-token> \
+BASECAMP_ACCOUNT_ID=<your-account> \
+make conformance-typescript-live
+```
+
+Optional env:
+
+- `BASECAMP_HOST` — backend **origin** only (e.g. `https://3.basecampapi.com`);
+  the runner appends `/{accountId}` to mirror `createBasecampClient`'s
+  default URL composition.
+- `BASECAMP_BACKEND=bc4|bc5` — namespaces persisted snapshots so BC4 and
+  BC5 runs don't collide.
+- `LIVE_RECORD_DIR=<path>` — persists wire snapshots to
+  `<path>/<backend>/wire/<test>.json`. Used by downstream cross-language
+  decoders (PR 3) and BC4↔BC5 comparison (PR 4).
+- `BASECAMP_BC4_PROJECT_ID` / `BASECAMP_BC5_PROJECT_ID` /
+  `BASECAMP_PROJECT_ID` etc. — explicit fixture-IDs override the runner's
+  discovery walk. Same pattern applies for `TODOSET_ID`, `TODOLIST_ID`,
+  `TODO_ID`.
+
+Tests skip with a clear `skipReason` when a fixture-ID can't be resolved
+(no env override, no discovery match) — they don't fail.
+
+Adding an operation to the live canary requires both a fixture entry in
+`live-my-surface.json` and a dispatch case in
+`conformance/runner/typescript/live-dispatch.ts`. The runner's startup
+gate refuses to run if any fixture operation lacks a dispatch.
+
+Because live canary fixtures live in the shared `conformance/tests/` directory,
+offline conformance runners must treat `mode` as part of the shared schema and
+execute only mock tests: omitted `mode` or `mode: "mock"`. `mode: "live"` entries
+belong to the TypeScript live wire-capture runner until replay runners are added.
+
 ## API gap registry (`spec/api-gaps/`)
 
 When BC ships a new user-visible feature without a JSON API (or with an

Makefile

@@ -370,7 +370,7 @@ py-clean:
 # Conformance Test targets
 #------------------------------------------------------------------------------
 
-.PHONY: conformance conformance-go conformance-kotlin conformance-typescript conformance-ruby conformance-python conformance-build
+.PHONY: conformance conformance-go conformance-kotlin conformance-typescript conformance-typescript-live conformance-ruby conformance-python conformance-build
 
 # Build conformance test runner
 conformance-build:
@@ -392,6 +392,17 @@ conformance-typescript:
 	@echo "==> Running TypeScript conformance tests..."
 	cd conformance/runner/typescript && npm ci && npm test
 
+# Run TypeScript live canary against a real Basecamp backend.
+#
+# Required env: BASECAMP_LIVE=1, BASECAMP_TOKEN, BASECAMP_ACCOUNT_ID.
+# Optional env: BASECAMP_HOST (origin only, e.g. https://3.basecampapi.com —
+# runner appends /{accountId}); BASECAMP_BACKEND=bc4|bc5 to namespace
+# snapshots; LIVE_RECORD_DIR to persist wire snapshots for downstream
+# replay/compare. Opt-in: not invoked by `make check`.
+conformance-typescript-live:
+	@echo "==> Running TypeScript live canary..."
+	cd conformance/runner/typescript && npm ci && BASECAMP_LIVE=1 npm test
+
 # Run Ruby conformance tests
 conformance-ruby:
 	@echo "==> Running Ruby conformance tests..."

conformance/runner/go/main.go

@@ -28,6 +28,11 @@ import (
 
 // TestCase represents a single conformance test.
 type TestCase struct {
+	// Mode is "mock" (default) or "live". Live tests are owned by the TS
+	// runner; non-TS runners filter them out at load time so unresolved
+	// fixture placeholders and unknown operations don't false-pass as
+	// mock conformance.
+	Mode            string                 `json:"mode"`
 	Name            string                 `json:"name"`
 	Description     string                 `json:"description"`
 	Operation       string                 `json:"operation"`
@@ -173,7 +178,16 @@ func loadTests(filename string) ([]TestCase, error) {
 		return nil, err
 	}
 
-	return tests, nil
+	// Live tests are TS-only — filter them out so this runner doesn't
+	// attempt mock dispatch on entries with unresolved ${PROJECT_ID}
+	// fixtures or operations that only the live runner knows about.
+	mockTests := tests[:0]
+	for _, tc := range tests {
+		if tc.Mode == "" || tc.Mode == "mock" {
+			mockTests = append(mockTests, tc)
+		}
+	}
+	return mockTests, nil
 }
 
 // Default account ID for conformance tests

conformance/runner/python/runner.py

@@ -483,8 +483,15 @@ def run(self) -> int:
         skipped = 0
 
         for file in files:
-            print(f"\n=== {file.name} ===")
             tests = json.loads(file.read_text())
+            # Live tests are TS-only (canonical wire-capturer); filter them out
+            # before mock dispatch so unresolved ${PROJECT_ID} fixtures and
+            # live-only operations don't surface here.
+            tests = [t for t in tests if t.get("mode", "mock") == "mock"]
+            if not tests:
+                continue
+
+            print(f"\n=== {file.name} ===")
 
             for test_case in tests:
                 name = test_case["name"]

conformance/runner/ruby/runner.rb

@@ -614,9 +614,16 @@ def run
     results = []
 
     files.each do |file|
+      tests = JSON.parse(File.read(file))
+      # Live tests are TS-only (canonical wire-capturer); accept only mock
+      # so unresolved ${PROJECT_ID} fixtures and live-only operations don't
+      # surface as mock failures or false passes — and any future mode added
+      # to the schema enum stays opt-in for this runner.
+      tests = tests.select { |t| (t["mode"] || "mock") == "mock" }
+      next if tests.empty?
+
       puts "\n=== #{File.basename(file)} ==="
 
-      tests = JSON.parse(File.read(file))
       tests.each do |test_case|
         if RUBY_SKIPS.include?(test_case["name"])
           skipped += 1

conformance/runner/typescript/fixtures.ts

@@ -0,0 +1,111 @@
+/**
+ * Fixture-ID resolution for live canary tests.
+ *
+ * Resolution ladder (per §5d of the BC5-readiness plan):
+ *   1. Explicit per-backend env var, e.g. BASECAMP_BC4_PROJECT_ID
+ *   2. Generic env var, e.g. BASECAMP_PROJECT_ID
+ *   3. Discovery via the SDK (ListProjects → first project; etc.)
+ *   4. Fall through: undefined; caller skips with skipReason.
+ *
+ * Resolution is cached per-backend so discovery only fires once per run.
+ */
+
+import type { BasecampClient } from "@37signals/basecamp";
+
+export type Backend = "bc4" | "bc5" | "unknown";
+
+export interface FixtureContext {
+  client: BasecampClient;
+  backend: Backend;
+}
+
+const cache = new Map<string, string | null>();
+
+function cacheKey(backend: Backend, name: string): string {
+  return `${backend}:${name}`;
+}
+
+function fromEnv(backend: Backend, name: string): string | undefined {
+  const upper = name.toUpperCase();
+  if (backend !== "unknown") {
+    const explicit = process.env[`BASECAMP_${backend.toUpperCase()}_${upper}`];
+    if (explicit) return explicit;
+  }
+  const generic = process.env[`BASECAMP_${upper}`];
+  if (generic) return generic;
+  return undefined;
+}
+
+/**
+ * Resolve a fixture-ID by name. Returns the resolved string or undefined if
+ * not resolvable; caller is responsible for the skip-with-reason path.
+ *
+ * Discovery walks:
+ *   PROJECT_ID    → ListProjects → first project
+ *   TODOSET_ID    → walk dock of resolved project, pick first todoset tool
+ *   TODOLIST_ID   → ListTodolists for resolved todoset, pick first
+ *   TODO_ID       → ListTodos for resolved todolist, pick first
+ */
+export async function resolveFixtureId(
+  ctx: FixtureContext,
+  name: string,
+): Promise<string | undefined> {
+  const key = cacheKey(ctx.backend, name);
+  if (cache.has(key)) {
+    const cached = cache.get(key);
+    return cached ?? undefined;
+  }
+
+  const env = fromEnv(ctx.backend, name);
+  if (env) {
+    cache.set(key, env);
+    return env;
+  }
+
+  let resolved: string | undefined;
+  try {
+    switch (name) {
+      case "PROJECT_ID": {
+        const projects = await ctx.client.projects.list({ maxItems: 1 });
+        const first = projects[0] as { id?: number } | undefined;
+        if (first?.id !== undefined) resolved = String(first.id);
+        break;
+      }
+      case "TODOSET_ID": {
+        const projectId = await resolveFixtureId(ctx, "PROJECT_ID");
+        if (!projectId) break;
+        const project = await ctx.client.projects.get(Number(projectId));
+        const dock = (project as { dock?: Array<{ name?: string; id?: number }> }).dock ?? [];
+        const todoset = dock.find((tool) => tool.name === "todoset");
+        if (todoset?.id !== undefined) resolved = String(todoset.id);
+        break;
+      }
+      case "TODOLIST_ID": {
+        const todosetId = await resolveFixtureId(ctx, "TODOSET_ID");
+        if (!todosetId) break;
+        const todolists = await ctx.client.todolists.list(Number(todosetId), { maxItems: 1 });
+        const first = todolists[0] as { id?: number } | undefined;
+        if (first?.id !== undefined) resolved = String(first.id);
+        break;
+      }
+      case "TODO_ID": {
+        const todolistId = await resolveFixtureId(ctx, "TODOLIST_ID");
+        if (!todolistId) break;
+        const todos = await ctx.client.todos.list(Number(todolistId), { maxItems: 1 });
+        const first = todos[0] as { id?: number } | undefined;
+        if (first?.id !== undefined) resolved = String(first.id);
+        break;
+      }
+    }
+  } catch {
+    // Discovery is best-effort; failures fall through to skip.
+  }
+
+  cache.set(key, resolved ?? null);
+  return resolved;
+}
+
+/** Test-only helper: clear the cache between runs. */
+export function _resetFixtureCache(): void {
+  cache.clear();
+}

conformance/runner/typescript/live-dispatch.test.ts

@@ -0,0 +1,45 @@
+/**
+ * Offline tests for the dispatch coverage gate.
+ *
+ * The gate must reject any operation referenced by a fixture that doesn't
+ * have a dispatch entry. Critically, it must also reject inherited
+ * Object.prototype keys (`toString`, `hasOwnProperty`, etc.) — pre-fix the
+ * gate used `in` and would have let those slip through.
+ */
+
+import { describe, it, expect } from "vitest";
+import { assertDispatchCoverage } from "./live-dispatch.js";
+
+describe("assertDispatchCoverage", () => {
+  it("does not throw for operations that have a dispatch entry", () => {
+    expect(() => assertDispatchCoverage(["ListProjects", "GetProject"])).not.toThrow();
+  });
+
+  it("throws for operations missing a dispatch entry", () => {
+    expect(() => assertDispatchCoverage(["NoSuchOperation"])).toThrow(
+      /missing dispatch cases for: NoSuchOperation/,
+    );
+  });
+
+  it("rejects Object.prototype keys via Object.hasOwn semantics", () => {
+    // Pre-fix the gate used `in`, which traverses the prototype chain.
+    // A fixture entry like { operation: "toString" } would have passed
+    // the gate (since toString is an inherited property of all objects)
+    // even though no dispatch case exists for it.
+    expect(() => assertDispatchCoverage(["toString"])).toThrow(
+      /missing dispatch cases for: toString/,
+    );
+    expect(() => assertDispatchCoverage(["hasOwnProperty"])).toThrow(
+      /missing dispatch cases for: hasOwnProperty/,
+    );
+    expect(() => assertDispatchCoverage(["constructor"])).toThrow(
+      /missing dispatch cases for: constructor/,
+    );
+  });
+
+  it("collects all missing operations into a single error", () => {
+    expect(() => assertDispatchCoverage(["ListProjects", "MissingA", "MissingB"])).toThrow(
+      /missing dispatch cases for: MissingA, MissingB/,
+    );
+  });
+});