diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fb123f14..bd15fc71 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -247,6 +247,54 @@ beyond the language-specific drift checks. These run as part of `make check`: endpoint or append a one-line entry to the allowlist with a justification comment. +## Live canary + +The TypeScript runner also drives a *live canary* against a real Basecamp +backend. It dispatches every operation in +[`conformance/tests/live-my-surface.json`](conformance/tests/live-my-surface.json) +through the SDK's typed surface, captures the raw wire response (bytes + +headers), and validates each response body against the OpenAPI response +schema. Forward-compat additions on the wire surface as "extras observed" +in the run summary — never as failures — so new BC5 fields don't break +the canary while still being visible. + +The canary is **opt-in** and **does not run as part of `make check`**: + +```bash +BASECAMP_LIVE=1 \ +BASECAMP_TOKEN= \ +BASECAMP_ACCOUNT_ID= \ +make conformance-typescript-live +``` + +Optional env: + +- `BASECAMP_HOST` — backend **origin** only (e.g. `https://3.basecampapi.com`); + the runner appends `/{accountId}` to mirror `createBasecampClient`'s + default URL composition. +- `BASECAMP_BACKEND=bc4|bc5` — namespaces persisted snapshots so BC4 and + BC5 runs don't collide. +- `LIVE_RECORD_DIR=` — persists wire snapshots to + `//wire/.json`. Used by downstream cross-language + decoders (PR 3) and BC4↔BC5 comparison (PR 4). +- `BASECAMP_BC4_PROJECT_ID` / `BASECAMP_BC5_PROJECT_ID` / + `BASECAMP_PROJECT_ID` etc. — explicit fixture-IDs override the runner's + discovery walk. Same pattern applies for `TODOSET_ID`, `TODOLIST_ID`, + `TODO_ID`. + +Tests skip with a clear `skipReason` when a fixture-ID can't be resolved +(no env override, no discovery match) — they don't fail. + +Adding an operation to the live canary requires both a fixture entry in +`live-my-surface.json` and a dispatch case in +`conformance/runner/typescript/live-dispatch.ts`. The runner's startup +gate refuses to run if any fixture operation lacks a dispatch. + +Because live canary fixtures live in the shared `conformance/tests/` directory, +offline conformance runners must treat `mode` as part of the shared schema and +execute only mock tests: omitted `mode` or `mode: "mock"`. `mode: "live"` entries +belong to the TypeScript live wire-capture runner until replay runners are added. + ## API gap registry (`spec/api-gaps/`) When BC ships a new user-visible feature without a JSON API (or with an diff --git a/Makefile b/Makefile index 10e97d7b..9a160333 100644 --- a/Makefile +++ b/Makefile @@ -370,7 +370,7 @@ py-clean: # Conformance Test targets #------------------------------------------------------------------------------ -.PHONY: conformance conformance-go conformance-kotlin conformance-typescript conformance-ruby conformance-python conformance-build +.PHONY: conformance conformance-go conformance-kotlin conformance-typescript conformance-typescript-live conformance-ruby conformance-python conformance-build # Build conformance test runner conformance-build: @@ -392,6 +392,17 @@ conformance-typescript: @echo "==> Running TypeScript conformance tests..." cd conformance/runner/typescript && npm ci && npm test +# Run TypeScript live canary against a real Basecamp backend. +# +# Required env: BASECAMP_LIVE=1, BASECAMP_TOKEN, BASECAMP_ACCOUNT_ID. +# Optional env: BASECAMP_HOST (origin only, e.g. https://3.basecampapi.com — +# runner appends /{accountId}); BASECAMP_BACKEND=bc4|bc5 to namespace +# snapshots; LIVE_RECORD_DIR to persist wire snapshots for downstream +# replay/compare. Opt-in: not invoked by `make check`. +conformance-typescript-live: + @echo "==> Running TypeScript live canary..." + cd conformance/runner/typescript && npm ci && BASECAMP_LIVE=1 npm test + # Run Ruby conformance tests conformance-ruby: @echo "==> Running Ruby conformance tests..." diff --git a/conformance/runner/go/main.go b/conformance/runner/go/main.go index 2b50e9ff..a9df5cb8 100644 --- a/conformance/runner/go/main.go +++ b/conformance/runner/go/main.go @@ -28,6 +28,11 @@ import ( // TestCase represents a single conformance test. type TestCase struct { + // Mode is "mock" (default) or "live". Live tests are owned by the TS + // runner; non-TS runners filter them out at load time so unresolved + // fixture placeholders and unknown operations don't false-pass as + // mock conformance. + Mode string `json:"mode"` Name string `json:"name"` Description string `json:"description"` Operation string `json:"operation"` @@ -173,7 +178,16 @@ func loadTests(filename string) ([]TestCase, error) { return nil, err } - return tests, nil + // Live tests are TS-only — filter them out so this runner doesn't + // attempt mock dispatch on entries with unresolved ${PROJECT_ID} + // fixtures or operations that only the live runner knows about. + mockTests := tests[:0] + for _, tc := range tests { + if tc.Mode == "" || tc.Mode == "mock" { + mockTests = append(mockTests, tc) + } + } + return mockTests, nil } // Default account ID for conformance tests diff --git a/conformance/runner/python/runner.py b/conformance/runner/python/runner.py index fdf94cd9..1486acfb 100644 --- a/conformance/runner/python/runner.py +++ b/conformance/runner/python/runner.py @@ -483,8 +483,15 @@ def run(self) -> int: skipped = 0 for file in files: - print(f"\n=== {file.name} ===") tests = json.loads(file.read_text()) + # Live tests are TS-only (canonical wire-capturer); filter them out + # before mock dispatch so unresolved ${PROJECT_ID} fixtures and + # live-only operations don't surface here. + tests = [t for t in tests if t.get("mode", "mock") == "mock"] + if not tests: + continue + + print(f"\n=== {file.name} ===") for test_case in tests: name = test_case["name"] diff --git a/conformance/runner/ruby/runner.rb b/conformance/runner/ruby/runner.rb index 8983f8a6..20d8a0d2 100644 --- a/conformance/runner/ruby/runner.rb +++ b/conformance/runner/ruby/runner.rb @@ -614,9 +614,16 @@ def run results = [] files.each do |file| + tests = JSON.parse(File.read(file)) + # Live tests are TS-only (canonical wire-capturer); accept only mock + # so unresolved ${PROJECT_ID} fixtures and live-only operations don't + # surface as mock failures or false passes — and any future mode added + # to the schema enum stays opt-in for this runner. + tests = tests.select { |t| (t["mode"] || "mock") == "mock" } + next if tests.empty? + puts "\n=== #{File.basename(file)} ===" - tests = JSON.parse(File.read(file)) tests.each do |test_case| if RUBY_SKIPS.include?(test_case["name"]) skipped += 1 diff --git a/conformance/runner/typescript/fixtures.ts b/conformance/runner/typescript/fixtures.ts new file mode 100644 index 00000000..744aab76 --- /dev/null +++ b/conformance/runner/typescript/fixtures.ts @@ -0,0 +1,111 @@ +/** + * Fixture-ID resolution for live canary tests. + * + * Resolution ladder (per §5d of the BC5-readiness plan): + * 1. Explicit per-backend env var, e.g. BASECAMP_BC4_PROJECT_ID + * 2. Generic env var, e.g. BASECAMP_PROJECT_ID + * 3. Discovery via the SDK (ListProjects → first project; etc.) + * 4. Fall through: undefined; caller skips with skipReason. + * + * Resolution is cached per-backend so discovery only fires once per run. + */ + +import type { BasecampClient } from "@37signals/basecamp"; + +export type Backend = "bc4" | "bc5" | "unknown"; + +export interface FixtureContext { + client: BasecampClient; + backend: Backend; +} + +const cache = new Map(); + +function cacheKey(backend: Backend, name: string): string { + return `${backend}:${name}`; +} + +function fromEnv(backend: Backend, name: string): string | undefined { + const upper = name.toUpperCase(); + if (backend !== "unknown") { + const explicit = process.env[`BASECAMP_${backend.toUpperCase()}_${upper}`]; + if (explicit) return explicit; + } + const generic = process.env[`BASECAMP_${upper}`]; + if (generic) return generic; + return undefined; +} + +/** + * Resolve a fixture-ID by name. Returns the resolved string or undefined if + * not resolvable; caller is responsible for the skip-with-reason path. + * + * Discovery walks: + * PROJECT_ID → ListProjects → first project + * TODOSET_ID → walk dock of resolved project, pick first todoset tool + * TODOLIST_ID → ListTodolists for resolved todoset, pick first + * TODO_ID → ListTodos for resolved todolist, pick first + */ +export async function resolveFixtureId( + ctx: FixtureContext, + name: string, +): Promise { + const key = cacheKey(ctx.backend, name); + if (cache.has(key)) { + const cached = cache.get(key); + return cached ?? undefined; + } + + const env = fromEnv(ctx.backend, name); + if (env) { + cache.set(key, env); + return env; + } + + let resolved: string | undefined; + try { + switch (name) { + case "PROJECT_ID": { + const projects = await ctx.client.projects.list({ maxItems: 1 }); + const first = projects[0] as { id?: number } | undefined; + if (first?.id !== undefined) resolved = String(first.id); + break; + } + case "TODOSET_ID": { + const projectId = await resolveFixtureId(ctx, "PROJECT_ID"); + if (!projectId) break; + const project = await ctx.client.projects.get(Number(projectId)); + const dock = (project as { dock?: Array<{ name?: string; id?: number }> }).dock ?? []; + const todoset = dock.find((tool) => tool.name === "todoset"); + if (todoset?.id !== undefined) resolved = String(todoset.id); + break; + } + case "TODOLIST_ID": { + const todosetId = await resolveFixtureId(ctx, "TODOSET_ID"); + if (!todosetId) break; + const todolists = await ctx.client.todolists.list(Number(todosetId), { maxItems: 1 }); + const first = todolists[0] as { id?: number } | undefined; + if (first?.id !== undefined) resolved = String(first.id); + break; + } + case "TODO_ID": { + const todolistId = await resolveFixtureId(ctx, "TODOLIST_ID"); + if (!todolistId) break; + const todos = await ctx.client.todos.list(Number(todolistId), { maxItems: 1 }); + const first = todos[0] as { id?: number } | undefined; + if (first?.id !== undefined) resolved = String(first.id); + break; + } + } + } catch { + // Discovery is best-effort; failures fall through to skip. + } + + cache.set(key, resolved ?? null); + return resolved; +} + +/** Test-only helper: clear the cache between runs. */ +export function _resetFixtureCache(): void { + cache.clear(); +} diff --git a/conformance/runner/typescript/live-dispatch.test.ts b/conformance/runner/typescript/live-dispatch.test.ts new file mode 100644 index 00000000..39ed7cab --- /dev/null +++ b/conformance/runner/typescript/live-dispatch.test.ts @@ -0,0 +1,45 @@ +/** + * Offline tests for the dispatch coverage gate. + * + * The gate must reject any operation referenced by a fixture that doesn't + * have a dispatch entry. Critically, it must also reject inherited + * Object.prototype keys (`toString`, `hasOwnProperty`, etc.) — pre-fix the + * gate used `in` and would have let those slip through. + */ + +import { describe, it, expect } from "vitest"; +import { assertDispatchCoverage } from "./live-dispatch.js"; + +describe("assertDispatchCoverage", () => { + it("does not throw for operations that have a dispatch entry", () => { + expect(() => assertDispatchCoverage(["ListProjects", "GetProject"])).not.toThrow(); + }); + + it("throws for operations missing a dispatch entry", () => { + expect(() => assertDispatchCoverage(["NoSuchOperation"])).toThrow( + /missing dispatch cases for: NoSuchOperation/, + ); + }); + + it("rejects Object.prototype keys via Object.hasOwn semantics", () => { + // Pre-fix the gate used `in`, which traverses the prototype chain. + // A fixture entry like { operation: "toString" } would have passed + // the gate (since toString is an inherited property of all objects) + // even though no dispatch case exists for it. + expect(() => assertDispatchCoverage(["toString"])).toThrow( + /missing dispatch cases for: toString/, + ); + expect(() => assertDispatchCoverage(["hasOwnProperty"])).toThrow( + /missing dispatch cases for: hasOwnProperty/, + ); + expect(() => assertDispatchCoverage(["constructor"])).toThrow( + /missing dispatch cases for: constructor/, + ); + }); + + it("collects all missing operations into a single error", () => { + expect(() => assertDispatchCoverage(["ListProjects", "MissingA", "MissingB"])).toThrow( + /missing dispatch cases for: MissingA, MissingB/, + ); + }); +}); diff --git a/conformance/runner/typescript/live-dispatch.ts b/conformance/runner/typescript/live-dispatch.ts new file mode 100644 index 00000000..b053404b --- /dev/null +++ b/conformance/runner/typescript/live-dispatch.ts @@ -0,0 +1,129 @@ +/** + * Live-mode operation dispatch for the canary. + * + * Each entry in `LIVE_OPERATIONS` declares (a) which fixture-IDs the call + * needs and (b) the SDK call itself. The runner pre-resolves fixture-IDs + * outside the wire-capture window so discovery traffic (e.g. the + * `ListProjects` call that backs PROJECT_ID resolution) doesn't bleed into + * the snapshot for the actual operation under test. + * + * `LIVE_OPERATIONS` is the single source of truth for the coverage gate: + * any operation referenced by a live test must appear here, or the runner + * refuses to start. + */ + +import type { BasecampClient } from "@37signals/basecamp"; +import type { FixtureContext } from "./fixtures.js"; + +export interface DispatchResult { + /** Resolved fixture-ID values, for diagnostics. */ + resolvedIds: Record; + /** SDK-decoded result (for downstream decode-success reporting). */ + result?: unknown; +} + +export interface DispatchSpec { + /** + * Fixture-ID names this operation requires. Pre-resolved by the runner + * before wire capture starts; missing fixtures cause the test to skip. + */ + fixtures: readonly string[]; + /** The SDK call itself, executed under wire capture. */ + call: (ctx: FixtureContext, ids: Record) => Promise; +} + +export const LIVE_OPERATIONS: Record = { + ListProjects: { + fixtures: [], + call: async (ctx) => { + const result = await ctx.client.projects.list(); + return { resolvedIds: {}, result }; + }, + }, + + GetProject: { + fixtures: ["PROJECT_ID"], + call: async (ctx, ids) => { + const result = await ctx.client.projects.get(Number(ids.PROJECT_ID)); + return { resolvedIds: ids, result }; + }, + }, + + GetMyAssignments: { + fixtures: [], + call: async (ctx) => { + const result = await ctx.client.myAssignments.myAssignments(); + return { resolvedIds: {}, result }; + }, + }, + + GetMyCompletedAssignments: { + fixtures: [], + call: async (ctx) => { + const result = await ctx.client.myAssignments.myCompletedAssignments(); + return { resolvedIds: {}, result }; + }, + }, + + GetMyDueAssignments: { + fixtures: [], + call: async (ctx) => { + const result = await ctx.client.myAssignments.myDueAssignments(); + return { resolvedIds: {}, result }; + }, + }, + + GetMyNotifications: { + fixtures: [], + call: async (ctx) => { + const result = await ctx.client.myNotifications.myNotifications(); + return { resolvedIds: {}, result }; + }, + }, + + GetMyProfile: { + fixtures: [], + call: async (ctx) => { + const result = await ctx.client.people.me(); + return { resolvedIds: {}, result }; + }, + }, + + GetTodoset: { + fixtures: ["TODOSET_ID"], + call: async (ctx, ids) => { + const result = await ctx.client.todosets.get(Number(ids.TODOSET_ID)); + return { resolvedIds: ids, result }; + }, + }, + + ListTodolists: { + fixtures: ["TODOSET_ID"], + call: async (ctx, ids) => { + const result = await ctx.client.todolists.list(Number(ids.TODOSET_ID)); + return { resolvedIds: ids, result }; + }, + }, + + ListTodos: { + fixtures: ["TODOLIST_ID"], + call: async (ctx, ids) => { + const result = await ctx.client.todos.list(Number(ids.TODOLIST_ID)); + return { resolvedIds: ids, result }; + }, + }, +}; + +/** + * Validate that every operation referenced in the fixture has a dispatch + * case. Uses `Object.hasOwn` rather than `in` so inherited keys + * (`toString`, `hasOwnProperty`, etc.) can't sneak past the gate. + */ +export function assertDispatchCoverage(operationsInFixture: string[]): void { + const missing = operationsInFixture.filter((op) => !Object.hasOwn(LIVE_OPERATIONS, op)); + if (missing.length === 0) return; + throw new Error( + `Live runner is missing dispatch cases for: ${missing.join(", ")}. ` + + `Add a DispatchSpec to LIVE_OPERATIONS in live-dispatch.ts.`, + ); +} diff --git a/conformance/runner/typescript/live-runner.test.ts b/conformance/runner/typescript/live-runner.test.ts new file mode 100644 index 00000000..9cfce685 --- /dev/null +++ b/conformance/runner/typescript/live-runner.test.ts @@ -0,0 +1,315 @@ +/** + * Live-mode conformance runner. + * + * Loads only mode="live" tests from conformance/tests/, dispatches each + * through the SDK against a real Basecamp backend, captures raw wire + * responses, validates them against the OpenAPI response schema, and + * reports per-test pass/skip/fail. + * + * Gating: opt-in via BASECAMP_LIVE=1. Without it, the entire suite skips + * — make check stays fully offline. + * + * This runner is the canonical wire-capturer (per §5f of the BC5-readiness + * plan); other-language runners replay these snapshots in PR 3. + */ + +import { describe, it, beforeAll, expect } from "vitest"; +import { createBasecampClient } from "@37signals/basecamp"; +import type { BasecampClient } from "@37signals/basecamp"; +import * as fs from "node:fs"; +import * as path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { installWireCapture, type WireSnapshot, type WirePage } from "./wire-capture.js"; +import { validateResponse, type ValidationResult } from "./schema-validator.js"; +import { + LIVE_OPERATIONS, + assertDispatchCoverage, +} from "./live-dispatch.js"; +import { resolveFixtureId, type Backend, type FixtureContext } from "./fixtures.js"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const TESTS_DIR = path.resolve(__dirname, "../../tests"); + +const LIVE_ENABLED = process.env.BASECAMP_LIVE === "1"; +const TOKEN = process.env.BASECAMP_TOKEN; +const ACCOUNT_ID = process.env.BASECAMP_ACCOUNT_ID; +// BASECAMP_HOST is origin-only (e.g. https://3.basecampapi.com); we append +// /{accountId} to mirror createBasecampClient's default URL composition. +const HOST = process.env.BASECAMP_HOST?.replace(/\/$/, ""); +const RECORD_DIR = process.env.LIVE_RECORD_DIR; +// Backend label namespaces snapshots so BC4 and BC5 runs don't collide. +const BACKEND: Backend = (process.env.BASECAMP_BACKEND ?? "unknown") as Backend; + +interface LiveAssertion { + type: "liveCallSucceeds" | "liveResponseFieldsRequired" | "liveResponseFieldsExpected" | "liveSchemaValidate"; + fields?: string[]; + enabled?: boolean; +} + +interface LiveTestCase { + mode: "live"; + name: string; + description?: string; + operation: string; + fixtureIds?: Record; + liveAssertions: LiveAssertion[]; + tags?: string[]; +} + +interface RawTestCase { + mode?: string; + operation?: string; +} + +function loadLiveTests(): { filename: string; tests: LiveTestCase[] }[] { + const files = fs + .readdirSync(TESTS_DIR) + .filter((f) => f.endsWith(".json")) + .sort(); + + return files + .map((filename) => { + const content = fs.readFileSync(path.join(TESTS_DIR, filename), "utf-8"); + const all = JSON.parse(content) as RawTestCase[]; + const tests = all.filter((tc) => tc.mode === "live") as LiveTestCase[]; + return { filename, tests }; + }) + .filter((suite) => suite.tests.length > 0); +} + +interface RunSummary { + /** Field paths seen on raw wire bodies but not declared in the OpenAPI schema. */ + extrasObserved: Map>; +} + +const summary: RunSummary = { + extrasObserved: new Map(), +}; + +function recordExtras(operation: string, extras: string[]): void { + if (extras.length === 0) return; + let bucket = summary.extrasObserved.get(operation); + if (!bucket) { + bucket = new Set(); + summary.extrasObserved.set(operation, bucket); + } + for (const e of extras) bucket.add(e); +} + +function persistSnapshot(testName: string, snapshot: WireSnapshot): void { + if (!RECORD_DIR) return; + const wireDir = path.join(RECORD_DIR, BACKEND, "wire"); + fs.mkdirSync(wireDir, { recursive: true }); + const safeName = testName.replace(/[^a-z0-9_-]+/gi, "_"); + const file = path.join(wireDir, `${safeName}.json`); + fs.writeFileSync(file, JSON.stringify(snapshot, null, 2)); +} + +function checkRequiredFields(page: WirePage, fields: string[]): string[] { + const errors: string[] = []; + const body = page.body; + for (const fieldPath of fields) { + if (!fieldExists(body, fieldPath)) { + errors.push(`required field absent: ${fieldPath}`); + } + } + return errors; +} + +function fieldExists(value: unknown, fieldPath: string): boolean { + const parts = fieldPath.split("."); + let cur: unknown = value; + for (let i = 0; i < parts.length; i++) { + const part = parts[i]; + if (cur === null || cur === undefined) return false; + if (typeof cur !== "object") return false; + if (Array.isArray(cur)) { + // Array path means "every item must have this field". Empty arrays count as present. + // Pass the suffix from the current segment (`i`), not `parts.indexOf(part)`: + // the latter returns the first occurrence and misroutes paths with repeated + // segment names (e.g. `a.b.b.x`). + if (cur.length === 0) return true; + const remaining = parts.slice(i).join("."); + return cur.every((item) => fieldExists(item, remaining)); + } + if (!(part in (cur as Record))) return false; + cur = (cur as Record)[part]; + } + return true; +} + +const LIVE_DESCRIBE = LIVE_ENABLED ? describe : describe.skip; + +LIVE_DESCRIBE("conformance live runner", () => { + const suites = LIVE_ENABLED ? loadLiveTests() : []; + let client: BasecampClient | null = null; + + beforeAll(() => { + if (!LIVE_ENABLED) return; + + const missing: string[] = []; + if (!TOKEN) missing.push("BASECAMP_TOKEN"); + if (!ACCOUNT_ID) missing.push("BASECAMP_ACCOUNT_ID"); + if (missing.length > 0) { + throw new Error( + `Live mode requires env vars: ${missing.join(", ")}. ` + + `BASECAMP_HOST is origin-only (e.g. https://3.basecampapi.com); the runner appends /{accountId}.`, + ); + } + + const allOperations = suites.flatMap((suite) => suite.tests.map((t) => t.operation)); + assertDispatchCoverage(allOperations); + + const baseUrl = HOST ? `${HOST}/${ACCOUNT_ID}` : undefined; + client = createBasecampClient({ + accountId: ACCOUNT_ID!, + accessToken: TOKEN!, + baseUrl, + }); + }); + + if (!LIVE_ENABLED) { + it.skip("BASECAMP_LIVE not set — live canary skipped", () => {}); + return; + } + + for (const { filename, tests } of suites) { + describe(`live/${filename}`, () => { + for (const tc of tests) { + it(tc.name, async (testCtx) => { + const spec = LIVE_OPERATIONS[tc.operation]; + // Coverage is enforced in beforeAll, but this guards against races. + if (!spec) { + throw new Error(`No dispatch for operation ${tc.operation}`); + } + if (!client) { + throw new Error("Live client not constructed"); + } + + const ctx: FixtureContext = { client, backend: BACKEND }; + + // Pre-resolve fixture-IDs OUTSIDE the wire-capture window so + // discovery traffic (e.g. ListProjects → first project) doesn't + // bleed into the snapshot for the operation under test. + const resolvedIds: Record = {}; + for (const fixture of spec.fixtures) { + const value = await resolveFixtureId(ctx, fixture); + if (!value) { + testCtx.skip(`Fixture ID for \${${fixture}} not available`); + return; + } + resolvedIds[fixture] = value; + } + + const capture = installWireCapture(); + let dispatchError: Error | undefined; + + try { + await spec.call(ctx, resolvedIds); + } catch (err) { + dispatchError = err instanceof Error ? err : new Error(String(err)); + } finally { + capture.restore(); + } + + const snapshot = capture.drain(); + persistSnapshot(tc.name, snapshot); + + if (dispatchError) { + throw new Error(`SDK dispatch threw: ${dispatchError.message}`); + } + + const failures: string[] = []; + const assertions = tc.liveAssertions ?? []; + // liveSchemaValidate defaults to enabled — ensure it runs even if absent. + const hasExplicitSchema = assertions.some((a) => a.type === "liveSchemaValidate"); + const effective = hasExplicitSchema + ? assertions + : [...assertions, { type: "liveSchemaValidate" } as LiveAssertion]; + + for (const a of effective) { + if (a.enabled === false) continue; + + if (a.type === "liveCallSucceeds") { + if (snapshot.pages_count === 0) { + failures.push("liveCallSucceeds: no pages captured"); + continue; + } + const firstStatus = snapshot.pages[0].status; + if (firstStatus < 200 || firstStatus >= 300) { + failures.push(`liveCallSucceeds: first page returned HTTP ${firstStatus}`); + } + } + + if (a.type === "liveSchemaValidate") { + const result = validatePages(tc.operation, snapshot.pages); + recordExtras(tc.operation, result.extras); + if (!result.ok) { + for (const err of result.errors) failures.push(`liveSchemaValidate: ${err}`); + } + } + + if (a.type === "liveResponseFieldsRequired") { + const fields = a.fields ?? []; + for (let i = 0; i < snapshot.pages.length; i++) { + const errors = checkRequiredFields(snapshot.pages[i], fields); + for (const e of errors) failures.push(`liveResponseFieldsRequired (page ${i + 1}): ${e}`); + } + } + + if (a.type === "liveResponseFieldsExpected") { + const fields = a.fields ?? []; + for (let i = 0; i < snapshot.pages.length; i++) { + const errors = checkRequiredFields(snapshot.pages[i], fields); + for (const e of errors) { + // eslint-disable-next-line no-console + console.warn(`[live-canary] ${tc.name}: liveResponseFieldsExpected (page ${i + 1}) ${e}`); + } + } + } + } + + if (failures.length > 0) { + throw new Error(failures.join("\n")); + } + }); + } + }); + } +}); + +function validatePages(operation: string, pages: WirePage[]): ValidationResult { + if (pages.length === 0) { + return { ok: false, errors: ["no pages to validate"], extras: [] }; + } + const errors: string[] = []; + const extras = new Set(); + let allOk = true; + for (let i = 0; i < pages.length; i++) { + const result = validateResponse(operation, pages[i].body); + if (!result.ok) { + allOk = false; + for (const e of result.errors) errors.push(`page ${i + 1}: ${e}`); + } + for (const e of result.extras) extras.add(e); + } + return { ok: allOk, errors, extras: [...extras] }; +} + +// After-all summary: emit extras observed so absorption planning has signal. +if (LIVE_ENABLED) { + // Vitest hooks at module scope: this fires after the file finishes. + process.on("beforeExit", () => { + if (summary.extrasObserved.size === 0) return; + // eslint-disable-next-line no-console + console.log("\n[live-canary] Extras observed (raw wire fields not in OpenAPI schema):"); + for (const [op, fields] of summary.extrasObserved) { + // eslint-disable-next-line no-console + console.log(` ${op}: ${[...fields].sort().join(", ")}`); + } + }); +} + +// Suppress unused-import warnings for type-only imports under noUnusedLocals. +export const _types: typeof expect = expect; diff --git a/conformance/runner/typescript/package-lock.json b/conformance/runner/typescript/package-lock.json index 8311789f..7bf44134 100644 --- a/conformance/runner/typescript/package-lock.json +++ b/conformance/runner/typescript/package-lock.json @@ -6,7 +6,9 @@ "": { "name": "conformance-runner-typescript", "dependencies": { - "@37signals/basecamp": "file:../../../typescript" + "@37signals/basecamp": "file:../../../typescript", + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1" }, "devDependencies": { "msw": "^2.12.0", @@ -1121,6 +1123,39 @@ "url": "https://opencollective.com/vitest" } }, + "node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, "node_modules/ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", @@ -1330,6 +1365,28 @@ "node": ">=12.0.0" } }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.0.tgz", + "integrity": "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/fdir": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", @@ -1407,6 +1464,12 @@ "dev": true, "license": "MIT" }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, "node_modules/magic-string": { "version": "0.30.21", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", @@ -1582,6 +1645,15 @@ "node": ">=0.10.0" } }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/rettime": { "version": "0.10.1", "resolved": "https://registry.npmjs.org/rettime/-/rettime-0.10.1.tgz", diff --git a/conformance/runner/typescript/package.json b/conformance/runner/typescript/package.json index ee05da93..6dd5065c 100644 --- a/conformance/runner/typescript/package.json +++ b/conformance/runner/typescript/package.json @@ -11,6 +11,8 @@ "msw": "^2.12.0" }, "dependencies": { - "@37signals/basecamp": "file:../../../typescript" + "@37signals/basecamp": "file:../../../typescript", + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1" } } diff --git a/conformance/runner/typescript/runner.test.ts b/conformance/runner/typescript/runner.test.ts index 55a40d63..0255af0d 100644 --- a/conformance/runner/typescript/runner.test.ts +++ b/conformance/runner/typescript/runner.test.ts @@ -49,6 +49,11 @@ interface TestCase { assertions: Assertion[]; tags?: string[]; configOverrides?: { baseUrl?: string; maxPages?: number; maxItems?: number }; + /** + * Live tests are loaded by live-runner.test.ts; this runner ignores them. + * Defaults to "mock" when omitted. + */ + mode?: "mock" | "live"; } // ============================================================================= @@ -704,10 +709,16 @@ function loadTestSuites(): { filename: string; tests: TestCase[] }[] { .filter((f) => f.endsWith(".json")) .sort(); - return files.map((filename) => { - const content = fs.readFileSync(path.join(TESTS_DIR, filename), "utf-8"); - return { filename, tests: JSON.parse(content) as TestCase[] }; - }); + return files + .map((filename) => { + const content = fs.readFileSync(path.join(TESTS_DIR, filename), "utf-8"); + const all = JSON.parse(content) as TestCase[]; + // Live tests are owned by live-runner.test.ts. Drop them here so they + // never reach installMockHandlers / MSW. + const tests = all.filter((tc) => (tc.mode ?? "mock") === "mock"); + return { filename, tests }; + }) + .filter((suite) => suite.tests.length > 0); } /** diff --git a/conformance/runner/typescript/schema-validator.test.ts b/conformance/runner/typescript/schema-validator.test.ts new file mode 100644 index 00000000..ec097354 --- /dev/null +++ b/conformance/runner/typescript/schema-validator.test.ts @@ -0,0 +1,161 @@ +/** + * Offline tests for the live-canary schema validator. + * + * The live canary path itself requires real Basecamp credentials, so these + * tests exercise the validator with crafted payloads to catch wiring bugs + * (Ajv $ref resolution, extras-collection on arrays + nested objects, etc.) + * without needing live access. + */ + +import { describe, it, expect } from "vitest"; +import { validateResponse } from "./schema-validator.js"; + +// ============================================================================= +// ListProjects (array response — Project[]) +// ============================================================================= + +const conformantProject = { + id: 1, + status: "active", + created_at: "2026-01-01T00:00:00Z", + updated_at: "2026-01-02T00:00:00Z", + name: "Test Project", + url: "https://3.basecampapi.com/999/projects/1.json", + app_url: "https://3.basecamp.com/999/projects/1", +}; + +describe("validateResponse — ListProjects (array root)", () => { + it("compiles with $ref resolved against the registered OpenAPI doc", () => { + // This is the exact bug the reviewer flagged: pre-fix, Ajv tried to + // resolve "#/components/schemas/ListProjectsResponseContent" against + // the fragment root and threw `can't resolve reference ... from id #`. + const result = validateResponse("ListProjects", [conformantProject]); + expect(result.errors).toEqual([]); + expect(result.ok).toBe(true); + }); + + it("flags missing required fields", () => { + const broken = { ...conformantProject } as Record; + delete broken.name; + const result = validateResponse("ListProjects", [broken]); + expect(result.ok).toBe(false); + expect(result.errors.some((e) => e.includes("name"))).toBe(true); + }); + + it("permits extra fields without failing (forward-compat)", () => { + const withExtras = { ...conformantProject, future_field: "BC5 addition" }; + const result = validateResponse("ListProjects", [withExtras]); + expect(result.ok).toBe(true); + }); + + it("collects item-level extras with [] path prefix", () => { + const withExtras = { ...conformantProject, future_field: "BC5 addition" }; + const result = validateResponse("ListProjects", [withExtras]); + // Path convention: "[]" segment for array items, then ".field" for keys. + expect(result.extras).toContain("[].future_field"); + }); + + it("emits known-property paths so nested extras stay visible", () => { + // Project has no nested object schemas exercised here; this asserts + // that declared properties don't get reported as extras. + const result = validateResponse("ListProjects", [conformantProject]); + expect(result.extras).not.toContain("[].id"); + expect(result.extras).not.toContain("[].status"); + }); +}); + +// ============================================================================= +// GetMyNotifications (object response with array properties) +// ============================================================================= + +describe("validateResponse — GetMyNotifications (object root, array fields)", () => { + it("validates an empty payload (all arrays absent)", () => { + const result = validateResponse("GetMyNotifications", {}); + expect(result.ok).toBe(true); + }); + + it("validates with empty arrays", () => { + const result = validateResponse("GetMyNotifications", { + unreads: [], + reads: [], + memories: [], + bubble_ups: [], + scheduled_bubble_ups: [], + }); + expect(result.ok).toBe(true); + }); + + it("collects extras at the root", () => { + const result = validateResponse("GetMyNotifications", { + unreads: [], + hypothetical_new_top_level: 42, + }); + expect(result.ok).toBe(true); + expect(result.extras).toContain("hypothetical_new_top_level"); + }); + + it("collects extras inside array-valued properties", () => { + const minimalNotification = { + id: 1, + created_at: "2026-01-01T00:00:00Z", + updated_at: "2026-01-02T00:00:00Z", + }; + const result = validateResponse("GetMyNotifications", { + unreads: [{ ...minimalNotification, future_envelope_field: "BC5 addition" }], + }); + expect(result.ok).toBe(true); + // Path: [].field — here `unreads[].future_envelope_field`. + expect(result.extras).toContain("unreads[].future_envelope_field"); + }); +}); + +// ============================================================================= +// Non-200 success responses +// ============================================================================= + +describe("validateResponse — non-200 success responses", () => { + it("resolves a schema for operations that return 201 (Create*)", () => { + // CreateProject's success status is 201 Created. Pre-fix the lookup + // checked only "200"/"default" and missed it. + const result = validateResponse("CreateProject", { + id: 1, + status: "active", + created_at: "2026-01-01T00:00:00Z", + updated_at: "2026-01-02T00:00:00Z", + name: "Test", + url: "https://3.basecampapi.com/999/projects/1.json", + app_url: "https://3.basecamp.com/999/projects/1", + }); + // Either ok=true or field-level errors — but never the + // "No response schema found" sentinel (which would prove the lookup + // failed entirely). + expect(result.errors.join("\n")).not.toContain("No response schema found"); + }); +}); + +// ============================================================================= +// Unknown operation +// ============================================================================= + +describe("validateResponse — error paths", () => { + it("returns ok=false with a clear error for unknown operations", () => { + const result = validateResponse("DoesNotExist", {}); + expect(result.ok).toBe(false); + expect(result.errors[0]).toContain("DoesNotExist"); + }); +}); + +// ============================================================================= +// Bodyless 2xx (204 No Content) +// ============================================================================= + +describe("validateResponse — bodyless success responses", () => { + it("returns ok=true for operations whose only success is 204 No Content", () => { + // DeleteBoost is documented with only a 204 response — no JSON body + // ever returns. Pre-fix, the lookup yielded no schema and the + // validator returned ok=false, blocking such ops from the canary. + const result = validateResponse("DeleteBoost", null); + expect(result.ok).toBe(true); + expect(result.errors).toEqual([]); + }); +}); diff --git a/conformance/runner/typescript/schema-validator.ts b/conformance/runner/typescript/schema-validator.ts new file mode 100644 index 00000000..1f2d13d2 --- /dev/null +++ b/conformance/runner/typescript/schema-validator.ts @@ -0,0 +1,270 @@ +/** + * OpenAPI response-body schema validation for the live canary. + * + * Operates on raw JSON (per §5b/5f of the plan) — never on SDK-decoded + * structures, since language-specific decoders silently drop unknown + * fields and we need the canary to surface them. + * + * Rules: + * - additionalProperties permissive (forward-compat must not break the canary) + * - required strict + * - type/format/nullable per OpenAPI + * - $ref rewritten from "#/..." to "openapi.json#/..." so refs in the + * compiled response-schema fragment resolve against the registered + * OpenAPI document, not the fragment root + * - extras collected per-run, walking arrays and nested objects, so + * item-level unknown fields on list responses are visible + */ + +import Ajv, { type ValidateFunction, type ErrorObject } from "ajv"; +import addFormats from "ajv-formats"; +import * as fs from "node:fs"; +import * as path from "node:path"; +import { fileURLToPath } from "node:url"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const OPENAPI_PATH = path.resolve(__dirname, "../../../openapi.json"); +const OPENAPI_KEY = "openapi.json"; + +interface OpenAPIDocument { + paths: Record>; + components?: { schemas?: Record }; +} + +interface OpenAPIOperation { + operationId?: string; + responses?: Record; +} + +interface OpenAPIResponse { + content?: Record; +} + +let ajv: Ajv | null = null; +let openapi: OpenAPIDocument | null = null; +const validatorByOperation = new Map(); + +function init(): { ajv: Ajv; doc: OpenAPIDocument } { + if (ajv && openapi) return { ajv, doc: openapi }; + + ajv = new Ajv({ + strict: false, + allErrors: true, + }); + addFormats(ajv); + + const raw = fs.readFileSync(OPENAPI_PATH, "utf-8"); + openapi = JSON.parse(raw) as OpenAPIDocument; + + // Register the OpenAPI document under a stable key so rewritten refs of + // the form `openapi.json#/...` resolve against it. + ajv.addSchema(openapi as object, OPENAPI_KEY); + + return { ajv, doc: openapi }; +} + +function findResponseSchema(doc: OpenAPIDocument, operationId: string): unknown | null { + for (const pathItem of Object.values(doc.paths)) { + for (const op of Object.values(pathItem)) { + if (op.operationId !== operationId) continue; + const responses = op.responses ?? {}; + // Prefer 200; fall through to any 2xx success (201, 202, 204, ...); + // last resort is "default". Operations that return 201 (Create*) + // shouldn't fall back to "" because their response body still has + // a schema worth validating. + const candidates = ["200", "201", "202", "203", "204", "default"]; + for (const code of candidates) { + if (!responses[code]) continue; + const schema = responses[code].content?.["application/json"]?.schema; + if (schema) return schema; + } + // Last resort: any 2xx key not in the explicit list above. + for (const [code, response] of Object.entries(responses)) { + if (!/^2\d\d$/.test(code)) continue; + const schema = response.content?.["application/json"]?.schema; + if (schema) return schema; + } + } + } + return null; +} + +/** + * Walk the schema tree, doing two things in one pass: + * 1. Rewrite "$ref": "#/..." to "$ref": "openapi.json#/..." so refs + * resolve against the registered OpenAPI doc, not the fragment root. + * 2. Drop "additionalProperties: false" so forward-compat fields on the + * wire don't fail validation. Required-field checks still apply. + */ +function prepareForCompile(schema: unknown): unknown { + if (!schema || typeof schema !== "object") return schema; + if (Array.isArray(schema)) return schema.map(prepareForCompile); + const out: Record = {}; + for (const [key, value] of Object.entries(schema as Record)) { + if (key === "additionalProperties" && value === false) continue; + if (key === "$ref" && typeof value === "string" && value.startsWith("#/")) { + out[key] = `${OPENAPI_KEY}${value}`; + continue; + } + out[key] = prepareForCompile(value); + } + return out; +} + +export interface ValidationResult { + ok: boolean; + /** Validation errors, formatted for human reading. */ + errors: string[]; + /** Field paths present on the wire that the schema did not declare. */ + extras: string[]; +} + +/** + * Validate a single response body against the operation's response schema. + * + * For paginated operations, call once per page; the caller unions extras + * across pages. + */ +export function validateResponse(operationId: string, body: unknown): ValidationResult { + const { ajv, doc } = init(); + + let validator = validatorByOperation.get(operationId); + if (validator === undefined) { + const schema = findResponseSchema(doc, operationId); + if (!schema) { + validatorByOperation.set(operationId, null); + validator = null; + } else { + const prepared = prepareForCompile(schema); + validator = ajv.compile(prepared as object); + validatorByOperation.set(operationId, validator); + } + } + + if (!validator) { + // Distinguish a bodyless success response (e.g. 204 No Content on a + // delete/update) from a missing operation. The former is structurally + // valid by design — no schema means no body to validate. The latter + // is still a hard failure: the operation isn't covered by the spec. + if (operationHasBodylessSuccessOnly(doc, operationId)) { + return { ok: true, errors: [], extras: [] }; + } + return { + ok: false, + errors: [`No response schema found for operation ${operationId}`], + extras: [], + }; + } + + const ok = validator(body) as boolean; + const errors = (validator.errors ?? []).map(formatError); + const schema = findResponseSchema(doc, operationId); + const extras = schema ? collectExtras("", body, schema, doc) : []; + return { ok, errors, extras }; +} + +/** + * True when the operation declares at least one 2xx success response and + * none of its 2xx responses carry an `application/json` schema — i.e. the + * operation is intentionally bodyless (204 No Content, etc). + */ +function operationHasBodylessSuccessOnly(doc: OpenAPIDocument, operationId: string): boolean { + for (const pathItem of Object.values(doc.paths)) { + for (const op of Object.values(pathItem)) { + if (op.operationId !== operationId) continue; + const responses = op.responses ?? {}; + let hasSuccess = false; + for (const [code, response] of Object.entries(responses)) { + if (!/^2\d\d$/.test(code)) continue; + hasSuccess = true; + if (response.content?.["application/json"]?.schema) return false; + } + return hasSuccess; + } + } + return false; +} + +function formatError(err: ErrorObject): string { + const where = err.instancePath || "(root)"; + const expected = err.schemaPath ? ` (schema ${err.schemaPath})` : ""; + return `${where}: ${err.message}${expected}`; +} + +/** + * Resolve $ref chains until we hit a non-ref schema (or a cycle). + * One-level resolution misreports valid fields as extras when the schema + * uses alias chains (e.g. Foo → Bar → Baz). + */ +function resolveRef(schema: unknown, doc: OpenAPIDocument): unknown { + const seen = new Set(); + let current: unknown = schema; + while (current && typeof current === "object" && !Array.isArray(current)) { + const ref = (current as Record)["$ref"]; + if (typeof ref !== "string") return current; + if (seen.has(ref)) return current; + seen.add(ref); + // Accept both "#/components/schemas/X" and "openapi.json#/components/schemas/X". + const m = ref.match(/^(?:openapi\.json)?#\/components\/schemas\/(.+)$/); + if (!m) return current; + const next = doc.components?.schemas?.[m[1]]; + if (!next) return current; + current = next; + } + return current; +} + +/** + * Walk body + schema together, emitting dotted-path field names that + * appear on the wire but are not declared in the schema. + * + * Conventions: + * - Object extras emit at the property path (e.g. `unreads`). + * - Array items use [] as the path segment (e.g. `unreads[].some_field`). + * - Recurses into known properties so nested extras surface. + * - Bounded depth as a cycle guard; OpenAPI schemas are typically shallow. + */ +function collectExtras( + prefix: string, + body: unknown, + schema: unknown, + doc: OpenAPIDocument, + depth = 0, +): string[] { + if (depth > 12) return []; + if (body === null || body === undefined) return []; + const resolved = resolveRef(schema, doc); + if (!resolved || typeof resolved !== "object") return []; + const s = resolved as Record; + + if (Array.isArray(body)) { + if (s.type !== "array" || !s.items) return []; + const seen = new Set(); + const childPrefix = prefix ? `${prefix}[]` : "[]"; + for (const item of body) { + for (const e of collectExtras(childPrefix, item, s.items, doc, depth + 1)) { + seen.add(e); + } + } + return [...seen]; + } + + if (typeof body !== "object") return []; + + // For non-object schemas (e.g., type: "string"), don't recurse. + if (s.type !== undefined && s.type !== "object") return []; + + const props = (s.properties as Record | undefined) ?? {}; + const extras: string[] = []; + for (const [key, value] of Object.entries(body as Record)) { + const fieldPath = prefix ? `${prefix}.${key}` : key; + if (!(key in props)) { + extras.push(fieldPath); + } else { + for (const e of collectExtras(fieldPath, value, props[key], doc, depth + 1)) { + extras.push(e); + } + } + } + return extras; +} diff --git a/conformance/runner/typescript/wire-capture.ts b/conformance/runner/typescript/wire-capture.ts new file mode 100644 index 00000000..05ceb1ee --- /dev/null +++ b/conformance/runner/typescript/wire-capture.ts @@ -0,0 +1,103 @@ +/** + * Wire-capture utilities for the live canary. + * + * Both fetch sites in the SDK (openapi-fetch's default fetch for page 1 and + * the fetchPage closure used for pagination follow-up) ultimately call + * globalThis.fetch. Wrapping the global gives us a single chokepoint where + * we can clone the response and record its raw bytes/headers without + * affecting SDK behavior. + * + * Snapshot format per test: + * { pages: [{status, headers, body}, ...], pages_count: N } + * + * Schema validation runs per page; extras-observed reporting unions extras + * across all pages. + */ + +export interface WirePage { + status: number; + headers: Record; + /** + * Parsed JSON body when the response body parses cleanly; raw text otherwise. + * Always preserved as raw text in `bodyText` so callers can re-parse if needed. + */ + body: unknown; + bodyText: string; + url: string; +} + +export interface WireSnapshot { + pages: WirePage[]; + pages_count: number; +} + +export interface WireCaptureSession { + /** Restore the original fetch. Idempotent. */ + restore(): void; + /** Take the captured pages and reset the buffer for the next test. */ + drain(): WireSnapshot; +} + +/** + * Install a global fetch wrapper that records each call's raw response. + * Returns a session object to drain captured pages and restore the original fetch. + * + * Caveats: + * - Records every fetch made while installed — caller is responsible for + * draining between tests so pages don't bleed across operations. + * - Body is read from a clone, so the original Response stream remains + * consumable by the SDK. + * - Non-JSON response bodies are recorded as text and `body` falls back to + * the raw text. + */ +export function installWireCapture(): WireCaptureSession { + const original = globalThis.fetch; + let pages: WirePage[] = []; + + const wrapped: typeof fetch = async (input, init) => { + const response = await original(input as RequestInfo | URL, init); + try { + const clone = response.clone(); + const text = await clone.text(); + const headers: Record = {}; + response.headers.forEach((value, key) => { + headers[key] = value; + }); + let body: unknown = text; + if (text.length > 0) { + try { + body = JSON.parse(text); + } catch { + // Leave as text — not JSON. + } + } else { + body = null; + } + pages.push({ + status: response.status, + headers, + body, + bodyText: text, + url: response.url || (typeof input === "string" ? input : ""), + }); + } catch { + // Capture is best-effort; never break the SDK if cloning fails. + } + return response; + }; + + globalThis.fetch = wrapped; + + return { + restore(): void { + if (globalThis.fetch === wrapped) { + globalThis.fetch = original; + } + }, + drain(): WireSnapshot { + const captured = pages; + pages = []; + return { pages: captured, pages_count: captured.length }; + }, + }; +} diff --git a/conformance/schema.json b/conformance/schema.json index befb5d9a..6215cce5 100644 --- a/conformance/schema.json +++ b/conformance/schema.json @@ -4,8 +4,30 @@ "title": "SDK Conformance Test", "description": "Schema for cross-language SDK conformance tests", "type": "object", - "required": ["name", "operation", "assertions"], + "required": ["name", "operation"], + "allOf": [ + { + "if": { "anyOf": [ + { "not": { "required": ["mode"] } }, + { "properties": { "mode": { "const": "mock" } } } + ] }, + "then": { "required": ["mockResponses", "assertions"] } + }, + { + "if": { "properties": { "mode": { "const": "live" } }, "required": ["mode"] }, + "then": { + "not": { "required": ["mockResponses"] }, + "required": ["liveAssertions"] + } + } + ], "properties": { + "mode": { + "type": "string", + "enum": ["mock", "live"], + "default": "mock", + "description": "Execution mode. 'mock' (default) registers MSW handlers and asserts SDK behavior offline. 'live' dispatches against a real Basecamp backend, captures raw wire bytes, and validates against the OpenAPI response schema." + }, "name": { "type": "string", "description": "Human-readable test name" @@ -141,6 +163,41 @@ } } }, + "liveAssertions": { + "type": "array", + "description": "Assertions evaluated when mode=live, against the raw wire response (not SDK-decoded structures).", + "items": { + "type": "object", + "required": ["type"], + "properties": { + "type": { + "type": "string", + "enum": [ + "liveCallSucceeds", + "liveResponseFieldsRequired", + "liveResponseFieldsExpected", + "liveSchemaValidate" + ], + "description": "liveCallSucceeds: dispatch returns without throwing, HTTP 2xx, JSON parses. liveResponseFieldsRequired: dotted paths that must be structurally present (per spec @required). liveResponseFieldsExpected: dotted paths that should be present; missing logs a warning, never fails. liveSchemaValidate: validate the raw wire body against the OpenAPI response schema for the operation." + }, + "fields": { + "type": "array", + "items": { "type": "string" }, + "description": "Dotted-path fields for liveResponseFieldsRequired / liveResponseFieldsExpected. Empty arrays count as present." + }, + "enabled": { + "type": "boolean", + "default": true, + "description": "Set false to disable a default-on assertion such as liveSchemaValidate." + } + } + } + }, + "fixtureIds": { + "type": "object", + "description": "Placeholder names referenced in pathParams/queryParams for live tests (e.g., PROJECT_ID, TODOLIST_ID). Resolved per the fixture-ID ladder: explicit env var → generic env var → discovery → skip.", + "additionalProperties": { "type": "string" } + }, "tags": { "type": "array", "description": "Tags for filtering tests (e.g., retry, pagination, idempotent)", diff --git a/conformance/tests/live-my-surface.json b/conformance/tests/live-my-surface.json new file mode 100644 index 00000000..1518f536 --- /dev/null +++ b/conformance/tests/live-my-surface.json @@ -0,0 +1,146 @@ +[ + { + "mode": "live", + "name": "ListProjects returns a list and validates against schema", + "description": "Smoke check + schema validation for the canonical list endpoint. Required fields are enforced per spec; new BC5-only fields surface as extras-observed.", + "operation": "ListProjects", + "method": "GET", + "path": "/projects.json", + "tags": ["live", "read-only"], + "liveAssertions": [ + { "type": "liveCallSucceeds" }, + { "type": "liveSchemaValidate" } + ] + }, + { + "mode": "live", + "name": "GetProject decodes the spec-required fields", + "description": "First-project fallback is acceptable; verifies Project @required fields are structurally present.", + "operation": "GetProject", + "method": "GET", + "path": "/projects/${PROJECT_ID}.json", + "fixtureIds": { "PROJECT_ID": "PROJECT_ID" }, + "pathParams": { "projectId": "${PROJECT_ID}" }, + "tags": ["live", "read-only"], + "liveAssertions": [ + { "type": "liveCallSucceeds" }, + { "type": "liveSchemaValidate" }, + { + "type": "liveResponseFieldsRequired", + "fields": ["id", "status", "created_at", "updated_at", "name", "url", "app_url"] + } + ] + }, + { + "mode": "live", + "name": "GetMyAssignments decodes priorities + non_priorities", + "description": "Verifies MyAssignment.bucket JSON-key compat — the field is preserved despite route renames.", + "operation": "GetMyAssignments", + "method": "GET", + "path": "/my/assignments.json", + "tags": ["live", "read-only"], + "liveAssertions": [ + { "type": "liveCallSucceeds" }, + { "type": "liveSchemaValidate" } + ] + }, + { + "mode": "live", + "name": "GetMyCompletedAssignments decodes the assignments list", + "operation": "GetMyCompletedAssignments", + "method": "GET", + "path": "/my/assignments/completed.json", + "tags": ["live", "read-only"], + "liveAssertions": [ + { "type": "liveCallSucceeds" }, + { "type": "liveSchemaValidate" } + ] + }, + { + "mode": "live", + "name": "GetMyDueAssignments decodes the assignments list", + "operation": "GetMyDueAssignments", + "method": "GET", + "path": "/my/assignments/due.json", + "tags": ["live", "read-only"], + "liveAssertions": [ + { "type": "liveCallSucceeds" }, + { "type": "liveSchemaValidate" } + ] + }, + { + "mode": "live", + "name": "GetMyNotifications decodes unreads/reads/memories/bubble_ups", + "description": "Verifies BC5 additions (bubble_ups, scheduled_bubble_ups) decode as arrays. Also exercises the memories[] observation that pairs with the COORDINATION.md decision.", + "operation": "GetMyNotifications", + "method": "GET", + "path": "/my/readings.json", + "tags": ["live", "read-only", "bc5-additive"], + "liveAssertions": [ + { "type": "liveCallSucceeds" }, + { "type": "liveSchemaValidate" } + ] + }, + { + "mode": "live", + "name": "GetMyProfile decodes the self-Person payload", + "description": "Self profile via people.me(). Exercises Person required fields and BC5 tagline alias.", + "operation": "GetMyProfile", + "method": "GET", + "path": "/my/profile.json", + "tags": ["live", "read-only", "bc5-additive"], + "liveAssertions": [ + { "type": "liveCallSucceeds" }, + { "type": "liveSchemaValidate" }, + { + "type": "liveResponseFieldsRequired", + "fields": ["id", "name"] + } + ] + }, + { + "mode": "live", + "name": "GetTodoset decodes the BC5 count + url additions", + "description": "Verifies todos_count, completed_loose_todos_count, todos_url, app_todos_url decode (when present — eligibility-gated like other BC5 additions).", + "operation": "GetTodoset", + "method": "GET", + "path": "/todosets/${TODOSET_ID}.json", + "fixtureIds": { "TODOSET_ID": "TODOSET_ID" }, + "pathParams": { "todosetId": "${TODOSET_ID}" }, + "tags": ["live", "read-only", "bc5-additive"], + "liveAssertions": [ + { "type": "liveCallSucceeds" }, + { "type": "liveSchemaValidate" } + ] + }, + { + "mode": "live", + "name": "ListTodolists returns a list under a todoset", + "description": "Single-arg dispatch via client.todolists.list(todosetId).", + "operation": "ListTodolists", + "method": "GET", + "path": "/todosets/${TODOSET_ID}/todolists.json", + "fixtureIds": { "TODOSET_ID": "TODOSET_ID" }, + "pathParams": { "todosetId": "${TODOSET_ID}" }, + "tags": ["live", "read-only"], + "liveAssertions": [ + { "type": "liveCallSucceeds" }, + { "type": "liveSchemaValidate" } + ] + }, + { + "mode": "live", + "name": "ListTodos exercises Todo.steps additive field", + "description": "When BC5 emits Todo.steps, the additive field decodes through the existing CardStep shape.", + "operation": "ListTodos", + "method": "GET", + "path": "/todolists/${TODOLIST_ID}/todos.json", + "fixtureIds": { "TODOLIST_ID": "TODOLIST_ID" }, + "pathParams": { "todolistId": "${TODOLIST_ID}" }, + "tags": ["live", "read-only", "bc5-additive"], + "liveAssertions": [ + { "type": "liveCallSucceeds" }, + { "type": "liveSchemaValidate" } + ] + } +] diff --git a/kotlin/conformance/src/main/kotlin/com/basecamp/sdk/conformance/Main.kt b/kotlin/conformance/src/main/kotlin/com/basecamp/sdk/conformance/Main.kt index d4045041..41f2710e 100644 --- a/kotlin/conformance/src/main/kotlin/com/basecamp/sdk/conformance/Main.kt +++ b/kotlin/conformance/src/main/kotlin/com/basecamp/sdk/conformance/Main.kt @@ -43,7 +43,12 @@ fun main() { var skipped = 0 for (file in testFiles) { + // Live tests are TS-only (canonical wire-capturer). Filter them out + // here so the offline Kotlin runner doesn't see live entries with + // unresolved ${PROJECT_ID} fixtures or unknown operations. val testCases = json.decodeFromString>(file.readText()) + .filter { it.mode == "mock" } + if (testCases.isEmpty()) continue println("\n=== ${file.name} ===") for (tc in testCases) { @@ -107,6 +112,12 @@ data class TestCase( val assertions: List = emptyList(), val tags: List = emptyList(), val configOverrides: ConfigOverrides? = null, + /** + * Execution mode. Defaults to "mock". Live tests are owned by the TS + * runner only (canonical wire-capturer); other-language runners filter + * them out at load time. + */ + val mode: String = "mock", ) @kotlinx.serialization.Serializable