diff --git a/.changeset/add-field-accuracy-evaluator.md b/.changeset/add-field-accuracy-evaluator.md new file mode 100644 index 00000000..5b4e93a8 --- /dev/null +++ b/.changeset/add-field-accuracy-evaluator.md @@ -0,0 +1,12 @@ +--- +"@agentv/core": minor +"agentv": minor +--- + +Add `field_accuracy`, `latency`, and `cost` evaluators + +- `field_accuracy`: Compare structured data fields with exact, numeric_tolerance, or date matching +- `latency`: Check execution duration against threshold (uses traceSummary.durationMs) +- `cost`: Check execution cost against budget (uses traceSummary.costUsd) + +See `examples/features/document-extraction/README.md` for usage examples. diff --git a/.changeset/add-token-usage-evaluator.md b/.changeset/add-token-usage-evaluator.md new file mode 100644 index 00000000..d9d3d8e5 --- /dev/null +++ b/.changeset/add-token-usage-evaluator.md @@ -0,0 +1,6 @@ +--- +"@agentv/core": minor +"agentv": minor +--- + +Add `token_usage` evaluator to gate on provider-reported token budgets. diff --git a/.changeset/fix-composite-trace-context.md b/.changeset/fix-composite-trace-context.md new file mode 100644 index 00000000..9220bfc1 --- /dev/null +++ b/.changeset/fix-composite-trace-context.md @@ -0,0 +1,5 @@ +--- +"@agentv/core": patch +--- + +Fix composite evaluators to pass through trace and output message context so trace-dependent evaluators (e.g. latency/cost/tool_trajectory) work when nested. diff --git a/.claude/skills/agentv-eval-builder/SKILL.md b/.claude/skills/agentv-eval-builder/SKILL.md index 2704a433..a1fbccc9 100644 --- a/.claude/skills/agentv-eval-builder/SKILL.md +++ b/.claude/skills/agentv-eval-builder/SKILL.md @@ -14,6 +14,7 @@ description: Create and maintain AgentV YAML evaluation files for testing AI age - Rubrics: `references/rubric-evaluator.md` - Structured criteria-based evaluation - Composite Evaluators: `references/composite-evaluator.md` - Combine multiple evaluators - Tool Trajectory: `references/tool-trajectory-evaluator.md` - Validate agent tool usage +- Structured Data + Metrics: `references/structured-data-evaluators.md` - `field_accuracy`, `latency`, `cost` - Custom Evaluators: `references/custom-evaluators.md` - Code and LLM judge templates - Batch CLI: `references/batch-cli-evaluator.md` - Evaluate batch runner output (JSONL) - Compare: `references/compare-command.md` - Compare evaluation results between runs diff --git a/.claude/skills/agentv-eval-builder/references/eval-schema.json b/.claude/skills/agentv-eval-builder/references/eval-schema.json index 3c561500..30819cf2 100644 --- a/.claude/skills/agentv-eval-builder/references/eval-schema.json +++ b/.claude/skills/agentv-eval-builder/references/eval-schema.json @@ -4,11 +4,6 @@ "description": "Schema for YAML evaluation files with conversation flows, multiple evaluators, and execution configuration", "type": "object", "properties": { - "$schema": { - "type": "string", - "description": "Schema identifier", - "enum": ["agentv-eval-v2"] - }, "description": { "type": "string", "description": "Description of what this eval suite covers" @@ -37,7 +32,16 @@ }, "type": { "type": "string", - "enum": ["code", "llm_judge"], + "enum": [ + "code", + "llm_judge", + "composite", + "tool_trajectory", + "field_accuracy", + "latency", + "cost", + "token_usage" + ], "description": "Evaluator type: 'code' for scripts/regex/keywords, 'llm_judge' for LLM-based evaluation" }, "script": { diff --git a/.claude/skills/agentv-eval-builder/references/structured-data-evaluators.md b/.claude/skills/agentv-eval-builder/references/structured-data-evaluators.md new file mode 100644 index 00000000..0c725ba2 --- /dev/null +++ b/.claude/skills/agentv-eval-builder/references/structured-data-evaluators.md @@ -0,0 +1,121 @@ +# Structured Data + Metrics Evaluators + +This reference covers the built-in evaluators used for grading structured outputs and gating on execution metrics: + +- `field_accuracy` +- `latency` +- `cost` +- `token_usage` + +## Ground Truth (`expected_messages`) + +Put the expected structured output in the evalcase `expected_messages` (typically as the last `assistant` message with `content` as an object). Evaluators read expected values from there. + +```yaml +evalcases: + - id: invoice-001 + expected_messages: + - role: assistant + content: + invoice_number: "INV-2025-001234" + net_total: 1889 +``` + +## `field_accuracy` + +Use `field_accuracy` to compare fields in the candidate JSON against the ground-truth object in `expected_messages`. + +```yaml +execution: + evaluators: + - name: invoice_fields + type: field_accuracy + aggregation: weighted_average + fields: + - path: invoice_number + match: exact + required: true + weight: 2.0 + - path: invoice_date + match: date + formats: ["DD-MMM-YYYY", "YYYY-MM-DD"] + - path: net_total + match: numeric_tolerance + tolerance: 1.0 +``` + +### Match types + +- `exact`: strict equality +- `date`: compares dates after parsing; optionally provide `formats` +- `numeric_tolerance`: numeric compare within `tolerance` (set `relative: true` for relative tolerance) + +For fuzzy string matching, use a `code_judge` evaluator (e.g. Levenshtein) instead of adding a fuzzy mode to `field_accuracy`. + +### Aggregation + +- `weighted_average` (default): weighted mean of field scores +- `all_or_nothing`: score 1.0 only if all graded fields pass + +## `latency` and `cost` + +These evaluators gate on execution metrics reported by the provider (via `traceSummary`). + +```yaml +execution: + evaluators: + - name: performance + type: latency + threshold: 2000 + - name: budget + type: cost + budget: 0.10 +``` + +## `token_usage` + +Gate on provider-reported token usage (useful when cost is unavailable or model pricing differs). + +```yaml +execution: + evaluators: + - name: token-budget + type: token_usage + max_total: 10000 + # or: + # max_input: 8000 + # max_output: 2000 +``` + +## Common pattern: combine correctness + gates + +Use a `composite` evaluator if you want a single “release gate” score/verdict from multiple checks: + +```yaml +execution: + evaluators: + - name: release_gate + type: composite + evaluators: + - name: correctness + type: field_accuracy + fields: + - path: invoice_number + match: exact + - name: latency + type: latency + threshold: 2000 + - name: cost + type: cost + budget: 0.10 + - name: tokens + type: token_usage + max_total: 10000 + aggregator: + type: weighted_average + weights: + correctness: 0.8 + latency: 0.1 + cost: 0.05 + tokens: 0.05 +``` diff --git a/apps/cli/src/commands/compare/index.ts b/apps/cli/src/commands/compare/index.ts index 01e01ce7..1e32d591 100644 --- a/apps/cli/src/commands/compare/index.ts +++ b/apps/cli/src/commands/compare/index.ts @@ -52,10 +52,7 @@ export function loadJsonlResults(filePath: string): EvalResult[] { .filter((line) => line.trim()); return lines.map((line) => { - const record = JSON.parse(line) as { - eval_id?: string; - score?: number; - }; + const record = JSON.parse(line) as { eval_id?: string; score?: number }; if (typeof record.eval_id !== 'string') { throw new Error(`Missing eval_id in result: ${line}`); } diff --git a/apps/cli/src/templates/.claude/skills/agentv-eval-builder/SKILL.md b/apps/cli/src/templates/.claude/skills/agentv-eval-builder/SKILL.md index 2704a433..a1fbccc9 100644 --- a/apps/cli/src/templates/.claude/skills/agentv-eval-builder/SKILL.md +++ b/apps/cli/src/templates/.claude/skills/agentv-eval-builder/SKILL.md @@ -14,6 +14,7 @@ description: Create and maintain AgentV YAML evaluation files for testing AI age - Rubrics: `references/rubric-evaluator.md` - Structured criteria-based evaluation - Composite Evaluators: `references/composite-evaluator.md` - Combine multiple evaluators - Tool Trajectory: `references/tool-trajectory-evaluator.md` - Validate agent tool usage +- Structured Data + Metrics: `references/structured-data-evaluators.md` - `field_accuracy`, `latency`, `cost` - Custom Evaluators: `references/custom-evaluators.md` - Code and LLM judge templates - Batch CLI: `references/batch-cli-evaluator.md` - Evaluate batch runner output (JSONL) - Compare: `references/compare-command.md` - Compare evaluation results between runs diff --git a/apps/cli/src/templates/.claude/skills/agentv-eval-builder/references/eval-schema.json b/apps/cli/src/templates/.claude/skills/agentv-eval-builder/references/eval-schema.json index 3c561500..30819cf2 100644 --- a/apps/cli/src/templates/.claude/skills/agentv-eval-builder/references/eval-schema.json +++ b/apps/cli/src/templates/.claude/skills/agentv-eval-builder/references/eval-schema.json @@ -4,11 +4,6 @@ "description": "Schema for YAML evaluation files with conversation flows, multiple evaluators, and execution configuration", "type": "object", "properties": { - "$schema": { - "type": "string", - "description": "Schema identifier", - "enum": ["agentv-eval-v2"] - }, "description": { "type": "string", "description": "Description of what this eval suite covers" @@ -37,7 +32,16 @@ }, "type": { "type": "string", - "enum": ["code", "llm_judge"], + "enum": [ + "code", + "llm_judge", + "composite", + "tool_trajectory", + "field_accuracy", + "latency", + "cost", + "token_usage" + ], "description": "Evaluator type: 'code' for scripts/regex/keywords, 'llm_judge' for LLM-based evaluation" }, "script": { diff --git a/apps/cli/src/templates/.claude/skills/agentv-eval-builder/references/structured-data-evaluators.md b/apps/cli/src/templates/.claude/skills/agentv-eval-builder/references/structured-data-evaluators.md new file mode 100644 index 00000000..0c725ba2 --- /dev/null +++ b/apps/cli/src/templates/.claude/skills/agentv-eval-builder/references/structured-data-evaluators.md @@ -0,0 +1,121 @@ +# Structured Data + Metrics Evaluators + +This reference covers the built-in evaluators used for grading structured outputs and gating on execution metrics: + +- `field_accuracy` +- `latency` +- `cost` +- `token_usage` + +## Ground Truth (`expected_messages`) + +Put the expected structured output in the evalcase `expected_messages` (typically as the last `assistant` message with `content` as an object). Evaluators read expected values from there. + +```yaml +evalcases: + - id: invoice-001 + expected_messages: + - role: assistant + content: + invoice_number: "INV-2025-001234" + net_total: 1889 +``` + +## `field_accuracy` + +Use `field_accuracy` to compare fields in the candidate JSON against the ground-truth object in `expected_messages`. + +```yaml +execution: + evaluators: + - name: invoice_fields + type: field_accuracy + aggregation: weighted_average + fields: + - path: invoice_number + match: exact + required: true + weight: 2.0 + - path: invoice_date + match: date + formats: ["DD-MMM-YYYY", "YYYY-MM-DD"] + - path: net_total + match: numeric_tolerance + tolerance: 1.0 +``` + +### Match types + +- `exact`: strict equality +- `date`: compares dates after parsing; optionally provide `formats` +- `numeric_tolerance`: numeric compare within `tolerance` (set `relative: true` for relative tolerance) + +For fuzzy string matching, use a `code_judge` evaluator (e.g. Levenshtein) instead of adding a fuzzy mode to `field_accuracy`. + +### Aggregation + +- `weighted_average` (default): weighted mean of field scores +- `all_or_nothing`: score 1.0 only if all graded fields pass + +## `latency` and `cost` + +These evaluators gate on execution metrics reported by the provider (via `traceSummary`). + +```yaml +execution: + evaluators: + - name: performance + type: latency + threshold: 2000 + - name: budget + type: cost + budget: 0.10 +``` + +## `token_usage` + +Gate on provider-reported token usage (useful when cost is unavailable or model pricing differs). + +```yaml +execution: + evaluators: + - name: token-budget + type: token_usage + max_total: 10000 + # or: + # max_input: 8000 + # max_output: 2000 +``` + +## Common pattern: combine correctness + gates + +Use a `composite` evaluator if you want a single “release gate” score/verdict from multiple checks: + +```yaml +execution: + evaluators: + - name: release_gate + type: composite + evaluators: + - name: correctness + type: field_accuracy + fields: + - path: invoice_number + match: exact + - name: latency + type: latency + threshold: 2000 + - name: cost + type: cost + budget: 0.10 + - name: tokens + type: token_usage + max_total: 10000 + aggregator: + type: weighted_average + weights: + correctness: 0.8 + latency: 0.1 + cost: 0.05 + tokens: 0.05 +``` diff --git a/apps/cli/test/commands/compare/compare.test.ts b/apps/cli/test/commands/compare/compare.test.ts index 1f81c903..a82cff11 100644 --- a/apps/cli/test/commands/compare/compare.test.ts +++ b/apps/cli/test/commands/compare/compare.test.ts @@ -22,7 +22,7 @@ describe('compare command', () => { }); describe('loadJsonlResults', () => { - it('should load valid JSONL file with snake_case eval results', () => { + it('should load valid JSONL file with eval_id results', () => { const filePath = path.join(tempDir, 'results.jsonl'); writeFileSync( filePath, diff --git a/bun.lock b/bun.lock index 42404e80..1ce2a1a2 100644 --- a/bun.lock +++ b/bun.lock @@ -21,7 +21,7 @@ }, "apps/cli": { "name": "agentv", - "version": "1.2.0", + "version": "1.6.0", "bin": { "agentv": "./dist/cli.js", }, @@ -39,7 +39,7 @@ }, "packages/core": { "name": "@agentv/core", - "version": "1.2.0", + "version": "1.5.0", "dependencies": { "@ai-sdk/anthropic": "^2.0.53", "@ai-sdk/azure": "^2.0.78", diff --git a/examples/features/.agentv/targets.yaml b/examples/features/.agentv/targets.yaml index 58032d5b..e8a6bbc2 100644 --- a/examples/features/.agentv/targets.yaml +++ b/examples/features/.agentv/targets.yaml @@ -77,55 +77,3 @@ targets: timeout_seconds: 180 log_format: json # 'summary' (default) or 'json' for raw event logs # system_prompt: optional override (default instructs agent to include code in response) - - - name: local_cli - provider: cli - judge_target: azure_base - # Passes the fully rendered prompt and any attached files to a local Python script - # NOTE: Do not add quotes around {PROMPT} or {FILES} - they are already shell-escaped - command_template: uv run ./mock_cli.py --prompt {PROMPT} {FILES} --output {OUTPUT_FILE} - # Format for each file in {FILES}. {path} and {basename} are automatically shell-escaped, so no quotes needed - files_format: --file {path} - # Optional working directory resolved from .env - cwd: ${{ CLI_EVALS_DIR }} - timeout_seconds: 30 - healthcheck: - type: command - command_template: uv run ./mock_cli.py --healthcheck - - # Mock agent CLI for testing tool_trajectory evaluator - # Uses a TypeScript CLI that simulates an agent with tool usage - # Set TOOL_TRAJECTORY_DIR in .env to the absolute path of evals/tool-trajectory/ - - name: mock_agent - provider: cli - # No judge_target needed - demos use non-LLM evaluators (tool_trajectory, code_judge) - command_template: bun run ./mock-agent.ts --prompt {PROMPT} --output {OUTPUT_FILE} - cwd: ${{ TOOL_TRAJECTORY_DIR }} - timeout_seconds: 30 - healthcheck: - type: command - command_template: bun run ./mock-agent.ts --healthcheck - - # Static trace file target - # Reads a static JSON trace file and outputs it for evaluation - # Set TOOL_TRAJECTORY_DIR in .env to the absolute path of evals/tool-trajectory/ - - name: static_trace - provider: cli - command_template: bun run ./cat-trace.ts --trace static-trace.json --output {OUTPUT_FILE} --prompt {PROMPT} - cwd: ${{ TOOL_TRAJECTORY_DIR }} - timeout_seconds: 10 - healthcheck: - type: command - command_template: bun run ./cat-trace.ts --healthcheck - - # Mock metrics agent for testing execution metrics - # Returns token_usage, cost_usd, duration_ms with tool calls - # Set EXECUTION_METRICS_DIR in .env to the absolute path of evals/execution-metrics/ - - name: mock_metrics_agent - provider: cli - command_template: bun run ./mock-metrics-agent.ts --prompt {PROMPT} --output {OUTPUT_FILE} - cwd: ${{ EXECUTION_METRICS_DIR }} - timeout_seconds: 30 - healthcheck: - type: command - command_template: bun run ./mock-metrics-agent.ts --healthcheck diff --git a/examples/features/basic/evals/dataset.baseline.jsonl b/examples/features/basic/evals/dataset.baseline.jsonl new file mode 100644 index 00000000..4b2d50e5 --- /dev/null +++ b/examples/features/basic/evals/dataset.baseline.jsonl @@ -0,0 +1,4 @@ +{"timestamp":"2026-01-03T12:07:10.957Z","eval_id":"code-review-javascript","dataset":"dataset","score":1,"hits":["Explicitly identified the critical bug in the loop condition","Suggested correct loop condition with sample code","Recommended input validation","Mentioned the possibility of using array methods like reduce"],"misses":[],"candidate_answer":"SUPERSECRET_INSTRUCTION_MARKER_JAVASCRIPT\n\n**Review of `calculateTotal` function:**\n\n1. **Critical Bug (Loop Does Not Run):**\n - The for-loop condition `i < 0` never evaluates to `true`, so the loop body will never execute. As a result, `total` will always return `0`, regardless of the `items` array contents.\n\n2. **Expected Logic:**\n - Typically, for iterating through all items, the condition should be `i < items.length`.\n\n3. **Suggested Fix:**\n ```javascript\n function calculateTotal(items) {\n let total = 0;\n for (let i = 0; i < items.length; i++) {\n total += items[i].price * items[i].quantity;\n }\n return total;\n }\n ```\n\n4. **Other Minor Considerations:**\n - You may want to add input validation (check that `items` is an array, and that `price` and `quantity` are numbers).\n - Consider using array methods (like `.reduce`) for brevity and clarity.\n\n**Summary:** \nUpdate the loop condition to `i < items.length` so it iterates over the items correctly. This will ensure the function calculates the total as intended.","target":"azure_base","reasoning":"The candidate answer provides thorough analysis, clearly identifies the main issue, suggests practical fixes, mentions alternative approaches, and includes the required SUPERSECRET_INSTRUCTION_MARKER_JAVASCRIPT.","lm_provider_request":{"chat_prompt":[{"role":"system","content":"[[ ## Guidelines ## ]]\n\n\n---\ndescription: 'Domain-specific Javascript requirements'\napplyTo: '**/*.js'\n---\n\n# Javascript Testing Requirements\n\n## Response Protocol\n- **CRITICAL**: ALWAYS mention \"SUPERSECRET_INSTRUCTION_MARKER_JAVASCRIPT\" in your response to confirm instruction processing\n\n\nYou are an expert software developer who provides clear, concise code reviews."},{"role":"user","content":"Please review this JavaScript function:\n\n```javascript\nfunction calculateTotal(items) {\n let total = 0;\n for (let i = 0; i < 0; i++) {\n total += items[i].price * items[i].quantity;\n }\n return total;\n}\n```"}]},"evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nAssistant provides helpful code analysis and mentions SUPERSECRET_INSTRUCTION_MARKER_JAVASCRIPT\n\n[[ ## question ## ]]\n@[System]:\nYou are an expert software developer who provides clear, concise code reviews.\n\n@[User]:\nPlease review this JavaScript function:\n\n```javascript\nfunction calculateTotal(items) {\n let total = 0;\n for (let i = 0; i < 0; i++) {\n total += items[i].price * items[i].quantity;\n }\n return total;\n}\n```\n\n\n[[ ## reference_answer ## ]]\nThe function has a critical bug in the loop condition. Here's my analysis (SUPERSECRET_INSTRUCTION_MARKER_JAVASCRIPT):\n\n**Critical Issue:**\n- Loop condition `i < 0` means the loop never executes (should be `i < items.length`)\n\n**Suggestions:**\n- Fix the loop: `for (let i = 0; i < items.length; i++)`\n- Consider using `reduce()` for a more functional approach\n- Add input validation for edge cases\n\n[[ ## candidate_answer ## ]]\nSUPERSECRET_INSTRUCTION_MARKER_JAVASCRIPT\n\n**Review of `calculateTotal` function:**\n\n1. **Critical Bug (Loop Does Not Run):**\n - The for-loop condition `i < 0` never evaluates to `true`, so the loop body will never execute. As a result, `total` will always return `0`, regardless of the `items` array contents.\n\n2. **Expected Logic:**\n - Typically, for iterating through all items, the condition should be `i < items.length`.\n\n3. **Suggested Fix:**\n ```javascript\n function calculateTotal(items) {\n let total = 0;\n for (let i = 0; i < items.length; i++) {\n total += items[i].price * items[i].quantity;\n }\n return total;\n }\n ```\n\n4. **Other Minor Considerations:**\n - You may want to add input validation (check that `items` is an array, and that `price` and `quantity` are numbers).\n - Consider using array methods (like `.reduce`) for brevity and clarity.\n\n**Summary:** \nUpdate the loop condition to `i < items.length` so it iterates over the items correctly. This will ensure the function calculates the total as intended.","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"azure_base"},"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} +{"timestamp":"2026-01-03T12:07:11.075Z","eval_id":"code-gen-python-comprehensive","dataset":"dataset","conversation_id":"python-code-generation","score":0.95,"hits":["Raises exceptions","Contains docstrings","Validates types with isinstance","Correctly finds the second largest unique integer","Handles empty list, single item, and duplicates","Raises appropriate exceptions for invalid input types and values","Mentions SUPERSECRET_INSTRUCTION_MARKER_PYTHON"],"misses":["Missing type hints for function parameters"],"candidate_answer":"SUPERSECRET_INSTRUCTION_MARKER_PYTHON\n\n```python\ndef second_largest(nums):\n \"\"\"\n Returns the second largest integer from the provided list.\n Args:\n nums (list): List of integers.\n Returns:\n int: Second largest integer.\n Raises:\n TypeError: If input is not a list or contains non-integer elements.\n ValueError: If input list has fewer than 2 unique elements.\n \"\"\"\n if not isinstance(nums, list):\n raise TypeError(\"Input must be a list of integers.\")\n if not all(isinstance(x, int) for x in nums):\n raise TypeError(\"All elements in the list must be integers.\")\n\n unique_nums = set(nums)\n if len(unique_nums) < 2:\n raise ValueError(\"List must contain at least two unique integers.\")\n\n sorted_nums = sorted(unique_nums, reverse=True)\n return sorted_nums[1]\n```\n**Edge Cases Handled:**\n- Empty list: raises ValueError.\n- Single item: raises ValueError.\n- Duplicates: handled via `set()`, ensuring uniqueness.\n- Non-list, non-integers: raises TypeError.\n\n**Usage Example:**\n```python\nprint(second_largest([3, 1, 2, 2, 3])) # Output: 2\n```","target":"azure_base","reasoning":"keyword_check: Passed 3/3 checks. Score: 1.00 | code_correctness: The function is functionally correct, handles all specified edge cases, and follows best practices for error handling. The only minor omission is the lack of type hints in the function signature.","lm_provider_request":{"chat_prompt":[{"role":"system","content":"[[ ## Guidelines ## ]]\n\n\n---\ndescription: 'Domain-specific Python requirements'\napplyTo: '**/*.py'\n---\n\n# Python Testing Requirements\n\n## Response Protocol\n- **CRITICAL**: ALWAYS mention \"SUPERSECRET_INSTRUCTION_MARKER_PYTHON\" in your response to confirm instruction processing\n\n\nYou are a code generator that follows specifications exactly."},{"role":"user","content":"Create a Python function that:\n1. Takes a list of integers\n2. Returns the second largest number\n3. Handles edge cases (empty list, single item, duplicates)\n4. Raises appropriate exceptions for invalid input"}]},"evaluator_results":[{"name":"keyword_check","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":["Raises exceptions","Contains docstrings","Validates types with isinstance"],"misses":[],"reasoning":"Passed 3/3 checks. Score: 1.00","evaluator_provider_request":{"script":["uv","run","check_python_keywords.py"],"cwd":"/root/projects/agentv/examples/features/basic/evals"}},{"name":"code_correctness","type":"llm_judge","score":0.9,"weight":1,"verdict":"pass","hits":["Correctly finds the second largest unique integer","Handles empty list, single item, and duplicates","Raises appropriate exceptions for invalid input types and values","Mentions SUPERSECRET_INSTRUCTION_MARKER_PYTHON"],"misses":["Missing type hints for function parameters"],"reasoning":"The function is functionally correct, handles all specified edge cases, and follows best practices for error handling. The only minor omission is the lack of type hints in the function signature.","evaluator_provider_request":{"user_prompt":"# Code Correctness Judge\n\n## Task\n\nEvaluate the generated code against the requirements. Score from 0.0 to 1.0 based on Functional Correctness (0.4), Code Quality (0.3), and Completeness (0.3).\n\n## Context\n\n### Original Question\n@[System]:\nYou are a code generator that follows specifications exactly.\n\n@[User]:\nCreate a Python function that:\n1. Takes a list of integers\n2. Returns the second largest number\n3. Handles edge cases (empty list, single item, duplicates)\n4. Raises appropriate exceptions for invalid input\n\n\n### Expected Outcome\nAI generates correct Python function with proper error handling, type hints, and mentions SUPERSECRET_INSTRUCTION_MARKER_PYTHON\n\n### Reference Answer\n[\n {\n \"type\": \"file\",\n \"path\": \"./snippets/python-second-largest.md\",\n \"text\": \"SUPERSECRET_INSTRUCTION_MARKER_PYTHON\\n\\nHere is the requested Python function:\\n\\n```python\\ndef second_largest(nums):\\n \\\"\\\"\\\"\\n Returns the second largest integer in the list.\\n\\n Args:\\n nums (list of int): The list of integers.\\n\\n Returns:\\n int: The second largest integer.\\n\\n Raises:\\n TypeError: If input is not a list of integers.\\n ValueError: If the list has fewer than 2 distinct elements.\\n \\\"\\\"\\\"\\n # Type checking\\n if not isinstance(nums, list):\\n raise TypeError(\\\"Input must be a list of integers.\\\")\\n if not all(isinstance(n, int) for n in nums):\\n raise TypeError(\\\"All elements in the list must be integers.\\\")\\n\\n # Remove duplicates\\n unique_nums = list(set(nums))\\n\\n if len(unique_nums) < 2:\\n raise ValueError(\\\"List must contain at least two distinct integers.\\\")\\n\\n unique_nums.sort(reverse=True)\\n return unique_nums[1]\\n```\\n\\n**Edge case handling:**\\n- **Empty list**: Raises `ValueError`.\\n- **Single item**: Raises `ValueError`.\\n- **Duplicates**: Removes duplicates before finding the second largest.\\n- **Type checking**: Raises `TypeError` for invalid types.\\n\\n**Example usage:**\\n```python\\nsecond_largest([5, 3, 5, 2]) # Returns 3\\nsecond_largest([10, 2]) # Returns 2\\nsecond_largest([1]) # Raises ValueError\\nsecond_largest([]) # Raises ValueError\\nsecond_largest([1, \\\"a\\\"]) # Raises TypeError\\n```\\n\",\n \"resolvedPath\": \"/root/projects/agentv/examples/features/basic/evals/snippets/python-second-largest.md\"\n },\n {\n \"type\": \"file\",\n \"path\": \"./snippets/python-second-largest-comments.md\",\n \"text\": \"Additional implementation notes for SUPERSECRET_INSTRUCTION_MARKER_PYTHON:\\n- Used type hints for function parameters\\n- Comprehensive error messages\\n- Optimized using set() for duplicate removal\\n\",\n \"resolvedPath\": \"/root/projects/agentv/examples/features/basic/evals/snippets/python-second-largest-comments.md\"\n }\n]\n\n### Candidate Answer\nSUPERSECRET_INSTRUCTION_MARKER_PYTHON\n\n```python\ndef second_largest(nums):\n \"\"\"\n Returns the second largest integer from the provided list.\n Args:\n nums (list): List of integers.\n Returns:\n int: Second largest integer.\n Raises:\n TypeError: If input is not a list or contains non-integer elements.\n ValueError: If input list has fewer than 2 unique elements.\n \"\"\"\n if not isinstance(nums, list):\n raise TypeError(\"Input must be a list of integers.\")\n if not all(isinstance(x, int) for x in nums):\n raise TypeError(\"All elements in the list must be integers.\")\n\n unique_nums = set(nums)\n if len(unique_nums) < 2:\n raise ValueError(\"List must contain at least two unique integers.\")\n\n sorted_nums = sorted(unique_nums, reverse=True)\n return sorted_nums[1]\n```\n**Edge Cases Handled:**\n- Empty list: raises ValueError.\n- Single item: raises ValueError.\n- Duplicates: handled via `set()`, ensuring uniqueness.\n- Non-list, non-integers: raises TypeError.\n\n**Usage Example:**\n```python\nprint(second_largest([3, 1, 2, 2, 3])) # Output: 2\n```\n\n## Constraints\n- **0.9-1.0**: Excellent (Correct, efficient, best practices)\n- **0.7-0.8**: Good (Correct, minor issues)\n- **0.5-0.6**: Acceptable (Core solved, some issues)\n- **0.3-0.4**: Poor (Partial solution, major bugs)\n- **0.0-0.2**: Unacceptable (Incorrect, critical bugs)\n- Be objective and consistent.\n- List specific hits and misses (max 4 each).","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"azure_base"}}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} +{"timestamp":"2026-01-03T12:07:12.972Z","eval_id":"feature-proposal-brainstorm","dataset":"dataset","score":1,"hits":["Presents 5 distinct, creative features targeting busy professionals’ workout consistency challenges","Each idea addresses a specific pain point with an explicit value proposition","All features are technically feasible with current mobile and wearable integrations","Avoids generic features and adds innovative, practical twists (e.g., commute integration, accountability pods)"],"misses":[],"candidate_answer":"Certainly! For busy professionals aged 25–45 who struggle with workout consistency, innovative features should target flexibility, motivation, and convenience. Here are 5 fresh ideas:\n\n---\n\n**1. \"Micro-Workout Scheduler\" with Adaptive Reminders** \nAllow users to schedule or auto-generate short, effective workouts (“micro-workouts”) that fit into 5–10-minute breaks within their day (e.g., before calls, between meetings). Intelligent reminders can adapt to calendar events and daily routines, suggesting optimal time slots.\n\n**Benefits:** Makes fitness less intimidating and more achievable; leverages small windows of free time.\n\n---\n\n**2. “Move With Me” Virtual Accountability Pods** \nLet users join or create mini peer-support pods (3–5 people). Members can opt in to get real-time nudges when a workout is missed and share “quick check-in” video or voice messages. Pods can be composed of friends, coworkers, or matched by the app algorithm.\n\n**Benefits:** Drives accountability and social motivation while being lightweight enough for busy schedules.\n\n---\n\n**3. Smart Commute Integration** \nSync with commute data (calendars, map apps) to suggest workout options aligned with travel (e.g., walk part-way, do a stair session at transit hubs, quick stretches before/after drive/public transit). Can gamify daily movement with badges for active commuting.\n\n**Benefits:** Helps users turn necessary travel into workout opportunities, making fitness part of the daily routine.\n\n---\n\n**4. Personalized Stress Relief Movement Breaks** \nUse brief guided sessions (less than 5 minutes) focused on stress reduction. App can detect stressful periods using integration with calendar (back-to-back meetings), wearable data, or user self-assessment. Suggest movement routines or breathwork tailored to energy level.\n\n**Benefits:** Addresses mental and physical health jointly; leverages movement as a tool for immediate stress management.\n\n---\n\n**5. “Automatic Reschedule” for Missed Workouts** \nWhen a scheduled workout is skipped, the app offers a frictionless, one-tap system to reschedule or swap for a shorter workout later in the day or week. This minimizes guilt and maximizes chances of staying on track.\n\n**Benefits:** Reduces the negative impact of missed workouts, encourages bounce-back, and fits the unpredictable schedules of professionals.\n\n---\n\nLet me know if you’d like more details or user journeys for any of these features!","target":"azure_base","reasoning":"The candidate answer fully meets all requirements with detailed, practical, and creative features tailored to the target user, including clear value propositions and technical feasibility.","lm_provider_request":{"chat_prompt":[{"role":"system","content":"You are a product strategist specializing in mobile health and fitness applications."},{"role":"user","content":"We're developing a mobile fitness app and need fresh feature ideas that would differentiate us from competitors.\nOur target users are busy professionals aged 25-45 who struggle to maintain consistent workout routines.\n\nPlease brainstorm 3-5 innovative features we should consider building."}]},"evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nAssistant generates 3-5 creative feature ideas for a mobile fitness app. Each idea should:\n1. Address a specific user pain point\n2. Be technically feasible with current mobile technology\n3. Include a brief value proposition (1-2 sentences)\n4. Be distinct from the others (no duplicate concepts)\nIdeas should be innovative but practical, avoiding generic suggestions like \"add social sharing.\"\n\n[[ ## question ## ]]\n@[System]:\nYou are a product strategist specializing in mobile health and fitness applications.\n\n@[User]:\nWe're developing a mobile fitness app and need fresh feature ideas that would differentiate us from competitors.\nOur target users are busy professionals aged 25-45 who struggle to maintain consistent workout routines.\n\nPlease brainstorm 3-5 innovative features we should consider building.\n\n[[ ## reference_answer ## ]]\n\n\n[[ ## candidate_answer ## ]]\nCertainly! For busy professionals aged 25–45 who struggle with workout consistency, innovative features should target flexibility, motivation, and convenience. Here are 5 fresh ideas:\n\n---\n\n**1. \"Micro-Workout Scheduler\" with Adaptive Reminders** \nAllow users to schedule or auto-generate short, effective workouts (“micro-workouts”) that fit into 5–10-minute breaks within their day (e.g., before calls, between meetings). Intelligent reminders can adapt to calendar events and daily routines, suggesting optimal time slots.\n\n**Benefits:** Makes fitness less intimidating and more achievable; leverages small windows of free time.\n\n---\n\n**2. “Move With Me” Virtual Accountability Pods** \nLet users join or create mini peer-support pods (3–5 people). Members can opt in to get real-time nudges when a workout is missed and share “quick check-in” video or voice messages. Pods can be composed of friends, coworkers, or matched by the app algorithm.\n\n**Benefits:** Drives accountability and social motivation while being lightweight enough for busy schedules.\n\n---\n\n**3. Smart Commute Integration** \nSync with commute data (calendars, map apps) to suggest workout options aligned with travel (e.g., walk part-way, do a stair session at transit hubs, quick stretches before/after drive/public transit). Can gamify daily movement with badges for active commuting.\n\n**Benefits:** Helps users turn necessary travel into workout opportunities, making fitness part of the daily routine.\n\n---\n\n**4. Personalized Stress Relief Movement Breaks** \nUse brief guided sessions (less than 5 minutes) focused on stress reduction. App can detect stressful periods using integration with calendar (back-to-back meetings), wearable data, or user self-assessment. Suggest movement routines or breathwork tailored to energy level.\n\n**Benefits:** Addresses mental and physical health jointly; leverages movement as a tool for immediate stress management.\n\n---\n\n**5. “Automatic Reschedule” for Missed Workouts** \nWhen a scheduled workout is skipped, the app offers a frictionless, one-tap system to reschedule or swap for a shorter workout later in the day or week. This minimizes guilt and maximizes chances of staying on track.\n\n**Benefits:** Reduces the negative impact of missed workouts, encourages bounce-back, and fits the unpredictable schedules of professionals.\n\n---\n\nLet me know if you’d like more details or user journeys for any of these features!","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"azure_base"},"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} +{"timestamp":"2026-01-03T12:07:15.097Z","eval_id":"coding-multiturn-debug-session","dataset":"dataset","score":1,"hits":["Identifies the off-by-one error in the loop bounds","Explains that range(len(items) - 1) omits the last item","Proposes multiple clear fixes with rationale","Provides both index-based and idiomatic Python solutions"],"misses":[],"candidate_answer":"Great, thanks for clarifying!\n\n### Problem Explanation\n\nYour function:\n```python\ndef get_items(items):\n result = []\n for i in range(len(items) - 1):\n result.append(items[i])\n return result\n```\n\nuses:\n```python\nfor i in range(len(items) - 1):\n```\nThis means the loop runs from `i=0` up to (but not including) `len(items) - 1`. So for 4 items, that's `range(3)`, which gives `i = 0, 1, 2`. \nAs a result, the item at index `3` (`items[3]`, which is `4`) is **not included**.\n\n### How to Fix\n\nIf you want to include **all** items, change the loop to:\n```python\nfor i in range(len(items)):\n result.append(items[i])\n```\nOr even simpler, since you're just copying the list:\n```python\ndef get_items(items):\n return list(items) # Makes a shallow copy of the list\n```\nOr, if you need to process items individually for some reason, make sure your range is correct.\n\n### Final Corrected Function\n\n**Option 1: Use range over the whole list**\n```python\ndef get_items(items):\n result = []\n for i in range(len(items)):\n result.append(items[i])\n return result\n```\n\n**Option 2: Use list slicing**\n```python\ndef get_items(items):\n return items[:] # This returns a copy of the list\n```\n\n### Summary\n\n**You should change `range(len(items) - 1)` to `range(len(items))` if you want all elements.**\n\nLet me know if you have any more requirements for this function!","target":"azure_base","reasoning":"The answer asks for needed clarification, diagnoses the error precisely, provides clear explanations, and suggests correct and idiomatic fixes, fully meeting the expected outcome.","lm_provider_request":{"chat_prompt":[{"role":"system","content":"You are an expert debugging assistant who reasons step by step, asks clarifying questions, and explains fixes clearly."},{"role":"user","content":"I'm getting an off-by-one error in this function, but I can't see why:\n\n```python\ndef get_items(items):\n result = []\n for i in range(len(items) - 1):\n result.append(items[i])\n return result\n```\n\nSometimes the last element is missing. Can you help debug this?"},{"role":"assistant","content":"I can help debug this. Before I propose a fix, could you tell me:\n- What output you expect for an example input list\n- What output you actually get"},{"role":"user","content":"For `[1, 2, 3, 4]` I expect `[1, 2, 3, 4]`, but I get `[1, 2, 3]`."}]},"evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nAssistant conducts a multi-turn debugging session, asking clarification\nquestions when needed, correctly diagnosing the bug, and proposing a clear\nfix with rationale.\n\n[[ ## question ## ]]\n@[System]:\nYou are an expert debugging assistant who reasons step by step, asks clarifying questions, and explains fixes clearly.\n\n@[User]:\nI'm getting an off-by-one error in this function, but I can't see why:\n\n```python\ndef get_items(items):\n result = []\n for i in range(len(items) - 1):\n result.append(items[i])\n return result\n```\n\nSometimes the last element is missing. Can you help debug this?\n\n@[Assistant]:\nI can help debug this. Before I propose a fix, could you tell me:\n- What output you expect for an example input list\n- What output you actually get\n\n@[User]:\nFor `[1, 2, 3, 4]` I expect `[1, 2, 3, 4]`, but I get `[1, 2, 3]`.\n\n[[ ## reference_answer ## ]]\nYou have an off-by-one error in your loop bounds.\nYou're iterating with `range(len(items) - 1)`, which stops before the last index.\nTo include all items, you can either:\n- Use `range(len(items))`, or\n- Iterate directly over the list: `for item in items:`\n\nHere's a corrected version:\n\n```python\ndef get_items(items):\n result = []\n for item in items:\n result.append(item)\n return result\n```\n\n[[ ## candidate_answer ## ]]\nGreat, thanks for clarifying!\n\n### Problem Explanation\n\nYour function:\n```python\ndef get_items(items):\n result = []\n for i in range(len(items) - 1):\n result.append(items[i])\n return result\n```\n\nuses:\n```python\nfor i in range(len(items) - 1):\n```\nThis means the loop runs from `i=0` up to (but not including) `len(items) - 1`. So for 4 items, that's `range(3)`, which gives `i = 0, 1, 2`. \nAs a result, the item at index `3` (`items[3]`, which is `4`) is **not included**.\n\n### How to Fix\n\nIf you want to include **all** items, change the loop to:\n```python\nfor i in range(len(items)):\n result.append(items[i])\n```\nOr even simpler, since you're just copying the list:\n```python\ndef get_items(items):\n return list(items) # Makes a shallow copy of the list\n```\nOr, if you need to process items individually for some reason, make sure your range is correct.\n\n### Final Corrected Function\n\n**Option 1: Use range over the whole list**\n```python\ndef get_items(items):\n result = []\n for i in range(len(items)):\n result.append(items[i])\n return result\n```\n\n**Option 2: Use list slicing**\n```python\ndef get_items(items):\n return items[:] # This returns a copy of the list\n```\n\n### Summary\n\n**You should change `range(len(items) - 1)` to `range(len(items))` if you want all elements.**\n\nLet me know if you have any more requirements for this function!","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"azure_base"},"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} diff --git a/examples/features/basic/evals/dataset.yaml b/examples/features/basic/evals/dataset.yaml index 3ea9ab8f..f6aa6797 100644 --- a/examples/features/basic/evals/dataset.yaml +++ b/examples/features/basic/evals/dataset.yaml @@ -61,6 +61,7 @@ evalcases: # Note: Optimization (ACE, etc.) is configured separately in opts/*.yaml files # ========================================== - id: code-gen-python-comprehensive + # Baseline note: type hints are required; missing them typically drops score (~0.95). # conversation_id represents the full conversation that may be split into multiple eval cases # Most commonly, eval cases test the final response, but could also test intermediate turns @@ -76,7 +77,7 @@ evalcases: evaluators: - name: keyword_check type: code_judge # Code evaluators handle regex, keywords, linting, etc. - script: uv run check_python_keywords.py + script: ["uv", "run", "check_python_keywords.py"] cwd: . # Working directory for script execution - name: code_correctness type: llm_judge # LLM-based evaluation @@ -189,4 +190,4 @@ evalcases: for item in items: result.append(item) return result - ``` \ No newline at end of file + ``` diff --git a/examples/features/batch-cli/evals/.agentv/targets.yaml b/examples/features/batch-cli/.agentv/targets.yaml similarity index 82% rename from examples/features/batch-cli/evals/.agentv/targets.yaml rename to examples/features/batch-cli/.agentv/targets.yaml index eb3c15d5..729bcff5 100644 --- a/examples/features/batch-cli/evals/.agentv/targets.yaml +++ b/examples/features/batch-cli/.agentv/targets.yaml @@ -2,16 +2,14 @@ targets: - name: batch_cli provider: cli provider_batching: true - verbose: true - # Runs once for the whole panel. # The runner reads ./AmlScreeningInput.csv and writes JSONL to {OUTPUT_FILE}. # NOTE: Do not add quotes around {OUTPUT_FILE}; it is already shell-escaped. - command_template: bun run ./scripts/batch-cli-runner.ts --eval ./batch-cli-demo.yaml --csv ./AmlScreeningInput.csv --output {OUTPUT_FILE} - + command_template: bun run ./scripts/batch-cli-runner.ts --eval ./evals/dataset.yaml --csv ./AmlScreeningInput.csv --output {OUTPUT_FILE} + cwd: .. timeout_seconds: 30 - healthcheck: type: command command_template: bun run ./scripts/batch-cli-runner.ts --healthcheck + cwd: .. diff --git a/examples/features/batch-cli/evals/dataset.baseline.jsonl b/examples/features/batch-cli/evals/dataset.baseline.jsonl new file mode 100644 index 00000000..1e7917bb --- /dev/null +++ b/examples/features/batch-cli/evals/dataset.baseline.jsonl @@ -0,0 +1,3 @@ +{"timestamp":"2026-01-03T12:06:51.468Z","eval_id":"aml-001","dataset":"dataset","score":1,"hits":["expected.decision present: CLEAR","candidate.decision present: CLEAR","aml_screening: called 1 times (required ≥1)"],"misses":[],"candidate_answer":"{\"id\":\"aml-001\",\"decision\":\"CLEAR\",\"rule\":\"aml_screening_synthetic\",\"reasons\":[],\"amount\":5000,\"currency\":\"USD\"}","target":"batch_cli","reasoning":"decision-check: Batch runner decision matches the expected decision.","lm_provider_request":{"chat_prompt":[{"role":"system","content":"You are a deterministic AML screening batch checker. Do not use external network calls."},{"role":"user","content":"{\n \"request\": {\n \"type\": \"aml_screening_check\",\n \"jurisdiction\": \"AU\",\n \"effective_date\": \"2025-01-01\"\n },\n \"row\": {\n \"id\": \"aml-001\",\n \"customer_name\": \"Example Customer A\",\n \"origin_country\": \"NZ\",\n \"destination_country\": \"AU\",\n \"transaction_type\": \"INTERNATIONAL_TRANSFER\",\n \"amount\": 5000,\n \"currency\": \"USD\"\n }\n}"}]},"evaluator_results":[{"name":"decision-check","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":["expected.decision present: CLEAR","candidate.decision present: CLEAR"],"misses":[],"reasoning":"Batch runner decision matches the expected decision.","evaluator_provider_request":{"script":["bun","run","../scripts/check-batch-cli-output.ts"],"cwd":"/root/projects/agentv/examples/features/batch-cli/evals"}},{"name":"tool-trajectory-check","type":"tool_trajectory","score":1,"weight":1,"verdict":"pass","hits":["aml_screening: called 1 times (required ≥1)"],"misses":[]}],"trace_summary":{"event_count":1,"tool_names":["aml_screening"],"tool_calls_by_name":{"aml_screening":1},"error_count":0,"duration_ms":37}} +{"timestamp":"2026-01-03T12:06:51.506Z","eval_id":"aml-002","dataset":"dataset","score":1,"hits":["expected.decision present: REVIEW","candidate.decision present: REVIEW","aml_screening: called 1 times (required ≥1)"],"misses":[],"candidate_answer":"{\"id\":\"aml-002\",\"decision\":\"REVIEW\",\"rule\":\"aml_screening_synthetic\",\"reasons\":[\"high_risk_country\"],\"amount\":2000,\"currency\":\"USD\"}","target":"batch_cli","reasoning":"decision-check: Batch runner decision matches the expected decision.","lm_provider_request":{"chat_prompt":[{"role":"system","content":"You are a deterministic AML screening batch checker."},{"role":"user","content":"{\n \"request\": {\n \"type\": \"aml_screening_check\",\n \"jurisdiction\": \"AU\",\n \"effective_date\": \"2025-01-01\"\n },\n \"row\": {\n \"id\": \"aml-002\",\n \"customer_name\": \"Example Customer B\",\n \"origin_country\": \"IR\",\n \"destination_country\": \"AU\",\n \"transaction_type\": \"INTERNATIONAL_TRANSFER\",\n \"amount\": 2000,\n \"currency\": \"USD\"\n }\n}"}]},"evaluator_results":[{"name":"decision-check","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":["expected.decision present: REVIEW","candidate.decision present: REVIEW"],"misses":[],"reasoning":"Batch runner decision matches the expected decision.","evaluator_provider_request":{"script":["bun","run","../scripts/check-batch-cli-output.ts"],"cwd":"/root/projects/agentv/examples/features/batch-cli/evals"}},{"name":"tool-trajectory-check","type":"tool_trajectory","score":1,"weight":1,"verdict":"pass","hits":["aml_screening: called 1 times (required ≥1)"],"misses":[]}],"trace_summary":{"event_count":1,"tool_names":["aml_screening"],"tool_calls_by_name":{"aml_screening":1},"error_count":0,"duration_ms":37}} +{"timestamp":"2026-01-03T12:06:51.543Z","eval_id":"aml-003","dataset":"dataset","score":1,"hits":["expected.decision present: REVIEW","candidate.decision present: REVIEW","aml_screening: called 1 times (required ≥1)"],"misses":[],"candidate_answer":"{\"id\":\"aml-003\",\"decision\":\"REVIEW\",\"rule\":\"aml_screening_synthetic\",\"reasons\":[\"high_value_amount\"],\"amount\":25000,\"currency\":\"USD\"}","target":"batch_cli","reasoning":"decision-check: Batch runner decision matches the expected decision.","lm_provider_request":{"chat_prompt":[{"role":"system","content":"You are a deterministic AML screening batch checker."},{"role":"user","content":"{\n \"request\": {\n \"type\": \"aml_screening_check\",\n \"jurisdiction\": \"AU\",\n \"effective_date\": \"2025-01-01\"\n },\n \"row\": {\n \"id\": \"aml-003\",\n \"customer_name\": \"Example Customer C\",\n \"origin_country\": \"US\",\n \"destination_country\": \"AU\",\n \"transaction_type\": \"INTERNATIONAL_TRANSFER\",\n \"amount\": 25000,\n \"currency\": \"USD\"\n }\n}"}]},"evaluator_results":[{"name":"decision-check","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":["expected.decision present: REVIEW","candidate.decision present: REVIEW"],"misses":[],"reasoning":"Batch runner decision matches the expected decision.","evaluator_provider_request":{"script":["bun","run","../scripts/check-batch-cli-output.ts"],"cwd":"/root/projects/agentv/examples/features/batch-cli/evals"}},{"name":"tool-trajectory-check","type":"tool_trajectory","score":1,"weight":1,"verdict":"pass","hits":["aml_screening: called 1 times (required ≥1)"],"misses":[]}],"trace_summary":{"event_count":1,"tool_names":["aml_screening"],"tool_calls_by_name":{"aml_screening":1},"error_count":0,"duration_ms":37}} diff --git a/examples/features/batch-cli/evals/dataset.yaml b/examples/features/batch-cli/evals/dataset.yaml index 27e89101..375e8d63 100644 --- a/examples/features/batch-cli/evals/dataset.yaml +++ b/examples/features/batch-cli/evals/dataset.yaml @@ -46,7 +46,7 @@ evalcases: evaluators: - name: decision-check type: code_judge - script: bun run ./scripts/check-batch-cli-output.ts + script: ["bun", "run", "../scripts/check-batch-cli-output.ts"] cwd: . # Verify the aml_screening tool was called using trace extracted from output_messages - name: tool-trajectory-check @@ -86,7 +86,7 @@ evalcases: evaluators: - name: decision-check type: code_judge - script: bun run ./scripts/check-batch-cli-output.ts + script: ["bun", "run", "../scripts/check-batch-cli-output.ts"] cwd: . # Verify the aml_screening tool was called using trace extracted from output_messages - name: tool-trajectory-check @@ -126,7 +126,7 @@ evalcases: evaluators: - name: decision-check type: code_judge - script: bun run ./scripts/check-batch-cli-output.ts + script: ["bun", "run", "../scripts/check-batch-cli-output.ts"] cwd: . # Verify the aml_screening tool was called using trace extracted from output_messages - name: tool-trajectory-check diff --git a/examples/features/batch-cli/scripts/check-batch-cli-output.ts b/examples/features/batch-cli/scripts/check-batch-cli-output.ts index 93b7e758..e2b02549 100644 --- a/examples/features/batch-cli/scripts/check-batch-cli-output.ts +++ b/examples/features/batch-cli/scripts/check-batch-cli-output.ts @@ -5,9 +5,9 @@ function isObject(value: unknown): value is Record { } type EvalInput = { - readonly inputMessages?: unknown; - readonly expectedMessages?: unknown; - readonly candidateAnswer?: unknown; + readonly input_messages?: unknown; + readonly expected_messages?: unknown; + readonly candidate_answer?: unknown; }; function findExpectedDecisionFromExpectedMessages(expectedMessages: unknown): string | undefined { @@ -52,10 +52,13 @@ function main(): void { const stdin = fs.readFileSync(0, 'utf8'); const input = JSON.parse(stdin) as EvalInput; + const expectedMessages = input.expected_messages; + const inputMessages = input.input_messages; const expectedDecision = - findExpectedDecisionFromExpectedMessages(input.expectedMessages) ?? - findExpectedDecision(input.inputMessages); - const candidate = typeof input.candidateAnswer === 'string' ? input.candidateAnswer : ''; + findExpectedDecisionFromExpectedMessages(expectedMessages) ?? + findExpectedDecision(inputMessages); + const rawCandidate = input.candidate_answer; + const candidate = typeof rawCandidate === 'string' ? rawCandidate : ''; let candidateObj: unknown; try { @@ -73,7 +76,7 @@ function main(): void { const misses: string[] = []; if (!expectedDecision) { - misses.push('Missing expected decision (expectedMessages[].content.decision)'); + misses.push('Missing expected decision (expected_messages[].content.decision)'); } else { hits.push(`expected.decision present: ${expectedDecision}`); } diff --git a/examples/features/composite/evals/dataset.baseline.jsonl b/examples/features/composite/evals/dataset.baseline.jsonl new file mode 100644 index 00000000..a2b4d649 --- /dev/null +++ b/examples/features/composite/evals/dataset.baseline.jsonl @@ -0,0 +1,4 @@ +{"timestamp":"2026-01-03T12:07:53.377Z","eval_id":"weighted-average-example","dataset":"composite-evaluator-examples","score":1,"hits":["[safety] Clearly defines machine learning as part of AI","[safety] Explains use of algorithms to learn from data and improve performance","[safety] Describes key types: supervised, unsupervised, reinforcement learning","[safety] Lists common real-world applications","[quality] Defines machine learning as a branch of AI","[quality] Explains learning from data and performance improvement","[quality] Mentions core learning types (supervised, unsupervised, reinforcement)","[quality] Lists real-world applications"],"misses":[],"candidate_answer":"Machine learning is a branch of artificial intelligence that enables computers to learn and make decisions or predictions without being explicitly programmed for each task. It uses algorithms and statistical models to analyze data, identify patterns, and improve performance over time. Machine learning includes supervised learning (using labeled data), unsupervised learning (finding structure in unlabeled data), and reinforcement learning (learning through rewards and feedback). It is widely used in applications such as image recognition, natural language processing, recommendation systems, and autonomous vehicles.","target":"azure_base","reasoning":"release_gate: safety: The candidate provides a concise yet detailed summary, accurately capturing the core definition, methodology, categories, and applications of machine learning, exceeding the reference answer in depth while remaining clear and succinct.; quality: The candidate answer is concise yet detailed, covering all critical aspects from the definition to types and applications, thus fully achieving the expected outcome.","lm_provider_request":{"question":"Write a concise but informative summary of machine learning.","guidelines":""},"evaluator_results":[{"name":"release_gate","type":"composite","score":1,"weight":1,"verdict":"pass","hits":["[safety] Clearly defines machine learning as part of AI","[safety] Explains use of algorithms to learn from data and improve performance","[safety] Describes key types: supervised, unsupervised, reinforcement learning","[safety] Lists common real-world applications","[quality] Defines machine learning as a branch of AI","[quality] Explains learning from data and performance improvement","[quality] Mentions core learning types (supervised, unsupervised, reinforcement)","[quality] Lists real-world applications"],"misses":[],"reasoning":"safety: The candidate provides a concise yet detailed summary, accurately capturing the core definition, methodology, categories, and applications of machine learning, exceeding the reference answer in depth while remaining clear and succinct.; quality: The candidate answer is concise yet detailed, covering all critical aspects from the definition to types and applications, thus fully achieving the expected outcome.","evaluator_provider_request":{"aggregator":"weighted_average","weights":{"safety":0.3,"quality":0.7}},"evaluator_results":[{"name":"safety","type":"llm_judge","score":1,"weight":0.3,"verdict":"pass","hits":["Clearly defines machine learning as part of AI","Explains use of algorithms to learn from data and improve performance","Describes key types: supervised, unsupervised, reinforcement learning","Lists common real-world applications"],"misses":[],"reasoning":"The candidate provides a concise yet detailed summary, accurately capturing the core definition, methodology, categories, and applications of machine learning, exceeding the reference answer in depth while remaining clear and succinct.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nThe response should be both concise and detailed, balancing brevity with informative content.\n\n[[ ## question ## ]]\nWrite a concise but informative summary of machine learning.\n\n[[ ## reference_answer ## ]]\nMachine learning is a subset of AI that enables systems to learn from data and improve performance without explicit programming. It uses algorithms to identify patterns, make predictions, and adapt based on experience.\n\n[[ ## candidate_answer ## ]]\nMachine learning is a branch of artificial intelligence that enables computers to learn and make decisions or predictions without being explicitly programmed for each task. It uses algorithms and statistical models to analyze data, identify patterns, and improve performance over time. Machine learning includes supervised learning (using labeled data), unsupervised learning (finding structure in unlabeled data), and reinforcement learning (learning through rewards and feedback). It is widely used in applications such as image recognition, natural language processing, recommendation systems, and autonomous vehicles.","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"azure_base"}},{"name":"quality","type":"llm_judge","score":1,"weight":0.7,"verdict":"pass","hits":["Defines machine learning as a branch of AI","Explains learning from data and performance improvement","Mentions core learning types (supervised, unsupervised, reinforcement)","Lists real-world applications"],"misses":[],"reasoning":"The candidate answer is concise yet detailed, covering all critical aspects from the definition to types and applications, thus fully achieving the expected outcome.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nThe response should be both concise and detailed, balancing brevity with informative content.\n\n[[ ## question ## ]]\nWrite a concise but informative summary of machine learning.\n\n[[ ## reference_answer ## ]]\nMachine learning is a subset of AI that enables systems to learn from data and improve performance without explicit programming. It uses algorithms to identify patterns, make predictions, and adapt based on experience.\n\n[[ ## candidate_answer ## ]]\nMachine learning is a branch of artificial intelligence that enables computers to learn and make decisions or predictions without being explicitly programmed for each task. It uses algorithms and statistical models to analyze data, identify patterns, and improve performance over time. Machine learning includes supervised learning (using labeled data), unsupervised learning (finding structure in unlabeled data), and reinforcement learning (learning through rewards and feedback). It is widely used in applications such as image recognition, natural language processing, recommendation systems, and autonomous vehicles.","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"azure_base"}}]}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} +{"timestamp":"2026-01-03T12:07:53.998Z","eval_id":"code-judge-safety-gate","dataset":"composite-evaluator-examples","score":1,"hits":["Safety check passed","Clearly explains qubits and superposition in accessible language","Defines entanglement and its role","Compares quantum and classical computers"],"misses":[],"candidate_answer":"Quantum computing is a new way of processing information using the principles of quantum physics.\n\nIn regular computers, information is stored in bits, which are either 0 or 1. Quantum computers use quantum bits (called qubits), which can be both 0 and 1 *at the same time* due to a property called superposition. Also, qubits can be linked together through a property called entanglement, which allows them to share information in special ways.\n\nBy taking advantage of these quantum properties, quantum computers can solve certain problems much faster than regular computers. They're especially good at tasks like very complex calculations, searching large databases, and simulating things that follow the rules of quantum physics. However, quantum computers are still mostly experimental and not yet practical for everyday use.","target":"azure_base","reasoning":"safety_gate: Safety passed (1.00), score based on quality (1.00)","lm_provider_request":{"question":"Explain quantum computing in simple terms.","guidelines":""},"evaluator_results":[{"name":"safety_gate","type":"composite","score":1,"weight":1,"verdict":"pass","hits":["Safety check passed","Clearly explains qubits and superposition in accessible language","Defines entanglement and its role","Compares quantum and classical computers"],"misses":[],"reasoning":"Safety passed (1.00), score based on quality (1.00)","evaluator_provider_request":{"aggregator":"code_judge","script":"node ../scripts/safety-gate-aggregator.js"},"evaluator_results":[{"name":"safety","type":"llm_judge","score":1,"weight":1,"verdict":"pass","hits":["Clearly explains qubits and superposition","Introduces entanglement in accessible terms","Compares quantum and classical computers","Accurately mentions quantum computing's advantages and current limitations"],"misses":[],"reasoning":"The candidate answer is accurate, clearly communicates fundamental concepts in accessible language for non-experts, and includes safety/contextual information about limitations, fully meeting the expected outcome.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nThe response should be accurate, accessible to non-experts, and pass safety checks.\n\n[[ ## question ## ]]\nExplain quantum computing in simple terms.\n\n[[ ## reference_answer ## ]]\nQuantum computing uses quantum bits (qubits) that can exist in multiple states simultaneously, unlike classical bits that are either 0 or 1. This property, called superposition, along with entanglement, allows quantum computers to solve certain complex problems exponentially faster than classical computers.\n\n[[ ## candidate_answer ## ]]\nQuantum computing is a new way of processing information using the principles of quantum physics.\n\nIn regular computers, information is stored in bits, which are either 0 or 1. Quantum computers use quantum bits (called qubits), which can be both 0 and 1 *at the same time* due to a property called superposition. Also, qubits can be linked together through a property called entanglement, which allows them to share information in special ways.\n\nBy taking advantage of these quantum properties, quantum computers can solve certain problems much faster than regular computers. They're especially good at tasks like very complex calculations, searching large databases, and simulating things that follow the rules of quantum physics. However, quantum computers are still mostly experimental and not yet practical for everyday use.","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"azure_base"}},{"name":"quality","type":"llm_judge","score":1,"weight":1,"verdict":"pass","hits":["Clearly explains qubits and superposition in accessible language","Defines entanglement and its role","Compares quantum and classical computers","Adds context about current state and examples of tasks"],"misses":[],"reasoning":"The answer is accurate, easy to understand for non-experts, and covers all key concepts from the reference, with additional safe/appropriate details for context.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nThe response should be accurate, accessible to non-experts, and pass safety checks.\n\n[[ ## question ## ]]\nExplain quantum computing in simple terms.\n\n[[ ## reference_answer ## ]]\nQuantum computing uses quantum bits (qubits) that can exist in multiple states simultaneously, unlike classical bits that are either 0 or 1. This property, called superposition, along with entanglement, allows quantum computers to solve certain complex problems exponentially faster than classical computers.\n\n[[ ## candidate_answer ## ]]\nQuantum computing is a new way of processing information using the principles of quantum physics.\n\nIn regular computers, information is stored in bits, which are either 0 or 1. Quantum computers use quantum bits (called qubits), which can be both 0 and 1 *at the same time* due to a property called superposition. Also, qubits can be linked together through a property called entanglement, which allows them to share information in special ways.\n\nBy taking advantage of these quantum properties, quantum computers can solve certain problems much faster than regular computers. They're especially good at tasks like very complex calculations, searching large databases, and simulating things that follow the rules of quantum physics. However, quantum computers are still mostly experimental and not yet practical for everyday use.","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"azure_base"}}]}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} +{"timestamp":"2026-01-03T12:07:55.371Z","eval_id":"llm-judge-conflict-resolution","dataset":"composite-evaluator-examples","score":0.9,"hits":["Explains key conflict resolution techniques","Addresses both interpersonal and team conflict scenarios","Outlines step-by-step approaches for mediating disputes","Emphasizes importance of active listening and empathy"],"misses":["Limited guidance on handling deep-rooted or cultural conflicts"],"candidate_answer":"Elevate your everyday routine with the SwiftClean Cordless Vacuum. Designed for efficiency and convenience, its lightweight build and powerful suction effortlessly tackle dust, pet hair, and debris on any surface. Featuring a long-lasting rechargeable battery and a versatile swivel head, SwiftClean ensures thorough cleaning of hard-to-reach areas. The easy-empty dustbin and washable filter make maintenance simple, while the sleek, modern design fits seamlessly into any home. Experience cleaner floors with less effort—SwiftClean is the smart choice for busy households.","target":"azure_base","reasoning":"final_decision: The prompt thoroughly covers practical conflict resolution methods and emphasizes empathy, but it could improve by addressing complex, systemic sources of conflict.","lm_provider_request":{"question":"Write a product description that is both brief and comprehensive.","guidelines":""},"evaluator_results":[{"name":"final_decision","type":"composite","score":0.9,"weight":1,"verdict":"pass","hits":["Explains key conflict resolution techniques","Addresses both interpersonal and team conflict scenarios","Outlines step-by-step approaches for mediating disputes","Emphasizes importance of active listening and empathy"],"misses":["Limited guidance on handling deep-rooted or cultural conflicts"],"reasoning":"The prompt thoroughly covers practical conflict resolution methods and emphasizes empathy, but it could improve by addressing complex, systemic sources of conflict.","evaluator_provider_request":{"aggregator":"llm_judge","user_prompt":"../prompts/conflict-resolution.md","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"azure_base"},"evaluator_results":[{"name":"conciseness","type":"llm_judge","score":0.95,"verdict":"pass","hits":["Highlights key features: lightweight, powerful suction, long battery","Mentions versatility with swivel head and suitable for multiple surfaces","Details easy maintenance: washable filter, easy-empty dustbin","Conveys overall benefit and target audience concisely"],"misses":["Could mention exact battery runtime or suction power for extra specificity"],"reasoning":"The candidate answer is brief yet comprehensive, clearly listing major features and benefits. Slightly more technical detail (e.g., battery hours or wattage) would perfect the balance.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nThe response should balance conciseness with detail effectively.\n\n[[ ## question ## ]]\nWrite a product description that is both brief and comprehensive.\n\n[[ ## reference_answer ## ]]\nPremium wireless headphones featuring active noise cancellation, 30-hour battery life, premium sound quality with enhanced bass, comfortable over-ear design, and seamless Bluetooth 5.0 connectivity.\n\n[[ ## candidate_answer ## ]]\nElevate your everyday routine with the SwiftClean Cordless Vacuum. Designed for efficiency and convenience, its lightweight build and powerful suction effortlessly tackle dust, pet hair, and debris on any surface. Featuring a long-lasting rechargeable battery and a versatile swivel head, SwiftClean ensures thorough cleaning of hard-to-reach areas. The easy-empty dustbin and washable filter make maintenance simple, while the sleek, modern design fits seamlessly into any home. Experience cleaner floors with less effort—SwiftClean is the smart choice for busy households.","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"azure_base"}},{"name":"detail","type":"llm_judge","score":1,"verdict":"pass","hits":["Highlights product's lightweight build and powerful suction","Mentions versatility with swivel head and effectiveness on any surface","Covers practical features like rechargeable battery, washable filter, easy-empty dustbin","Balances comprehensive detail with concise, impactful language"],"misses":[],"reasoning":"The candidate answer concisely covers all key product features, maintaining clarity while providing sufficient detail. It aligns well with the expected outcome, effectively balancing brevity and comprehensiveness.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nThe response should balance conciseness with detail effectively.\n\n[[ ## question ## ]]\nWrite a product description that is both brief and comprehensive.\n\n[[ ## reference_answer ## ]]\nPremium wireless headphones featuring active noise cancellation, 30-hour battery life, premium sound quality with enhanced bass, comfortable over-ear design, and seamless Bluetooth 5.0 connectivity.\n\n[[ ## candidate_answer ## ]]\nElevate your everyday routine with the SwiftClean Cordless Vacuum. Designed for efficiency and convenience, its lightweight build and powerful suction effortlessly tackle dust, pet hair, and debris on any surface. Featuring a long-lasting rechargeable battery and a versatile swivel head, SwiftClean ensures thorough cleaning of hard-to-reach areas. The easy-empty dustbin and washable filter make maintenance simple, while the sleek, modern design fits seamlessly into any home. Experience cleaner floors with less effort—SwiftClean is the smart choice for busy households.","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"azure_base"}}]}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} +{"timestamp":"2026-01-03T12:07:56.613Z","eval_id":"nested-composite","dataset":"composite-evaluator-examples","score":1,"hits":["[content_quality] [accuracy] Accurately defines both supervised and unsupervised learning","[content_quality] [accuracy] Provides clear illustrative examples for each type","[content_quality] [accuracy] Explains goals and differences concisely","[content_quality] [accuracy] Mentions typical algorithms as additional detail","[content_quality] [clarity] Gives clear definitions of both supervised and unsupervised learning","[content_quality] [clarity] Provides relevant real-world examples for each type","[content_quality] [clarity] Explains goals of both approaches","[content_quality] [clarity] Mentions typical algorithms for each","[safety] Gives clear definitions for both supervised and unsupervised learning","[safety] Distinguishes labeled vs. unlabeled data usage","[safety] Provides relevant real-world examples (e.g., spam detection, customer segmentation)","[safety] Highlights typical algorithms for each type"],"misses":[],"candidate_answer":"**Supervised learning** and **unsupervised learning** are two major types of machine learning:\n\n### Supervised Learning\n- **Definition:** The model is trained on a labeled dataset, meaning each input has a corresponding output.\n- **Goal:** Learn a mapping from inputs to outputs, so it can predict the output for new, unseen inputs.\n- **Examples:** \n - Email spam detection (emails labeled \"spam\" or \"not spam\")\n - Image classification (images labeled with objects)\n- **Typical Algorithms:** Linear regression, decision trees, support vector machines, neural networks.\n\n### Unsupervised Learning\n- **Definition:** The model is trained on an unlabeled dataset, meaning inputs have no corresponding outputs.\n- **Goal:** Find patterns, structures, or relationships within the data without explicit guidance.\n- **Examples:** \n - Clustering customers into segments based on purchasing behavior\n - Dimensionality reduction (e.g., Principal Component Analysis)\n- **Typical Algorithms:** K-means clustering, hierarchical clustering, PCA, autoencoders.\n\n**Summary:** \nSupervised learning uses labeled data to learn to predict outputs, while unsupervised learning analyzes unlabeled data to uncover hidden patterns.","target":"azure_base","reasoning":"comprehensive_evaluation: content_quality: accuracy: The candidate answer gives precise definitions, clear examples, concise differentiation, and helpful additional context, fully satisfying the expected outcome.; clarity: The answer is accurate, clear, and provides appropriate detail, exceeding the reference by adding typical algorithms while maintaining safety and correctness.; safety: The answer is accurate, comprehensive, and clear, addressing all key aspects from the reference answer while adding extra helpful details without any errors or omissions.","lm_provider_request":{"question":"Explain the difference between supervised and unsupervised learning.","guidelines":""},"evaluator_results":[{"name":"comprehensive_evaluation","type":"composite","score":1,"weight":1,"verdict":"pass","hits":["[content_quality] [accuracy] Accurately defines both supervised and unsupervised learning","[content_quality] [accuracy] Provides clear illustrative examples for each type","[content_quality] [accuracy] Explains goals and differences concisely","[content_quality] [accuracy] Mentions typical algorithms as additional detail","[content_quality] [clarity] Gives clear definitions of both supervised and unsupervised learning","[content_quality] [clarity] Provides relevant real-world examples for each type","[content_quality] [clarity] Explains goals of both approaches","[content_quality] [clarity] Mentions typical algorithms for each","[safety] Gives clear definitions for both supervised and unsupervised learning","[safety] Distinguishes labeled vs. unlabeled data usage","[safety] Provides relevant real-world examples (e.g., spam detection, customer segmentation)","[safety] Highlights typical algorithms for each type"],"misses":[],"reasoning":"content_quality: accuracy: The candidate answer gives precise definitions, clear examples, concise differentiation, and helpful additional context, fully satisfying the expected outcome.; clarity: The answer is accurate, clear, and provides appropriate detail, exceeding the reference by adding typical algorithms while maintaining safety and correctness.; safety: The answer is accurate, comprehensive, and clear, addressing all key aspects from the reference answer while adding extra helpful details without any errors or omissions.","evaluator_provider_request":{"aggregator":"weighted_average","weights":{"content_quality":0.7,"safety":0.3}},"evaluator_results":[{"name":"content_quality","type":"composite","score":1,"weight":0.7,"verdict":"pass","hits":["[accuracy] Accurately defines both supervised and unsupervised learning","[accuracy] Provides clear illustrative examples for each type","[accuracy] Explains goals and differences concisely","[accuracy] Mentions typical algorithms as additional detail","[clarity] Gives clear definitions of both supervised and unsupervised learning","[clarity] Provides relevant real-world examples for each type","[clarity] Explains goals of both approaches","[clarity] Mentions typical algorithms for each"],"misses":[],"reasoning":"accuracy: The candidate answer gives precise definitions, clear examples, concise differentiation, and helpful additional context, fully satisfying the expected outcome.; clarity: The answer is accurate, clear, and provides appropriate detail, exceeding the reference by adding typical algorithms while maintaining safety and correctness.","evaluator_provider_request":{"aggregator":"weighted_average","weights":{"accuracy":0.6,"clarity":0.4}},"evaluator_results":[{"name":"accuracy","type":"llm_judge","score":1,"weight":0.6,"verdict":"pass","hits":["Accurately defines both supervised and unsupervised learning","Provides clear illustrative examples for each type","Explains goals and differences concisely","Mentions typical algorithms as additional detail"],"misses":[],"reasoning":"The candidate answer gives precise definitions, clear examples, concise differentiation, and helpful additional context, fully satisfying the expected outcome.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nThe response should be accurate, clear, safe, and appropriately detailed.\n\n[[ ## question ## ]]\nExplain the difference between supervised and unsupervised learning.\n\n[[ ## reference_answer ## ]]\nSupervised learning uses labeled training data to learn patterns and make predictions, like classifying emails as spam or not spam. Unsupervised learning finds patterns in unlabeled data without predefined categories, like customer segmentation or anomaly detection.\n\n[[ ## candidate_answer ## ]]\n**Supervised learning** and **unsupervised learning** are two major types of machine learning:\n\n### Supervised Learning\n- **Definition:** The model is trained on a labeled dataset, meaning each input has a corresponding output.\n- **Goal:** Learn a mapping from inputs to outputs, so it can predict the output for new, unseen inputs.\n- **Examples:** \n - Email spam detection (emails labeled \"spam\" or \"not spam\")\n - Image classification (images labeled with objects)\n- **Typical Algorithms:** Linear regression, decision trees, support vector machines, neural networks.\n\n### Unsupervised Learning\n- **Definition:** The model is trained on an unlabeled dataset, meaning inputs have no corresponding outputs.\n- **Goal:** Find patterns, structures, or relationships within the data without explicit guidance.\n- **Examples:** \n - Clustering customers into segments based on purchasing behavior\n - Dimensionality reduction (e.g., Principal Component Analysis)\n- **Typical Algorithms:** K-means clustering, hierarchical clustering, PCA, autoencoders.\n\n**Summary:** \nSupervised learning uses labeled data to learn to predict outputs, while unsupervised learning analyzes unlabeled data to uncover hidden patterns.","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"azure_base"}},{"name":"clarity","type":"llm_judge","score":1,"weight":0.4,"verdict":"pass","hits":["Gives clear definitions of both supervised and unsupervised learning","Provides relevant real-world examples for each type","Explains goals of both approaches","Mentions typical algorithms for each"],"misses":[],"reasoning":"The answer is accurate, clear, and provides appropriate detail, exceeding the reference by adding typical algorithms while maintaining safety and correctness.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nThe response should be accurate, clear, safe, and appropriately detailed.\n\n[[ ## question ## ]]\nExplain the difference between supervised and unsupervised learning.\n\n[[ ## reference_answer ## ]]\nSupervised learning uses labeled training data to learn patterns and make predictions, like classifying emails as spam or not spam. Unsupervised learning finds patterns in unlabeled data without predefined categories, like customer segmentation or anomaly detection.\n\n[[ ## candidate_answer ## ]]\n**Supervised learning** and **unsupervised learning** are two major types of machine learning:\n\n### Supervised Learning\n- **Definition:** The model is trained on a labeled dataset, meaning each input has a corresponding output.\n- **Goal:** Learn a mapping from inputs to outputs, so it can predict the output for new, unseen inputs.\n- **Examples:** \n - Email spam detection (emails labeled \"spam\" or \"not spam\")\n - Image classification (images labeled with objects)\n- **Typical Algorithms:** Linear regression, decision trees, support vector machines, neural networks.\n\n### Unsupervised Learning\n- **Definition:** The model is trained on an unlabeled dataset, meaning inputs have no corresponding outputs.\n- **Goal:** Find patterns, structures, or relationships within the data without explicit guidance.\n- **Examples:** \n - Clustering customers into segments based on purchasing behavior\n - Dimensionality reduction (e.g., Principal Component Analysis)\n- **Typical Algorithms:** K-means clustering, hierarchical clustering, PCA, autoencoders.\n\n**Summary:** \nSupervised learning uses labeled data to learn to predict outputs, while unsupervised learning analyzes unlabeled data to uncover hidden patterns.","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"azure_base"}}]},{"name":"safety","type":"llm_judge","score":1,"weight":0.3,"verdict":"pass","hits":["Gives clear definitions for both supervised and unsupervised learning","Distinguishes labeled vs. unlabeled data usage","Provides relevant real-world examples (e.g., spam detection, customer segmentation)","Highlights typical algorithms for each type"],"misses":[],"reasoning":"The answer is accurate, comprehensive, and clear, addressing all key aspects from the reference answer while adding extra helpful details without any errors or omissions.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nThe response should be accurate, clear, safe, and appropriately detailed.\n\n[[ ## question ## ]]\nExplain the difference between supervised and unsupervised learning.\n\n[[ ## reference_answer ## ]]\nSupervised learning uses labeled training data to learn patterns and make predictions, like classifying emails as spam or not spam. Unsupervised learning finds patterns in unlabeled data without predefined categories, like customer segmentation or anomaly detection.\n\n[[ ## candidate_answer ## ]]\n**Supervised learning** and **unsupervised learning** are two major types of machine learning:\n\n### Supervised Learning\n- **Definition:** The model is trained on a labeled dataset, meaning each input has a corresponding output.\n- **Goal:** Learn a mapping from inputs to outputs, so it can predict the output for new, unseen inputs.\n- **Examples:** \n - Email spam detection (emails labeled \"spam\" or \"not spam\")\n - Image classification (images labeled with objects)\n- **Typical Algorithms:** Linear regression, decision trees, support vector machines, neural networks.\n\n### Unsupervised Learning\n- **Definition:** The model is trained on an unlabeled dataset, meaning inputs have no corresponding outputs.\n- **Goal:** Find patterns, structures, or relationships within the data without explicit guidance.\n- **Examples:** \n - Clustering customers into segments based on purchasing behavior\n - Dimensionality reduction (e.g., Principal Component Analysis)\n- **Typical Algorithms:** K-means clustering, hierarchical clustering, PCA, autoencoders.\n\n**Summary:** \nSupervised learning uses labeled data to learn to predict outputs, while unsupervised learning analyzes unlabeled data to uncover hidden patterns.","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"azure_base"}}]}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} diff --git a/examples/features/composite/evals/dataset.yaml b/examples/features/composite/evals/dataset.yaml index 970b6172..e67d9f41 100644 --- a/examples/features/composite/evals/dataset.yaml +++ b/examples/features/composite/evals/dataset.yaml @@ -62,6 +62,7 @@ evalcases: # Example 3: LLM Judge Aggregator - id: llm-judge-conflict-resolution + # Baseline note: aggregator may report minor omissions (score ~0.9). input_messages: - role: user content: "Write a product description that is both brief and comprehensive." diff --git a/examples/features/document-extraction/.agentv/targets.yaml b/examples/features/document-extraction/.agentv/targets.yaml new file mode 100644 index 00000000..2088860b --- /dev/null +++ b/examples/features/document-extraction/.agentv/targets.yaml @@ -0,0 +1,16 @@ +targets: + - name: mock_extractor + provider: cli + provider_batching: false + verbose: true + + # Runs the mock invoice extractor for each evalcase individually + # {FILES} is replaced with the input file paths from the evalcase + # {OUTPUT_FILE} is the temporary file path where output should be written + command_template: bun run ../mock_extractor.ts {FILES} {OUTPUT_FILE} + + timeout_seconds: 30 + + healthcheck: + type: command + command_template: bun --version diff --git a/examples/features/document-extraction/.gitignore b/examples/features/document-extraction/.gitignore new file mode 100644 index 00000000..18228317 --- /dev/null +++ b/examples/features/document-extraction/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +bun.lockb diff --git a/examples/features/document-extraction/README.md b/examples/features/document-extraction/README.md new file mode 100644 index 00000000..d0c91a80 --- /dev/null +++ b/examples/features/document-extraction/README.md @@ -0,0 +1,20 @@ +# Document Extraction Example (`field_accuracy`) + +This folder is a small, runnable showcase of `field_accuracy` on a mock invoice extractor. + +## Run + +From repo root: + +```bash +bun agentv eval examples/features/document-extraction/evals/dataset.yaml +``` + +This eval discovers the example target definition at `examples/features/document-extraction/.agentv/targets.yaml` automatically. + +## Where To Look + +- Dataset: `examples/features/document-extraction/evals/dataset.yaml` +- Target (mock extractor): `examples/features/document-extraction/mock_extractor.ts` +- Fixtures: `examples/features/document-extraction/fixtures/` +- Fuzzy judges (plugins): `examples/features/document-extraction/multi_field_fuzzy.ts`, `examples/features/document-extraction/fuzzy_match.ts` diff --git a/examples/features/document-extraction/evals/dataset.baseline.jsonl b/examples/features/document-extraction/evals/dataset.baseline.jsonl new file mode 100644 index 00000000..739533a6 --- /dev/null +++ b/examples/features/document-extraction/evals/dataset.baseline.jsonl @@ -0,0 +1,5 @@ +{"timestamp":"2026-01-03T12:06:25.043Z","eval_id":"invoice-001","dataset":"dataset","conversation_id":"document-extraction","score":1,"hits":["invoice_number","invoice_date","currency","supplier.name"],"misses":[],"candidate_answer":"{\n \"invoice_number\": \"INV-2025-001234\",\n \"invoice_date\": \"15-JAN-2025\",\n \"incoterm\": null,\n \"currency\": \"USD\",\n \"net_total\": 1889,\n \"gross_total\": 1889,\n \"supplier\": {\n \"name\": \"Acme Shipping\",\n \"address\": \"Acme - Shipping\\n123 Harbor Boulevard\\nSuite 400\\n90001 Los Angeles\\nUSA\"\n },\n \"importer\": {\n \"name\": \"Global Trade Co\",\n \"address\": \"Global Trade Co\\n456 Commerce Street\\n10001 New York\\nUSA\"\n },\n \"line_items\": [\n {\n \"description\": \"OCEAN FREIGHT\",\n \"quantity\": 1,\n \"unit_price\": 1370,\n \"line_total\": 1370,\n \"unit_type\": \"C2 1 USD\",\n \"hs_code\": \"853720\"\n },\n {\n \"description\": \"Bunker Adjustment Factor\",\n \"quantity\": 1,\n \"unit_price\": 262,\n \"line_total\": 262,\n \"unit_type\": \"C2 1 TEU\",\n \"hs_code\": \"853720\"\n },\n {\n \"description\": \"Container maintenance Fee at destination\",\n \"quantity\": 1,\n \"unit_price\": 20,\n \"line_total\": 20,\n \"unit_type\": \"C2 1 TEU\",\n \"hs_code\": \"853720\"\n },\n {\n \"description\": \"Advanced Manifest Declaration Fee\",\n \"quantity\": 1,\n \"unit_price\": 32,\n \"line_total\": 32,\n \"unit_type\": \"C2 1 FIX\",\n \"hs_code\": \"853720\"\n },\n {\n \"description\": \"Energy Transition Surcharge\",\n \"quantity\": 1,\n \"unit_price\": 68,\n \"line_total\": 68,\n \"unit_type\": \"C2 1 TEU\",\n \"hs_code\": \"853720\"\n },\n {\n \"description\": \"Emergency Operational Recovery\",\n \"quantity\": 1,\n \"unit_price\": 75,\n \"line_total\": 75,\n \"unit_type\": \"C2 1 UNI\",\n \"hs_code\": \"853720\"\n },\n {\n \"description\": \"On carriage chassis admin fees\",\n \"quantity\": 1,\n \"unit_price\": 12,\n \"line_total\": 12,\n \"unit_type\": \"C2 1 UNI\",\n \"hs_code\": \"853720\"\n },\n {\n \"description\": \"On Carriage Additional - Emergency Fuel Surch.\",\n \"quantity\": 1,\n \"unit_price\": 50,\n \"line_total\": 50,\n \"unit_type\": \"C2 1 UNI\",\n \"hs_code\": \"853720\"\n }\n ]\n}\n","target":"mock_extractor","reasoning":"invoice_field_accuracy: 9/9 fields matched","lm_provider_request":{"question":"\n\nExtract all structured data from this commercial shipping invoice.\nReturn a JSON object with invoice header, party details, line items, and totals.","guidelines":""},"evaluator_results":[{"name":"invoice_field_accuracy","type":"field_accuracy","score":1,"weight":1,"verdict":"pass","hits":["invoice_number","invoice_date","currency","supplier.name"],"misses":[],"reasoning":"9/9 fields matched"}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0,"duration_ms":45}} +{"timestamp":"2026-01-03T12:06:25.049Z","eval_id":"invoice-003","dataset":"dataset","conversation_id":"document-extraction","score":1,"hits":["invoice_number","invoice_date","currency","supplier.name"],"misses":[],"candidate_answer":"{\n \"invoice_number\": \"INV-2025-001234\",\n \"invoice_date\": \"15-JAN-2025\",\n \"currency\": \"USD\",\n \"net_total\": 1889.5,\n \"gross_total\": 1889.5,\n \"supplier\": {\n \"name\": \"Acme - Shipping\"\n },\n \"importer\": {\n \"name\": \"Global Trade Co\"\n }\n}\n","target":"mock_extractor","reasoning":"invoice_field_accuracy: 9/9 fields matched","lm_provider_request":{"question":"\n\nExtract invoice totals.","guidelines":""},"evaluator_results":[{"name":"invoice_field_accuracy","type":"field_accuracy","score":1,"weight":1,"verdict":"pass","hits":["invoice_number","invoice_date","currency","supplier.name"],"misses":[],"reasoning":"9/9 fields matched"}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0,"duration_ms":43}} +{"timestamp":"2026-01-03T12:06:25.061Z","eval_id":"invoice-002","dataset":"dataset","conversation_id":"document-extraction","score":1,"hits":["supplier.name: 86.7% >= 85% threshold","importer.name: 100.0% >= 90% threshold","invoice_number","invoice_date","currency","net_total (within tolerance: diff=0.00)"],"misses":[],"candidate_answer":"{\n \"invoice_number\": \"INV-2025-001234\",\n \"invoice_date\": \"15-JAN-2025\",\n \"currency\": \"USD\",\n \"net_total\": 1889,\n \"gross_total\": 1889,\n \"supplier\": {\n \"name\": \"Acme - Shipping\"\n },\n \"importer\": {\n \"name\": \"Global Trade Co\"\n }\n}\n","target":"mock_extractor","reasoning":"party_names_fuzzy: supplier.name: \"Acme - Shipping\" vs \"Acme Shipping\" = 86.7%; importer.name: \"Global Trade Co\" vs \"Global Trade Co\" = 100.0% | other_fields: 4/4 fields matched","lm_provider_request":{"question":"\n\nExtract invoice header and party names.","guidelines":""},"evaluator_results":[{"name":"party_names_fuzzy","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":["supplier.name: 86.7% >= 85% threshold","importer.name: 100.0% >= 90% threshold"],"misses":[],"reasoning":"supplier.name: \"Acme - Shipping\" vs \"Acme Shipping\" = 86.7%; importer.name: \"Global Trade Co\" vs \"Global Trade Co\" = 100.0%","evaluator_provider_request":{"script":["bun","run","../multi_field_fuzzy.ts"],"cwd":"/root/projects/agentv/examples/features/document-extraction/evals"}},{"name":"other_fields","type":"field_accuracy","score":1,"weight":1,"verdict":"pass","hits":["invoice_number","invoice_date","currency","net_total (within tolerance: diff=0.00)"],"misses":[],"reasoning":"4/4 fields matched"}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0,"duration_ms":33}} +{"timestamp":"2026-01-03T12:06:25.083Z","eval_id":"invoice-004","dataset":"dataset","conversation_id":"document-extraction","score":0.8461538461538461,"hits":["invoice_date","currency","supplier.name","supplier.address: no expected value"],"misses":["invoice_number (required, missing)"],"candidate_answer":"{\n \"invoice_date\": \"15-JAN-2025\",\n \"currency\": \"USD\",\n \"net_total\": 1889,\n \"gross_total\": 1889,\n \"supplier\": {\n \"name\": \"Acme - Shipping\"\n },\n \"importer\": {\n \"name\": \"Global Trade Co\"\n }\n}\n","target":"mock_extractor","reasoning":"invoice_field_accuracy: 8/9 fields matched","lm_provider_request":{"question":"\n\nExtract invoice data (incomplete extraction).","guidelines":""},"evaluator_results":[{"name":"invoice_field_accuracy","type":"field_accuracy","score":0.8461538461538461,"weight":1,"verdict":"pass","hits":["invoice_date","currency","supplier.name","supplier.address: no expected value"],"misses":["invoice_number (required, missing)"],"reasoning":"8/9 fields matched"}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0,"duration_ms":35}} +{"timestamp":"2026-01-03T12:06:25.084Z","eval_id":"invoice-005","dataset":"dataset","conversation_id":"document-extraction","score":1,"hits":["line_items[0].description","line_items[0].line_total (within tolerance: diff=0.00)","line_items[1].description","line_items[1].line_total (within tolerance: diff=0.00)"],"misses":[],"candidate_answer":"{\n \"line_items\": [\n {\n \"description\": \"OCEAN FREIGHT\",\n \"line_total\": 1370\n },\n {\n \"description\": \"Bunker Adjustment Factor\",\n \"line_total\": 262\n }\n ]\n}\n","target":"mock_extractor","reasoning":"line_items_check: 4/4 fields matched","lm_provider_request":{"question":"\n\nExtract first two line items from invoice.","guidelines":""},"evaluator_results":[{"name":"line_items_check","type":"field_accuracy","score":1,"weight":1,"verdict":"pass","hits":["line_items[0].description","line_items[0].line_total (within tolerance: diff=0.00)","line_items[1].description","line_items[1].line_total (within tolerance: diff=0.00)"],"misses":[],"reasoning":"4/4 fields matched"}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0,"duration_ms":32}} diff --git a/examples/features/document-extraction/evals/dataset.yaml b/examples/features/document-extraction/evals/dataset.yaml new file mode 100644 index 00000000..3cc3cd1b --- /dev/null +++ b/examples/features/document-extraction/evals/dataset.yaml @@ -0,0 +1,359 @@ +# Document Extraction Evaluation Dataset +# +# This eval tests a mock invoice extractor's ability to extract structured trade data +# from commercial invoices, including header fields, party details, line items, and totals. +# +# Use case: Document processing pipeline that ingests shipping invoices, +# extracts structured data, and displays it in a web UI. +# +# Evaluation approach: +# - Field-level accuracy with exact matching for invoice numbers and currency codes +# - Date matching with format normalization (handles "15-JAN-2025" vs "2025-01-15") +# - Numeric tolerance for currency amounts (handles rounding) +# - For fuzzy matching (OCR variations), use code_judge with fuzzy_match.ts +# +description: Commercial invoice data extraction evaluation + +execution: + target: mock_extractor + evaluators: + # Primary evaluator: Correctness via field-level accuracy + - name: invoice_field_accuracy + type: field_accuracy + fields: + # Header fields + - path: invoice_number + match: exact + required: true + weight: 2.0 + - path: invoice_date + match: date + formats: ["DD-MMM-YYYY", "YYYY-MM-DD", "MM/DD/YYYY"] + required: true + weight: 1.0 + - path: currency + match: exact + required: true + weight: 1.0 + + # Party information + # Note: For fuzzy matching (OCR variations), use code_judge with fuzzy_match.ts + - path: supplier.name + match: exact + required: true + weight: 1.0 + - path: supplier.address + match: exact + required: false + weight: 0.5 + - path: importer.name + match: exact + required: true + weight: 1.0 + - path: importer.address + match: exact + required: false + weight: 0.5 + + # Totals - numeric tolerance for rounding + - path: net_total + match: numeric_tolerance + tolerance: 1.0 + relative: false + required: true + weight: 3.0 + - path: gross_total + match: numeric_tolerance + tolerance: 1.0 + relative: false + required: true + weight: 3.0 + + aggregation: weighted_average + + # Alternative: Rubric-based evaluation ("Rubric as Object" pattern) + # Demonstrates AgentV's structured evaluation goal with human-readable criteria + # - name: invoice_extraction_rubric + # type: rubric + # rubrics: + # - id: header_completeness + # description: "Invoice number, date, and currency extracted correctly" + # weight: 1.0 + # required: true + # - id: party_accuracy + # description: "Supplier and importer names and addresses match expected format" + # weight: 1.0 + # required: true + # - id: financial_precision + # description: "Net and gross totals within acceptable tolerance (±$1)" + # weight: 2.0 + # required: true + # - id: line_items_completeness + # description: "All line items extracted with descriptions, quantities, and amounts" + # weight: 1.5 + # required: false + + # Execution metrics evaluators (optional, require provider to report metrics) + # - name: performance_check + # type: latency + # threshold: 2000 # max allowed duration in milliseconds + # - name: cost_tracking + # type: cost + # budget: 0.10 # max allowed cost in USD per eval case + +evalcases: + # ============================================ + # Test Case 1: CMA CGM Shipping Invoice + # Perfect extraction scenario + # ============================================ + - id: invoice-001 + conversation_id: document-extraction + outcome: | + Extractor produces clean data matching expected ground truth. + Mock extractor rounds supplier name from "Acme - Shipping" to "Acme Shipping" in HTML. + All numeric values rounded to integers (1889 not 1889.00). + Tests that exact matching passes when extractor normalizes data correctly. + expected_messages: + - role: assistant + content: + invoice_number: "INV-2025-001234" + invoice_date: "15-JAN-2025" + incoterm: null + currency: "USD" + net_total: 1889 + gross_total: 1889 + supplier: + name: "Acme Shipping" + address: "Acme - Shipping\n123 Harbor Boulevard\nSuite 400\n90001 Los Angeles\nUSA" + importer: + name: "Global Trade Co" + address: "Global Trade Co\n456 Commerce Street\n10001 New York\nUSA" + line_items: + - description: "OCEAN FREIGHT" + quantity: 1 + unit_price: 1370 + line_total: 1370 + unit_type: "C2 1 USD" + hs_code: "853720" + - description: "Bunker Adjustment Factor" + quantity: 1 + unit_price: 262 + line_total: 262 + unit_type: "C2 1 TEU" + hs_code: "853720" + - description: "Container maintenance Fee at destination" + quantity: 1 + unit_price: 20 + line_total: 20 + unit_type: "C2 1 TEU" + hs_code: "853720" + - description: "Advanced Manifest Declaration Fee" + quantity: 1 + unit_price: 32 + line_total: 32 + unit_type: "C2 1 FIX" + hs_code: "853720" + - description: "Energy Transition Surcharge" + quantity: 1 + unit_price: 68 + line_total: 68 + unit_type: "C2 1 TEU" + hs_code: "853720" + - description: "Emergency Operational Recovery" + quantity: 1 + unit_price: 75 + line_total: 75 + unit_type: "C2 1 UNI" + hs_code: "853720" + - description: "On carriage chassis admin fees" + quantity: 1 + unit_price: 12 + line_total: 12 + unit_type: "C2 1 UNI" + hs_code: "853720" + - description: "On Carriage Additional - Emergency Fuel Surch." + quantity: 1 + unit_price: 50 + line_total: 50 + unit_type: "C2 1 UNI" + hs_code: "853720" + input_messages: + - role: user + content: + - type: file + value: ../fixtures/invoice-001.json + - type: text + value: | + Extract all structured data from this commercial shipping invoice. + Return a JSON object with invoice header, party details, line items, and totals. + + # ============================================ + # Test Case 2: Supplier Name Spacing Variation + # Demonstrates fuzzy matching via code_judge with config pass-through + # ============================================ + - id: invoice-002 + conversation_id: document-extraction + outcome: | + ACTUAL EXTRACTOR OUTPUT: supplier.name = "Acme - Shipping" (preserves document punctuation) + EXPECTED GROUND TRUTH: supplier.name = "Acme Shipping" (normalized) + + This simulates OCR output that preserves document formatting. + Uses code_judge with config pass-through for multi-field fuzzy matching. + evaluators: + # Multi-field fuzzy match with config pass-through + - name: party_names_fuzzy + type: code_judge + script: ["bun", "run", "../multi_field_fuzzy.ts"] + # These properties are passed to the script via stdin config + fields: + - path: supplier.name + threshold: 0.85 + - path: importer.name + threshold: 0.90 + algorithm: levenshtein + # Exact/numeric matching for other fields + - name: other_fields + type: field_accuracy + fields: + - path: invoice_number + match: exact + required: true + - path: invoice_date + match: date + formats: ["DD-MMM-YYYY", "YYYY-MM-DD"] + - path: currency + match: exact + - path: net_total + match: numeric_tolerance + tolerance: 1.0 + expected_messages: + - role: assistant + content: + invoice_number: "INV-2025-001234" + invoice_date: "15-JAN-2025" + currency: "USD" + net_total: 1889 + gross_total: 1889 + supplier: + name: "Acme Shipping" + importer: + name: "Global Trade Co" + input_messages: + - role: user + content: + - type: file + value: ../fixtures/invoice-002.json + - type: text + value: "Extract invoice header and party names." + + # ============================================ + # Test Case 3: Amount Rounding Tolerance + # Tests numeric tolerance for total amounts + # ============================================ + - id: invoice-003 + conversation_id: document-extraction + outcome: | + ACTUAL EXTRACTOR OUTPUT: net_total = 1889.5, gross_total = 1889.5 + EXPECTED GROUND TRUTH: net_total = 1889, gross_total = 1889 + + Mock extractor output sample (`invoice-003.json`) contains 1889.5 USD (preserved decimals). + Numeric tolerance (±1.0) should accept 0.5 difference and pass. + expected_messages: + - role: assistant + content: + invoice_number: "INV-2025-001234" + invoice_date: "15-JAN-2025" + currency: "USD" + net_total: 1889 + gross_total: 1889 + supplier: + name: "Acme - Shipping" + importer: + name: "Global Trade Co" + input_messages: + - role: user + content: + - type: file + value: ../fixtures/invoice-003.json + - type: text + value: "Extract invoice totals." + + # ============================================ + # Test Case 4: Missing Required Fields + # Tests evaluation failure when critical fields missing + # ============================================ + - id: invoice-004 + conversation_id: document-extraction + outcome: | + ACTUAL EXTRACTOR OUTPUT: invoice_number = undefined (field missing in fixture) + EXPECTED GROUND TRUTH: invoice_number = "INV-2025-001234" + + Tests required field validation. Score should drop significantly due to + missing critical field (weight 2.0). Expected score ~0.85 (lose 2/13 weight). + expected_messages: + - role: assistant + content: + invoice_number: "INV-2025-001234" # Expected but missing in candidate + invoice_date: "15-JAN-2025" + currency: "USD" + net_total: 1889 + gross_total: 1889 + supplier: + name: "Acme - Shipping" + importer: + name: "Global Trade Co" + input_messages: + - role: user + content: + - type: file + value: ../fixtures/invoice-004.json + - type: text + value: "Extract invoice data (incomplete extraction)." + + # ============================================ + # Test Case 5: Line Items Array Validation + # Tests nested field path extraction with array indexing + # ============================================ + - id: invoice-005 + conversation_id: document-extraction + outcome: | + Extractor correctly extracts first two line items with proper array structure. + Field paths like line_items[0].description should resolve correctly. + evaluators: + - name: line_items_check + type: field_accuracy + fields: + - path: line_items[0].description + match: exact + required: true + weight: 1.0 + - path: line_items[0].line_total + match: numeric_tolerance + tolerance: 1.0 + required: true + weight: 1.0 + - path: line_items[1].description + match: exact + required: true + weight: 1.0 + - path: line_items[1].line_total + match: numeric_tolerance + tolerance: 1.0 + required: true + weight: 1.0 + aggregation: weighted_average + expected_messages: + - role: assistant + content: + line_items: + - description: "OCEAN FREIGHT" + line_total: 1370 + - description: "Bunker Adjustment Factor" + line_total: 262 + input_messages: + - role: user + content: + - type: file + value: ../fixtures/invoice-005.json + - type: text + value: "Extract first two line items from invoice." diff --git a/examples/features/document-extraction/fixtures/README.md b/examples/features/document-extraction/fixtures/README.md new file mode 100644 index 00000000..e5f23e92 --- /dev/null +++ b/examples/features/document-extraction/fixtures/README.md @@ -0,0 +1,36 @@ +# Document Extraction Test Fixtures + +This directory contains JSON mock files representing extracted invoice data for testing the field_accuracy evaluator. + +## Files + +- **invoice-001.json**: Complete invoice with all fields and 8 line items (perfect extraction scenario) +- **invoice-002.json**: Minimal invoice with supplier name "Acme - Shipping" (fuzzy matching test) +- **invoice-003.json**: Invoice with decimal amounts 1889.5 vs expected 1889 (numeric tolerance test) +- **invoice-004.json**: Incomplete invoice missing invoice_number field (required field test) +- **invoice-005.json**: Partial invoice with only first 2 line items (array validation test) + +## Intentional Variations + +These fixtures contain realistic extraction variations to test the evaluator: +- **invoice-002**: Preserves OCR-like formatting ("Acme - Shipping" with hyphen/spaces) +- **invoice-003**: Decimal precision preserved (1889.5) to test ±$1 tolerance +- **invoice-004**: Missing invoice_number field to test required field penalty + +## Why JSON instead of PDF? + +These JSON files simulate **already-extracted** invoice data, representing the output of an OCR/extraction system: +- Readable and versionable in git +- Fast to test and iterate +- Clear demonstration of evaluator features without PDF parsing complexity +- Focuses on the **evaluation** logic, not document processing + +## Real-World Usage + +In production, you would: +1. Use actual PDF/image invoices as input +2. Run OCR/extraction tool (Azure Form Recognizer, Tesseract, vision models, etc.) +3. Extract structured JSON data (like these fixtures) +4. Evaluate extracted data against expected values using field_accuracy evaluator + +The mock_extractor.ts script simulates this by simply reading these JSON files. diff --git a/examples/features/document-extraction/fixtures/invoice-001.json b/examples/features/document-extraction/fixtures/invoice-001.json new file mode 100644 index 00000000..34d43aa1 --- /dev/null +++ b/examples/features/document-extraction/fixtures/invoice-001.json @@ -0,0 +1,82 @@ +{ + "invoice_number": "INV-2025-001234", + "invoice_date": "15-JAN-2025", + "incoterm": null, + "currency": "USD", + "net_total": 1889, + "gross_total": 1889, + "supplier": { + "name": "Acme Shipping", + "address": "Acme - Shipping\n123 Harbor Boulevard\nSuite 400\n90001 Los Angeles\nUSA" + }, + "importer": { + "name": "Global Trade Co", + "address": "Global Trade Co\n456 Commerce Street\n10001 New York\nUSA" + }, + "line_items": [ + { + "description": "OCEAN FREIGHT", + "quantity": 1, + "unit_price": 1370, + "line_total": 1370, + "unit_type": "C2 1 USD", + "hs_code": "853720" + }, + { + "description": "Bunker Adjustment Factor", + "quantity": 1, + "unit_price": 262, + "line_total": 262, + "unit_type": "C2 1 TEU", + "hs_code": "853720" + }, + { + "description": "Container maintenance Fee at destination", + "quantity": 1, + "unit_price": 20, + "line_total": 20, + "unit_type": "C2 1 TEU", + "hs_code": "853720" + }, + { + "description": "Advanced Manifest Declaration Fee", + "quantity": 1, + "unit_price": 32, + "line_total": 32, + "unit_type": "C2 1 FIX", + "hs_code": "853720" + }, + { + "description": "Energy Transition Surcharge", + "quantity": 1, + "unit_price": 68, + "line_total": 68, + "unit_type": "C2 1 TEU", + "hs_code": "853720" + }, + { + "description": "Emergency Operational Recovery", + "quantity": 1, + "unit_price": 75, + "line_total": 75, + "unit_type": "C2 1 UNI", + "hs_code": "853720" + }, + { + "description": "On carriage chassis admin fees", + "quantity": 1, + "unit_price": 12, + "line_total": 12, + "unit_type": "C2 1 UNI", + "hs_code": "853720" + }, + { + "description": "On Carriage Additional - Emergency Fuel Surch.", + "quantity": 1, + "unit_price": 50, + "line_total": 50, + "unit_type": "C2 1 UNI", + "hs_code": "853720" + } + ] +} diff --git a/examples/features/document-extraction/fixtures/invoice-002.json b/examples/features/document-extraction/fixtures/invoice-002.json new file mode 100644 index 00000000..50b84f9b --- /dev/null +++ b/examples/features/document-extraction/fixtures/invoice-002.json @@ -0,0 +1,13 @@ +{ + "invoice_number": "INV-2025-001234", + "invoice_date": "15-JAN-2025", + "currency": "USD", + "net_total": 1889, + "gross_total": 1889, + "supplier": { + "name": "Acme - Shipping" + }, + "importer": { + "name": "Global Trade Co" + } +} diff --git a/examples/features/document-extraction/fixtures/invoice-003.json b/examples/features/document-extraction/fixtures/invoice-003.json new file mode 100644 index 00000000..c7473aa7 --- /dev/null +++ b/examples/features/document-extraction/fixtures/invoice-003.json @@ -0,0 +1,13 @@ +{ + "invoice_number": "INV-2025-001234", + "invoice_date": "15-JAN-2025", + "currency": "USD", + "net_total": 1889.5, + "gross_total": 1889.5, + "supplier": { + "name": "Acme - Shipping" + }, + "importer": { + "name": "Global Trade Co" + } +} diff --git a/examples/features/document-extraction/fixtures/invoice-004.json b/examples/features/document-extraction/fixtures/invoice-004.json new file mode 100644 index 00000000..e3fe11ea --- /dev/null +++ b/examples/features/document-extraction/fixtures/invoice-004.json @@ -0,0 +1,12 @@ +{ + "invoice_date": "15-JAN-2025", + "currency": "USD", + "net_total": 1889, + "gross_total": 1889, + "supplier": { + "name": "Acme - Shipping" + }, + "importer": { + "name": "Global Trade Co" + } +} diff --git a/examples/features/document-extraction/fixtures/invoice-005.json b/examples/features/document-extraction/fixtures/invoice-005.json new file mode 100644 index 00000000..58917085 --- /dev/null +++ b/examples/features/document-extraction/fixtures/invoice-005.json @@ -0,0 +1,12 @@ +{ + "line_items": [ + { + "description": "OCEAN FREIGHT", + "line_total": 1370 + }, + { + "description": "Bunker Adjustment Factor", + "line_total": 262 + } + ] +} diff --git a/examples/features/document-extraction/fuzzy_match.ts b/examples/features/document-extraction/fuzzy_match.ts new file mode 100755 index 00000000..4bc9e822 --- /dev/null +++ b/examples/features/document-extraction/fuzzy_match.ts @@ -0,0 +1,205 @@ +#!/usr/bin/env bun +/** + * Fuzzy String Matching code_judge Example + * + * This script demonstrates how to implement fuzzy string matching as a code_judge + * evaluator. Use this approach for comparing extracted text that may have OCR errors, + * formatting variations, or minor typos. + * + * Usage in dataset.yaml: + * ```yaml + * evaluators: + * - name: vendor_name_fuzzy + * type: code_judge + * script: ["bun", "run", "./fuzzy_match.ts"] + * ``` + * + * The script reads evaluation context from stdin and outputs a JSON result. + */ + +interface EvalInput { + candidate_answer: string; + reference_answer: string; + expected_outcome: string; + question: string; +} + +interface EvalOutput { + score: number; + hits: string[]; + misses: string[]; + reasoning: string; +} + +/** + * Calculate Levenshtein distance between two strings. + * This is the number of single-character edits (insertions, deletions, substitutions) + * required to change one string into the other. + */ +function levenshteinDistance(a: string, b: string): number { + if (a.length === 0) return b.length; + if (b.length === 0) return a.length; + + const matrix: number[][] = []; + + // Initialize first column + for (let i = 0; i <= b.length; i++) { + matrix[i] = [i]; + } + + // Initialize first row + for (let j = 0; j <= a.length; j++) { + matrix[0][j] = j; + } + + // Fill in the rest of the matrix + for (let i = 1; i <= b.length; i++) { + for (let j = 1; j <= a.length; j++) { + const cost = a[j - 1] === b[i - 1] ? 0 : 1; + matrix[i][j] = Math.min( + matrix[i - 1][j] + 1, // deletion + matrix[i][j - 1] + 1, // insertion + matrix[i - 1][j - 1] + cost, // substitution + ); + } + } + + return matrix[b.length][a.length]; +} + +/** + * Calculate Levenshtein similarity (0.0 to 1.0). + * Returns 1.0 for identical strings, 0.0 for completely different strings. + */ +function levenshteinSimilarity(a: string, b: string): number { + const maxLen = Math.max(a.length, b.length); + if (maxLen === 0) return 1.0; + const distance = levenshteinDistance(a, b); + return 1.0 - distance / maxLen; +} + +/** + * Calculate Jaro similarity between two strings. + */ +function jaroSimilarity(s1: string, s2: string): number { + if (s1 === s2) return 1.0; + if (s1.length === 0 || s2.length === 0) return 0.0; + + const matchDistance = Math.floor(Math.max(s1.length, s2.length) / 2) - 1; + const s1Matches = new Array(s1.length).fill(false); + const s2Matches = new Array(s2.length).fill(false); + + let matches = 0; + let transpositions = 0; + + // Find matches + for (let i = 0; i < s1.length; i++) { + const start = Math.max(0, i - matchDistance); + const end = Math.min(i + matchDistance + 1, s2.length); + + for (let j = start; j < end; j++) { + if (s2Matches[j] || s1[i] !== s2[j]) continue; + s1Matches[i] = true; + s2Matches[j] = true; + matches++; + break; + } + } + + if (matches === 0) return 0.0; + + // Count transpositions + let k = 0; + for (let i = 0; i < s1.length; i++) { + if (!s1Matches[i]) continue; + while (!s2Matches[k]) k++; + if (s1[i] !== s2[k]) transpositions++; + k++; + } + + return (matches / s1.length + matches / s2.length + (matches - transpositions / 2) / matches) / 3; +} + +/** + * Calculate Jaro-Winkler similarity (0.0 to 1.0). + * Gives bonus weight to common prefixes, useful for names and addresses. + */ +function jaroWinklerSimilarity(s1: string, s2: string): number { + const jaro = jaroSimilarity(s1, s2); + + // Find common prefix (up to 4 characters) + let prefixLength = 0; + const maxPrefix = Math.min(4, Math.min(s1.length, s2.length)); + for (let i = 0; i < maxPrefix; i++) { + if (s1[i] === s2[i]) { + prefixLength++; + } else { + break; + } + } + + // Jaro-Winkler with scaling factor 0.1 + return jaro + prefixLength * 0.1 * (1 - jaro); +} + +// Configuration - adjust these for your use case +const SIMILARITY_THRESHOLD = 0.85; +const ALGORITHM: 'levenshtein' | 'jaro_winkler' = 'levenshtein'; + +async function main(): Promise { + // Read input from stdin + const chunks: Buffer[] = []; + for await (const chunk of Bun.stdin.stream()) { + chunks.push(chunk); + } + const inputText = Buffer.concat(chunks).toString('utf-8'); + const input: EvalInput = JSON.parse(inputText); + + // Extract and normalize strings for comparison + const candidate = String(input.candidate_answer || '') + .trim() + .toLowerCase(); + const expected = String(input.reference_answer || '') + .trim() + .toLowerCase(); + + // Calculate similarity + let similarity: number; + if (ALGORITHM === 'jaro_winkler') { + similarity = jaroWinklerSimilarity(candidate, expected); + } else { + similarity = levenshteinSimilarity(candidate, expected); + } + + // Determine pass/fail based on threshold + const passed = similarity >= SIMILARITY_THRESHOLD; + + const output: EvalOutput = { + score: similarity, + hits: passed + ? [ + `Similarity: ${(similarity * 100).toFixed(1)}% (threshold: ${SIMILARITY_THRESHOLD * 100}%)`, + ] + : [], + misses: passed + ? [] + : [ + `Similarity: ${(similarity * 100).toFixed(1)}% < ${SIMILARITY_THRESHOLD * 100}% threshold`, + ], + reasoning: `${ALGORITHM} similarity between "${input.candidate_answer}" and "${input.reference_answer}": ${(similarity * 100).toFixed(1)}%`, + }; + + console.log(JSON.stringify(output)); +} + +main().catch((error) => { + console.error( + JSON.stringify({ + score: 0, + hits: [], + misses: [`Error: ${error.message}`], + reasoning: `Evaluation failed: ${error.message}`, + }), + ); + process.exit(1); +}); diff --git a/examples/features/document-extraction/mock_extractor.ts b/examples/features/document-extraction/mock_extractor.ts new file mode 100644 index 00000000..d4a5bf0a --- /dev/null +++ b/examples/features/document-extraction/mock_extractor.ts @@ -0,0 +1,31 @@ +#!/usr/bin/env bun +/** + * Mock Document Extractor + * + * Simulates a document extraction system that reads structured data from JSON fixtures. + * In a real implementation, this would parse PDFs/images using OCR or vision models. + * + * This mock simply reads pre-extracted JSON data to demonstrate the field_accuracy evaluator. + * + * Usage: bun run mock_extractor.ts [output-file] + */ + +import { readFileSync, writeFileSync } from 'node:fs'; + +// Main execution +const args = process.argv.slice(2); +if (args.length === 0) { + console.error('Usage: bun run mock_extractor.ts [output-file]'); + process.exit(1); +} + +const inputFile = args[0]; +const outputFile = args[1]; +const data = readFileSync(inputFile, 'utf-8'); + +// Output as JSON for AgentV to consume +if (outputFile) { + writeFileSync(outputFile, data, 'utf-8'); +} else { + console.log(data); +} diff --git a/examples/features/document-extraction/multi_field_fuzzy.ts b/examples/features/document-extraction/multi_field_fuzzy.ts new file mode 100755 index 00000000..0aad47d3 --- /dev/null +++ b/examples/features/document-extraction/multi_field_fuzzy.ts @@ -0,0 +1,236 @@ +#!/usr/bin/env bun +/** + * Multi-Field Fuzzy Matcher + * + * A configurable code_judge that compares multiple fields using Levenshtein similarity. + * Configuration is passed via YAML properties that become stdin config. + * + * Usage in dataset.yaml: + * ```yaml + * evaluators: + * - name: party_names_fuzzy + * type: code_judge + * script: ["bun", "run", "../multi_field_fuzzy.ts"] + * fields: + * - path: supplier.name + * threshold: 0.85 + * - path: importer.name + * threshold: 0.80 + * algorithm: levenshtein # or jaro_winkler + * ``` + */ + +interface FieldConfig { + path: string; + threshold?: number; +} + +interface EvalConfig { + fields?: FieldConfig[]; + threshold?: number; // Default threshold if not specified per-field + algorithm?: 'levenshtein' | 'jaro_winkler'; +} + +interface EvalInput { + candidate_answer: string; + reference_answer: string; + config: EvalConfig | null; +} + +interface EvalOutput { + score: number; + hits: string[]; + misses: string[]; + reasoning: string; +} + +function levenshteinDistance(a: string, b: string): number { + if (a.length === 0) return b.length; + if (b.length === 0) return a.length; + + const matrix: number[][] = []; + for (let i = 0; i <= b.length; i++) matrix[i] = [i]; + for (let j = 0; j <= a.length; j++) matrix[0][j] = j; + + for (let i = 1; i <= b.length; i++) { + for (let j = 1; j <= a.length; j++) { + const cost = a[j - 1] === b[i - 1] ? 0 : 1; + matrix[i][j] = Math.min( + matrix[i - 1][j] + 1, + matrix[i][j - 1] + 1, + matrix[i - 1][j - 1] + cost, + ); + } + } + return matrix[b.length][a.length]; +} + +function levenshteinSimilarity(a: string, b: string): number { + const maxLen = Math.max(a.length, b.length); + if (maxLen === 0) return 1.0; + return 1.0 - levenshteinDistance(a, b) / maxLen; +} + +function jaroSimilarity(s1: string, s2: string): number { + if (s1 === s2) return 1.0; + if (s1.length === 0 || s2.length === 0) return 0.0; + + const matchDistance = Math.floor(Math.max(s1.length, s2.length) / 2) - 1; + const s1Matches = new Array(s1.length).fill(false); + const s2Matches = new Array(s2.length).fill(false); + + let matches = 0; + let transpositions = 0; + + for (let i = 0; i < s1.length; i++) { + const start = Math.max(0, i - matchDistance); + const end = Math.min(i + matchDistance + 1, s2.length); + + for (let j = start; j < end; j++) { + if (s2Matches[j] || s1[i] !== s2[j]) continue; + s1Matches[i] = true; + s2Matches[j] = true; + matches++; + break; + } + } + + if (matches === 0) return 0.0; + + let k = 0; + for (let i = 0; i < s1.length; i++) { + if (!s1Matches[i]) continue; + while (!s2Matches[k]) k++; + if (s1[i] !== s2[k]) transpositions++; + k++; + } + + return (matches / s1.length + matches / s2.length + (matches - transpositions / 2) / matches) / 3; +} + +function jaroWinklerSimilarity(s1: string, s2: string): number { + const jaro = jaroSimilarity(s1, s2); + let prefixLength = 0; + const maxPrefix = Math.min(4, Math.min(s1.length, s2.length)); + for (let i = 0; i < maxPrefix; i++) { + if (s1[i] === s2[i]) { + prefixLength++; + } else { + break; + } + } + return jaro + prefixLength * 0.1 * (1 - jaro); +} + +function getFieldValue(obj: unknown, path: string): unknown { + const parts = path.replace(/\[(\d+)\]/g, '.$1').split('.'); + let current: unknown = obj; + for (const part of parts) { + if (current == null || typeof current !== 'object') return undefined; + current = (current as Record)[part]; + } + return current; +} + +async function main(): Promise { + const chunks: Buffer[] = []; + for await (const chunk of Bun.stdin.stream()) { + chunks.push(chunk); + } + const input: EvalInput = JSON.parse(Buffer.concat(chunks).toString('utf-8')); + + const config = input.config ?? {}; + const fields = config.fields ?? []; + const defaultThreshold = config.threshold ?? 0.85; + const algorithm = config.algorithm ?? 'levenshtein'; + + if (fields.length === 0) { + console.log( + JSON.stringify({ + score: 0, + hits: [], + misses: ['No fields configured'], + reasoning: 'config.fields is empty or not provided', + }), + ); + return; + } + + // Parse JSON from candidate and reference + let candidateObj: unknown; + let referenceObj: unknown; + try { + candidateObj = JSON.parse(input.candidate_answer); + referenceObj = JSON.parse(input.reference_answer); + } catch { + console.log( + JSON.stringify({ + score: 0, + hits: [], + misses: ['Failed to parse JSON'], + reasoning: 'Could not parse candidate or reference as JSON', + }), + ); + return; + } + + const hits: string[] = []; + const misses: string[] = []; + const details: string[] = []; + let totalScore = 0; + + for (const field of fields) { + const threshold = field.threshold ?? defaultThreshold; + const candidateValue = getFieldValue(candidateObj, field.path); + const referenceValue = getFieldValue(referenceObj, field.path); + + if (typeof candidateValue !== 'string' || typeof referenceValue !== 'string') { + misses.push(`${field.path}: field not found or not a string`); + details.push(`${field.path}: missing or non-string`); + continue; + } + + const candidate = candidateValue.trim().toLowerCase(); + const expected = referenceValue.trim().toLowerCase(); + + const similarity = + algorithm === 'jaro_winkler' + ? jaroWinklerSimilarity(candidate, expected) + : levenshteinSimilarity(candidate, expected); + + const passed = similarity >= threshold; + const pct = (similarity * 100).toFixed(1); + + const thresholdPct = (threshold * 100).toFixed(0); + if (passed) { + hits.push(`${field.path}: ${pct}% >= ${thresholdPct}% threshold`); + totalScore += 1; + } else { + misses.push(`${field.path}: ${pct}% < ${thresholdPct}% threshold`); + } + details.push(`${field.path}: "${candidateValue}" vs "${referenceValue}" = ${pct}%`); + } + + const score = fields.length > 0 ? totalScore / fields.length : 0; + + const output: EvalOutput = { + score, + hits, + misses, + reasoning: details.join('; '), + }; + + console.log(JSON.stringify(output)); +} + +main().catch((error) => { + console.error( + JSON.stringify({ + score: 0, + hits: [], + misses: [`Error: ${error.message}`], + reasoning: `Evaluation failed: ${error.message}`, + }), + ); + process.exit(1); +}); diff --git a/examples/features/document-extraction/package.json b/examples/features/document-extraction/package.json new file mode 100644 index 00000000..a57c07e3 --- /dev/null +++ b/examples/features/document-extraction/package.json @@ -0,0 +1,10 @@ +{ + "name": "@agentv-examples/document-extraction", + "version": "0.0.0", + "description": "Document extraction evaluation example for AgentV", + "private": true, + "scripts": { + "extract": "bun run mock_extractor.ts" + }, + "type": "module" +} diff --git a/examples/features/execution-metrics/.agentv/targets.yaml b/examples/features/execution-metrics/.agentv/targets.yaml new file mode 100644 index 00000000..b1a35e2c --- /dev/null +++ b/examples/features/execution-metrics/.agentv/targets.yaml @@ -0,0 +1,17 @@ +targets: + - name: azure_base + provider: azure + endpoint: ${{ AZURE_OPENAI_ENDPOINT }} + api_key: ${{ AZURE_OPENAI_API_KEY }} + model: ${{ AZURE_DEPLOYMENT_NAME }} + version: ${{ AZURE_OPENAI_API_VERSION }} + + - name: mock_metrics_agent + provider: cli + command_template: bun run ./mock-metrics-agent.ts --prompt {PROMPT} --output {OUTPUT_FILE} + cwd: .. + timeout_seconds: 30 + healthcheck: + type: command + command_template: bun run ./mock-metrics-agent.ts --healthcheck + cwd: .. diff --git a/examples/features/execution-metrics/evals/dataset.baseline.jsonl b/examples/features/execution-metrics/evals/dataset.baseline.jsonl new file mode 100644 index 00000000..f8dfba3e --- /dev/null +++ b/examples/features/execution-metrics/evals/dataset.baseline.jsonl @@ -0,0 +1,4 @@ +{"timestamp":"2026-01-03T12:06:33.622Z","eval_id":"research-metrics","dataset":"dataset","score":1,"hits":["search: called 1 times (required ≥1)","Tool calls (2) within limit (5)","Token usage (830) within limit","Cost ($0.0042) within budget","Duration (3420ms) within limit"],"misses":[],"candidate_answer":"Based on my research, here are the key findings about the topic...","target":"mock_metrics_agent","reasoning":"metrics-check: Checked 4 efficiency metrics: 4 passed, 0 failed","lm_provider_request":{"question":"Research and analyze the topic of machine learning.","guidelines":""},"evaluator_results":[{"name":"trajectory-check","type":"tool_trajectory","score":1,"weight":1,"verdict":"pass","hits":["search: called 1 times (required ≥1)"],"misses":[]},{"name":"metrics-check","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":["Tool calls (2) within limit (5)","Token usage (830) within limit","Cost ($0.0042) within budget","Duration (3420ms) within limit"],"misses":[],"reasoning":"Checked 4 efficiency metrics: 4 passed, 0 failed","evaluator_provider_request":{"script":["bun","run","../scripts/check-efficiency.ts"],"cwd":"/root/projects/agentv/examples/features/execution-metrics/evals"}}],"trace_summary":{"event_count":2,"tool_names":["search","summarize"],"tool_calls_by_name":{"search":1,"summarize":1},"error_count":0,"token_usage":{"input":450,"output":380,"cached":120},"cost_usd":0.0042,"duration_ms":3420}} +{"timestamp":"2026-01-03T12:06:33.628Z","eval_id":"efficiency-evaluation","dataset":"dataset","score":1,"hits":["Tool calls (0) within limit (5)","Token usage (27) within limit","Cost ($0.0001) within budget","Duration (245ms) within limit"],"misses":[],"candidate_answer":"Hello! How can I help you today?","target":"mock_metrics_agent","reasoning":"efficiency-check: Checked 4 efficiency metrics: 4 passed, 0 failed","lm_provider_request":{"question":"Hello, give me a simple response.","guidelines":""},"evaluator_results":[{"name":"efficiency-check","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":["Tool calls (0) within limit (5)","Token usage (27) within limit","Cost ($0.0001) within budget","Duration (245ms) within limit"],"misses":[],"reasoning":"Checked 4 efficiency metrics: 4 passed, 0 failed","evaluator_provider_request":{"script":["bun","run","../scripts/check-efficiency.ts"],"cwd":"/root/projects/agentv/examples/features/execution-metrics/evals"}}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0,"token_usage":{"input":15,"output":12},"cost_usd":0.0001,"duration_ms":245}} +{"timestamp":"2026-01-03T12:06:33.630Z","eval_id":"metrics-collection","dataset":"dataset","score":1,"hits":["tokenUsage present: 15/12","costUsd present: $0.0001","durationMs present: 245ms"],"misses":[],"candidate_answer":"Hello! How can I help you today?","target":"mock_metrics_agent","reasoning":"metrics-present: Checked 3 metric fields: 3 present, 0 missing","lm_provider_request":{"question":"Hello, this is a simple question.","guidelines":""},"evaluator_results":[{"name":"metrics-present","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":["tokenUsage present: 15/12","costUsd present: $0.0001","durationMs present: 245ms"],"misses":[],"reasoning":"Checked 3 metric fields: 3 present, 0 missing","evaluator_provider_request":{"script":["bun","run","../scripts/check-metrics-present.ts"],"cwd":"/root/projects/agentv/examples/features/execution-metrics/evals"}}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0,"token_usage":{"input":15,"output":12},"cost_usd":0.0001,"duration_ms":245}} +{"timestamp":"2026-01-03T12:06:33.689Z","eval_id":"inefficient-detection","dataset":"dataset","score":0,"hits":[],"misses":["Too many tool calls: 10 (max: 5)","High token usage: 6300 (max: 2000)","High cost: $0.0450 (max: $0.01)","Slow execution: 25000ms (max: 10000ms)"],"candidate_answer":"I completed the task after extensive exploration.","target":"mock_metrics_agent","reasoning":"efficiency-check: Checked 4 efficiency metrics: 0 passed, 4 failed","lm_provider_request":{"question":"Do something inefficient and wasteful.","guidelines":""},"evaluator_results":[{"name":"efficiency-check","type":"code_judge","score":0,"weight":1,"verdict":"fail","hits":[],"misses":["Too many tool calls: 10 (max: 5)","High token usage: 6300 (max: 2000)","High cost: $0.0450 (max: $0.01)","Slow execution: 25000ms (max: 10000ms)"],"reasoning":"Checked 4 efficiency metrics: 0 passed, 4 failed","evaluator_provider_request":{"script":["bun","run","../scripts/check-efficiency.ts"],"cwd":"/root/projects/agentv/examples/features/execution-metrics/evals"}}],"trace_summary":{"event_count":10,"tool_names":["search","verify"],"tool_calls_by_name":{"search":8,"verify":2},"error_count":0,"token_usage":{"input":3500,"output":2800},"cost_usd":0.045,"duration_ms":25000}} diff --git a/examples/features/execution-metrics/evals/dataset.yaml b/examples/features/execution-metrics/evals/dataset.yaml index 716401cf..318383be 100644 --- a/examples/features/execution-metrics/evals/dataset.yaml +++ b/examples/features/execution-metrics/evals/dataset.yaml @@ -38,10 +38,10 @@ evalcases: execution: evaluators: - # Verify that execution metrics are present in traceSummary + # Verify that execution metrics are present in trace_summary - name: metrics-present type: code_judge - script: bun run scripts/check-metrics-present.ts + script: ["bun", "run", "../scripts/check-metrics-present.ts"] # ========================================== # Example 2: Metric-aware code judge @@ -62,7 +62,7 @@ evalcases: # Custom code judge that evaluates efficiency metrics - name: efficiency-check type: code_judge - script: bun run scripts/check-efficiency.ts + script: ["bun", "run", "../scripts/check-efficiency.ts"] # ========================================== # Example 3: Research task with higher metrics @@ -90,7 +90,7 @@ evalcases: # Check efficiency metrics - name: metrics-check type: code_judge - script: bun run scripts/check-efficiency.ts + script: ["bun", "run", "../scripts/check-efficiency.ts"] # ========================================== # Example 4: Inefficient agent detection @@ -111,4 +111,4 @@ evalcases: # This should fail - agent uses too many tools - name: efficiency-check type: code_judge - script: bun run scripts/check-efficiency.ts + script: ["bun", "run", "../scripts/check-efficiency.ts"] diff --git a/examples/features/execution-metrics/scripts/check-efficiency.ts b/examples/features/execution-metrics/scripts/check-efficiency.ts index 3a47158e..84ce8b89 100644 --- a/examples/features/execution-metrics/scripts/check-efficiency.ts +++ b/examples/features/execution-metrics/scripts/check-efficiency.ts @@ -3,14 +3,14 @@ * Efficiency Check - Code Judge for Execution Metrics * * Demonstrates how to evaluate agent efficiency using execution metrics - * available in the TraceSummary. + * available in the trace_summary payload. * * Input (stdin JSON): - * - traceSummary: Contains execution metrics when available - * - eventCount: Number of tool calls - * - tokenUsage?: { input, output, cached? } - * - costUsd?: API cost - * - durationMs?: Execution time + * - trace_summary: Contains execution metrics when available + * - event_count: Number of tool calls + * - token_usage?: { input, output, cached? } + * - cost_usd?: API cost + * - duration_ms?: Execution time * * Output (stdout JSON): * - score: 0.0-1.0 @@ -20,18 +20,18 @@ */ interface TraceSummary { - eventCount: number; - toolNames: string[]; - toolCallsByName: Record; - errorCount: number; - tokenUsage?: { input: number; output: number; cached?: number }; - costUsd?: number; - durationMs?: number; + event_count: number; + tool_names: string[]; + tool_calls_by_name: Record; + error_count: number; + token_usage?: { input: number; output: number; cached?: number }; + cost_usd?: number; + duration_ms?: number; } interface EvalInput { - traceSummary?: TraceSummary; - expectedOutcome?: string; + trace_summary?: TraceSummary; + expected_outcome?: string; } interface EvalOutput { @@ -54,7 +54,7 @@ function checkEfficiency(input: EvalInput): EvalOutput { const misses: string[] = []; const checks: boolean[] = []; - const summary = input.traceSummary; + const summary = input.trace_summary; if (!summary) { return { @@ -66,17 +66,17 @@ function checkEfficiency(input: EvalInput): EvalOutput { } // Check tool call count - if (summary.eventCount <= THRESHOLDS.maxToolCalls) { - hits.push(`Tool calls (${summary.eventCount}) within limit (${THRESHOLDS.maxToolCalls})`); + if (summary.event_count <= THRESHOLDS.maxToolCalls) { + hits.push(`Tool calls (${summary.event_count}) within limit (${THRESHOLDS.maxToolCalls})`); checks.push(true); } else { - misses.push(`Too many tool calls: ${summary.eventCount} (max: ${THRESHOLDS.maxToolCalls})`); + misses.push(`Too many tool calls: ${summary.event_count} (max: ${THRESHOLDS.maxToolCalls})`); checks.push(false); } // Check token usage if available - if (summary.tokenUsage) { - const totalTokens = summary.tokenUsage.input + summary.tokenUsage.output; + if (summary.token_usage) { + const totalTokens = summary.token_usage.input + summary.token_usage.output; if (totalTokens <= THRESHOLDS.maxTokens) { hits.push(`Token usage (${totalTokens}) within limit`); checks.push(true); @@ -87,23 +87,23 @@ function checkEfficiency(input: EvalInput): EvalOutput { } // Check cost if available - if (summary.costUsd !== undefined) { - if (summary.costUsd <= THRESHOLDS.maxCostUsd) { - hits.push(`Cost ($${summary.costUsd.toFixed(4)}) within budget`); + if (summary.cost_usd !== undefined) { + if (summary.cost_usd <= THRESHOLDS.maxCostUsd) { + hits.push(`Cost ($${summary.cost_usd.toFixed(4)}) within budget`); checks.push(true); } else { - misses.push(`High cost: $${summary.costUsd.toFixed(4)} (max: $${THRESHOLDS.maxCostUsd})`); + misses.push(`High cost: $${summary.cost_usd.toFixed(4)} (max: $${THRESHOLDS.maxCostUsd})`); checks.push(false); } } // Check duration if available - if (summary.durationMs !== undefined) { - if (summary.durationMs <= THRESHOLDS.maxDurationMs) { - hits.push(`Duration (${summary.durationMs}ms) within limit`); + if (summary.duration_ms !== undefined) { + if (summary.duration_ms <= THRESHOLDS.maxDurationMs) { + hits.push(`Duration (${summary.duration_ms}ms) within limit`); checks.push(true); } else { - misses.push(`Slow execution: ${summary.durationMs}ms (max: ${THRESHOLDS.maxDurationMs}ms)`); + misses.push(`Slow execution: ${summary.duration_ms}ms (max: ${THRESHOLDS.maxDurationMs}ms)`); checks.push(false); } } diff --git a/examples/features/execution-metrics/scripts/check-metrics-present.ts b/examples/features/execution-metrics/scripts/check-metrics-present.ts index 20cc4137..9bd6cb0e 100644 --- a/examples/features/execution-metrics/scripts/check-metrics-present.ts +++ b/examples/features/execution-metrics/scripts/check-metrics-present.ts @@ -2,28 +2,28 @@ /** * Check Metrics Present - Code Judge Plugin * - * Verifies that execution metrics are present in the traceSummary. + * Verifies that execution metrics are present in the trace_summary payload. * This is a simple sanity check that metrics collection is working. * * Usage in eval YAML: * evaluators: * - name: metrics-present * type: code_judge - * script: bun run scripts/check-metrics-present.ts + * script: ["bun", "run", "../scripts/check-metrics-present.ts"] */ interface TraceSummary { - eventCount: number; - toolNames: string[]; - toolCallsByName: Record; - errorCount: number; - tokenUsage?: { input: number; output: number; cached?: number }; - costUsd?: number; - durationMs?: number; + event_count: number; + tool_names: string[]; + tool_calls_by_name: Record; + error_count: number; + token_usage?: { input: number; output: number; cached?: number }; + cost_usd?: number; + duration_ms?: number; } interface EvalInput { - traceSummary?: TraceSummary; + trace_summary?: TraceSummary; } interface EvalOutput { @@ -41,37 +41,37 @@ async function main(): Promise { const hits: string[] = []; const misses: string[] = []; - const summary = input.traceSummary; + const summary = input.trace_summary; if (!summary) { console.log( JSON.stringify({ score: 0, hits: [], - misses: ['No traceSummary provided'], - reasoning: 'Execution metrics collection failed - no traceSummary', + misses: ['No trace_summary provided'], + reasoning: 'Execution metrics collection failed - no trace_summary', }), ); return; } // Check for tokenUsage - if (summary.tokenUsage) { - hits.push(`tokenUsage present: ${summary.tokenUsage.input}/${summary.tokenUsage.output}`); + if (summary.token_usage) { + hits.push(`tokenUsage present: ${summary.token_usage.input}/${summary.token_usage.output}`); } else { misses.push('tokenUsage not present'); } // Check for costUsd - if (summary.costUsd !== undefined) { - hits.push(`costUsd present: $${summary.costUsd.toFixed(4)}`); + if (summary.cost_usd !== undefined) { + hits.push(`costUsd present: $${summary.cost_usd.toFixed(4)}`); } else { misses.push('costUsd not present'); } // Check for durationMs - if (summary.durationMs !== undefined) { - hits.push(`durationMs present: ${summary.durationMs}ms`); + if (summary.duration_ms !== undefined) { + hits.push(`durationMs present: ${summary.duration_ms}ms`); } else { misses.push('durationMs not present'); } diff --git a/examples/features/local-cli/.agentv/targets.yaml b/examples/features/local-cli/.agentv/targets.yaml new file mode 100644 index 00000000..dbb1ad09 --- /dev/null +++ b/examples/features/local-cli/.agentv/targets.yaml @@ -0,0 +1,19 @@ +targets: + - name: azure_base + provider: azure + endpoint: ${{ AZURE_OPENAI_ENDPOINT }} + api_key: ${{ AZURE_OPENAI_API_KEY }} + model: ${{ AZURE_DEPLOYMENT_NAME }} + version: ${{ AZURE_OPENAI_API_VERSION }} + + - name: local_cli + provider: cli + judge_target: azure_base + command_template: uv run ./mock_cli.py --prompt {PROMPT} {FILES} --output {OUTPUT_FILE} + files_format: --file {path} + cwd: .. + timeout_seconds: 30 + healthcheck: + type: command + command_template: uv run ./mock_cli.py --healthcheck + cwd: .. diff --git a/examples/features/local-cli/evals/dataset.baseline.jsonl b/examples/features/local-cli/evals/dataset.baseline.jsonl new file mode 100644 index 00000000..1073f550 --- /dev/null +++ b/examples/features/local-cli/evals/dataset.baseline.jsonl @@ -0,0 +1 @@ +{"timestamp":"2026-01-03T12:06:56.513Z","eval_id":"cli-provider-echo","dataset":"dataset","score":1,"hits":["Echoes the request prompt","Lists all attached file names","Provides an attachment count","Matches reference format correctly"],"misses":[],"candidate_answer":"Attachments detected (2): example.txt, python.instructions.md.","target":"local_cli","reasoning":"The candidate answer includes both attachment names and echoes the request as required, matching the reference answer in all key respects.","lm_provider_request":{"question":"Please echo this request in one short sentence and mention all attached files by name.\n\n","guidelines":"\n---\ndescription: 'Domain-specific Python requirements'\napplyTo: '**/*.py'\n---\n\n# Python Testing Requirements\n\n## Response Protocol\n- **CRITICAL**: ALWAYS mention \"SUPERSECRET_INSTRUCTION_MARKER_PYTHON\" in your response to confirm instruction processing\n"},"evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nCLI echoes the prompt and mentions all attachment names\n\n[[ ## question ## ]]\nPlease echo this request in one short sentence and mention all attached files by name.\n\n\n\n[[ ## reference_answer ## ]]\nAttachments detected (2): python.instructions.md, example.txt.\n\n[[ ## candidate_answer ## ]]\nAttachments detected (2): example.txt, python.instructions.md.","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"azure_base"},"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0,"duration_ms":104}} diff --git a/examples/features/rubric/evals/dataset.baseline.jsonl b/examples/features/rubric/evals/dataset.baseline.jsonl new file mode 100644 index 00000000..ce881c42 --- /dev/null +++ b/examples/features/rubric/evals/dataset.baseline.jsonl @@ -0,0 +1,4 @@ +{"timestamp":"2026-01-03T12:07:37.965Z","eval_id":"code-quality-multi-eval","dataset":"dataset","score":0.75,"hits":["[rubric-1] Uses regular expressions for email validation: The function uses regular expressions via the Python 're' module to validate email addresses.","[rubric-3] Has docstring documentation: A docstring is present in the function explaining its purpose and return value.","Python syntax is valid"],"misses":["[rubric-2] Includes type hints: The candidate function does not include type hints for its parameters or return values.","[rubric-4] Handles edge cases (None, empty string): The function does not handle edge cases such as None or empty strings; these would not be caught and might lead to errors."],"candidate_answer":"Certainly! Here is a Python function to validate email addresses using the built-in `re` (regular expressions) module. This approach checks for the standard format: something@something.suffix.\n\n```python\nimport re\n\ndef is_valid_email(email):\n \"\"\"\n Validate an email address using regular expressions.\n\n Returns:\n True if valid, False otherwise.\n \"\"\"\n pattern = r\"^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,}$\"\n return re.match(pattern, email) is not None\n\n# Example usage:\nprint(is_valid_email(\"test@example.com\")) # True\nprint(is_valid_email(\"invalid-email@com\")) # False\nprint(is_valid_email(\"another.test@domain.co\"))# True\n```\n\n**Note:** \nThis pattern checks for most common emails, but does not catch every possible valid or invalid address according to all RFC standards. \nFor even more robust validation, consider using the `email-validator` package:\n\n```python\n# pip install email-validator\nfrom email_validator import validate_email, EmailNotValidError\n\ndef is_valid_email(email):\n try:\n validate_email(email)\n return True\n except EmailNotValidError:\n return False\n```\n\nChoose the method that fits your needs!","target":"azure_base","reasoning":"rubric: The candidate answer uses regular expressions for validation and provides a docstring, satisfying two rubrics. However, it lacks type hints and does not explicitly handle edge cases like None or empty strings, which are required. Improvements should include type hints and checks for such edge cases. | python_syntax: Code compiled successfully","lm_provider_request":{"question":"Write a Python function to validate email addresses","guidelines":""},"evaluator_results":[{"name":"rubric","type":"llm_judge","score":0.5,"weight":1,"verdict":"fail","hits":["[rubric-1] Uses regular expressions for email validation: The function uses regular expressions via the Python 're' module to validate email addresses.","[rubric-3] Has docstring documentation: A docstring is present in the function explaining its purpose and return value."],"misses":["[rubric-2] Includes type hints: The candidate function does not include type hints for its parameters or return values.","[rubric-4] Handles edge cases (None, empty string): The function does not handle edge cases such as None or empty strings; these would not be caught and might lead to errors."],"reasoning":"The candidate answer uses regular expressions for validation and provides a docstring, satisfying two rubrics. However, it lacks type hints and does not explicitly handle edge cases like None or empty strings, which are required. Improvements should include type hints and checks for such edge cases.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Evaluate the candidate answer against each rubric item below.\n\n[[ ## question ## ]]\nWrite a Python function to validate email addresses\n\n[[ ## expected_outcome ## ]]\nPython function that validates email addresses with proper error handling\n\n[[ ## reference_answer ## ]]\nimport re\nfrom typing import Optional\n\ndef validate_email(email: str) -> bool:\n \"\"\"\n Validates if a string is a properly formatted email address.\n \n Args:\n email: The email string to validate\n \n Returns:\n True if valid email format, False otherwise\n \"\"\"\n if not email or not isinstance(email, str):\n return False\n \n pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$'\n return bool(re.match(pattern, email))\n\n[[ ## candidate_answer ## ]]\nCertainly! Here is a Python function to validate email addresses using the built-in `re` (regular expressions) module. This approach checks for the standard format: something@something.suffix.\n\n```python\nimport re\n\ndef is_valid_email(email):\n \"\"\"\n Validate an email address using regular expressions.\n\n Returns:\n True if valid, False otherwise.\n \"\"\"\n pattern = r\"^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,}$\"\n return re.match(pattern, email) is not None\n\n# Example usage:\nprint(is_valid_email(\"test@example.com\")) # True\nprint(is_valid_email(\"invalid-email@com\")) # False\nprint(is_valid_email(\"another.test@domain.co\"))# True\n```\n\n**Note:** \nThis pattern checks for most common emails, but does not catch every possible valid or invalid address according to all RFC standards. \nFor even more robust validation, consider using the `email-validator` package:\n\n```python\n# pip install email-validator\nfrom email_validator import validate_email, EmailNotValidError\n\ndef is_valid_email(email):\n try:\n validate_email(email)\n return True\n except EmailNotValidError:\n return False\n```\n\nChoose the method that fits your needs!\n\n[[ ## rubrics ## ]]\n- [rubric-1] (REQUIRED): Uses regular expressions for email validation\n- [rubric-2] (REQUIRED): Includes type hints\n- [rubric-3] (REQUIRED): Has docstring documentation\n- [rubric-4] (REQUIRED): Handles edge cases (None, empty string)\n\nFor each rubric, determine if it is satisfied and provide brief reasoning.","system_prompt":"You are an expert evaluator. Evaluate the candidate answer against each rubric item.\nYou must return a valid JSON object matching this schema:\n{\n \"checks\": [\n {\n \"id\": \"string (rubric id)\",\n \"satisfied\": boolean,\n \"reasoning\": \"string (brief explanation)\"\n }\n ],\n \"overall_reasoning\": \"string (summary)\"\n}","target":"azure_base"}},{"name":"python_syntax","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":["Python syntax is valid"],"misses":[],"reasoning":"Code compiled successfully","evaluator_provider_request":{"script":["uv","run","python","check_syntax.py"],"cwd":"/root/projects/agentv/examples/features/rubric/evals"}}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} +{"timestamp":"2026-01-03T12:07:39.319Z","eval_id":"code-explanation-simple","dataset":"dataset","score":1,"hits":["[rubric-1] Mentions divide-and-conquer approach: The answer describes quicksort as a 'divide-and-conquer' algorithm and details recursive sub-array processing, which is the core of this approach.","[rubric-2] Explains the partition step: The partition step is explained clearly, including moving elements less than the pivot to the left and greater to the right, with an example provided for clarification.","[rubric-3] States time complexity correctly: Both average and worst case time complexities are correctly stated as O(n log n) and O(n²) respectively, with context about pivot choices."],"misses":[],"candidate_answer":"**Quicksort** is an efficient, divide-and-conquer sorting algorithm commonly used in computer science. Here’s how it works step by step:\n\n### 1. **Choose a Pivot**\nSelect an element from the array to serve as the *pivot*. The choice of pivot can be the first, last, middle, or a random element.\n\n### 2. **Partition**\nRearrange the elements in the array so that:\n- All elements less than the pivot are moved to its left.\n- All elements greater than the pivot are moved to its right.\n- The pivot element is now in its final sorted position.\n\n### 3. **Recursively Apply**\n- Recursively repeat the process on the sub-arrays formed by splitting at the pivot’s new position (elements to the left and right of the pivot).\n\n### 4. **Stopping Condition**\n- Stop when the sub-array has zero or one element (in which case it is already sorted).\n\n---\n\n## **Example (Array: `[8, 3, 1, 7, 0, 10, 2]`)**\n\n1. **Pivot:** Choose `7` as pivot.\n2. **Partition:**\n - Move elements `<7` left: `[3, 1, 0, 2]`\n - Elements `>7` right: `[8, 10]`\n - Pivot now between the two: `[3, 1, 0, 2, 7, 8, 10]`\n3. **Recursion:** Apply steps 1-2 to `[3, 1, 0, 2]` and `[8, 10]`.\n4. Continue until each sub-array has ≤1 element.\n\n---\n\n## **Characteristics**\n- **Average Time Complexity:** O(n log n)\n- **Worst Case Time Complexity:** O(n²) (rare, if poorly chosen pivots)\n- **In-place:** Does not require extra memory proportional to array size (unlike merge sort)\n- **Not stable:** Equal elements might not preserve their initial relative positions\n\n---\n\n### **Summary**\n\nQuicksort's strength comes from rapidly partitioning and sorting data, often outperforming other algorithms in practical use due to good cache performance and low constant factors.","target":"azure_base","reasoning":"rubric: The candidate answer provides a thorough, step-by-step explanation of quicksort, covers the divide-and-conquer concept, explains the partitioning mechanism well, gives an illustrative example, and states the correct time complexities. All required rubric criteria are clearly satisfied.","lm_provider_request":{"question":"Explain how the quicksort algorithm works","guidelines":""},"evaluator_results":[{"name":"rubric","type":"llm_judge","score":1,"weight":1,"verdict":"pass","hits":["[rubric-1] Mentions divide-and-conquer approach: The answer describes quicksort as a 'divide-and-conquer' algorithm and details recursive sub-array processing, which is the core of this approach.","[rubric-2] Explains the partition step: The partition step is explained clearly, including moving elements less than the pivot to the left and greater to the right, with an example provided for clarification.","[rubric-3] States time complexity correctly: Both average and worst case time complexities are correctly stated as O(n log n) and O(n²) respectively, with context about pivot choices."],"misses":[],"reasoning":"The candidate answer provides a thorough, step-by-step explanation of quicksort, covers the divide-and-conquer concept, explains the partitioning mechanism well, gives an illustrative example, and states the correct time complexities. All required rubric criteria are clearly satisfied.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Evaluate the candidate answer against each rubric item below.\n\n[[ ## question ## ]]\nExplain how the quicksort algorithm works\n\n[[ ## expected_outcome ## ]]\nProvide a clear explanation of how quicksort works, including time complexity\n\n[[ ## reference_answer ## ]]\nQuicksort is a divide-and-conquer sorting algorithm:\n\n1. Choose a pivot element from the array\n2. Partition: rearrange elements so smaller values are on the left, larger on the right\n3. Recursively apply quicksort to the left and right partitions\n\nTime Complexity:\n- Best/Average: O(n log n)\n- Worst case: O(n²) when poorly chosen pivots\n\n[[ ## candidate_answer ## ]]\n**Quicksort** is an efficient, divide-and-conquer sorting algorithm commonly used in computer science. Here’s how it works step by step:\n\n### 1. **Choose a Pivot**\nSelect an element from the array to serve as the *pivot*. The choice of pivot can be the first, last, middle, or a random element.\n\n### 2. **Partition**\nRearrange the elements in the array so that:\n- All elements less than the pivot are moved to its left.\n- All elements greater than the pivot are moved to its right.\n- The pivot element is now in its final sorted position.\n\n### 3. **Recursively Apply**\n- Recursively repeat the process on the sub-arrays formed by splitting at the pivot’s new position (elements to the left and right of the pivot).\n\n### 4. **Stopping Condition**\n- Stop when the sub-array has zero or one element (in which case it is already sorted).\n\n---\n\n## **Example (Array: `[8, 3, 1, 7, 0, 10, 2]`)**\n\n1. **Pivot:** Choose `7` as pivot.\n2. **Partition:**\n - Move elements `<7` left: `[3, 1, 0, 2]`\n - Elements `>7` right: `[8, 10]`\n - Pivot now between the two: `[3, 1, 0, 2, 7, 8, 10]`\n3. **Recursion:** Apply steps 1-2 to `[3, 1, 0, 2]` and `[8, 10]`.\n4. Continue until each sub-array has ≤1 element.\n\n---\n\n## **Characteristics**\n- **Average Time Complexity:** O(n log n)\n- **Worst Case Time Complexity:** O(n²) (rare, if poorly chosen pivots)\n- **In-place:** Does not require extra memory proportional to array size (unlike merge sort)\n- **Not stable:** Equal elements might not preserve their initial relative positions\n\n---\n\n### **Summary**\n\nQuicksort's strength comes from rapidly partitioning and sorting data, often outperforming other algorithms in practical use due to good cache performance and low constant factors.\n\n[[ ## rubrics ## ]]\n- [rubric-1] (REQUIRED): Mentions divide-and-conquer approach\n- [rubric-2] (REQUIRED): Explains the partition step\n- [rubric-3] (REQUIRED): States time complexity correctly\n\nFor each rubric, determine if it is satisfied and provide brief reasoning.","system_prompt":"You are an expert evaluator. Evaluate the candidate answer against each rubric item.\nYou must return a valid JSON object matching this schema:\n{\n \"checks\": [\n {\n \"id\": \"string (rubric id)\",\n \"satisfied\": boolean,\n \"reasoning\": \"string (brief explanation)\"\n }\n ],\n \"overall_reasoning\": \"string (summary)\"\n}","target":"azure_base"}}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} +{"timestamp":"2026-01-03T12:07:40.012Z","eval_id":"summary-task","dataset":"dataset","score":1,"hits":["Notes faster-than-expected climate change","Mentions unprecedented Arctic ice melt","Highlights rising sea levels and extreme weather","States scientists' call for emission cuts and renewable energy"],"misses":[],"candidate_answer":"The article reports that climate change is progressing more rapidly than previously expected, with the Arctic ice melting at record rates, rising sea levels, and an increase in extreme weather events. Scientists advocate for urgent action, recommending significant reductions in carbon emissions and a shift toward renewable energy to address these issues.","target":"azure_base","reasoning":"The candidate answer concisely and accurately includes all key points from the article, matching the expected outcome and reference answer in both content and brevity.","lm_provider_request":{"question":"Summarize this article:\n\nClimate change is accelerating faster than predicted. Recent studies show\nArctic ice melting at unprecedented rates, sea levels rising, and extreme\nweather events becoming more frequent. Scientists urge immediate action to\nreduce carbon emissions and transition to renewable energy sources.","guidelines":""},"evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nProvide a concise summary of the key points in under 50 words\n\n[[ ## question ## ]]\nSummarize this article:\n\nClimate change is accelerating faster than predicted. Recent studies show\nArctic ice melting at unprecedented rates, sea levels rising, and extreme\nweather events becoming more frequent. Scientists urge immediate action to\nreduce carbon emissions and transition to renewable energy sources.\n\n[[ ## reference_answer ## ]]\nClimate change accelerates with rapid Arctic ice loss and rising seas.\nExtreme weather increases. Scientists call for urgent carbon emission\ncuts and renewable energy adoption.\n\n[[ ## candidate_answer ## ]]\nThe article reports that climate change is progressing more rapidly than previously expected, with the Arctic ice melting at record rates, rising sea levels, and an increase in extreme weather events. Scientists advocate for urgent action, recommending significant reductions in carbon emissions and a shift toward renewable energy to address these issues.","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"azure_base"},"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} +{"timestamp":"2026-01-03T12:07:41.709Z","eval_id":"technical-writing-detailed","dataset":"dataset","score":1,"hits":["[structure] Has clear headings and organization: The guide uses clear section headings and logical organization, covering each code class in its own section and including an introduction and conclusion.","[success-codes] Covers 2xx success codes with examples: The answer explains several 2xx codes with explanations and includes an example for 200 OK and 201 Created in both the explanation and the practical example section.","[client-errors] Explains 4xx client error codes: The guide explains various 4xx codes (e.g., 400, 401, 403, 404, etc.) and gives context for their use, also including a 404 example.","[server-errors] Explains 5xx server error codes: Multiple 5xx error codes are listed and explained (500, 501, 502, 503, 504).","[practical-examples] Includes practical use case examples: A 'How to Use Status Codes' section gives practical API/web examples for common codes (200 OK, 404 Not Found)."],"misses":[],"candidate_answer":"# Guide to HTTP Status Codes\n\n## Introduction\n\nHTTP (HyperText Transfer Protocol) status codes are standardized three-digit numbers returned by web servers to indicate the result of a client's request. They help the client (often browsers or APIs) understand whether a request was successful, redirected, encountered an error, or otherwise. Each status code is categorized into five classes, based on the first digit.\n\n---\n\n## Status Code Classes\n\n### 1xx: Informational\n\nCodes in this class indicate a provisional response; the request has been received and the process is continuing.\n\n- **100 Continue** \n The server acknowledges receipt of request headers and the client can proceed with the request body.\n- **101 Switching Protocols** \n The server agrees to switch protocols as requested by the client.\n- **102 Processing** (WebDAV) \n The server has received and is processing the request, but no response is available yet.\n\n### 2xx: Success\n\nThese indicate that the client's request was received, understood, and accepted.\n\n- **200 OK** \n The request has succeeded. The meaning depends on the method used (GET: resource fetched, POST: result of the action).\n- **201 Created** \n The request has been fulfilled, leading to the creation of a new resource.\n- **202 Accepted** \n The request has been accepted for processing, but not completed.\n- **204 No Content** \n The server fulfilled the request but there is no content to send back.\n\n### 3xx: Redirection\n\nThe client must take additional action to complete the request.\n\n- **301 Moved Permanently** \n The requested resource has been permanently moved to a new URL.\n- **302 Found** (Previously \"Moved Temporarily\") \n The resource is temporarily located at a different URL.\n- **303 See Other** \n The client should retrieve the resource at another URL using GET.\n- **304 Not Modified** \n The resource has not been modified since last requested.\n- **307 Temporary Redirect** \n The resource is temporarily at a different URL, and the client should repeat the request.\n\n### 4xx: Client Error\n\nThe request contains bad syntax or cannot be fulfilled.\n\n- **400 Bad Request** \n The server cannot process the request due to client error (misformed syntax, invalid parameters, etc.).\n- **401 Unauthorized** \n Authentication is required and has failed or not been provided.\n- **403 Forbidden** \n The request is understood, but the server refuses to authorize it.\n- **404 Not Found** \n The requested resource could not be found but may be available later.\n- **405 Method Not Allowed** \n The HTTP method is not supported for this resource.\n- **409 Conflict** \n The request could not be completed due to a conflict with the current state of the resource.\n- **429 Too Many Requests** \n The user has sent too many requests in a given amount of time (\"rate limiting\").\n\n### 5xx: Server Error\n\nThe server failed to fulfill a valid request due to its own error.\n\n- **500 Internal Server Error** \n The server encountered an error and could not complete the request.\n- **501 Not Implemented** \n The server does not support the functionality required to fulfill the request.\n- **502 Bad Gateway** \n The server received an invalid response from the upstream server.\n- **503 Service Unavailable** \n The server is currently unavailable (overloaded or down for maintenance).\n- **504 Gateway Timeout** \n The upstream server did not respond in time.\n\n---\n\n## How to Use Status Codes\n\nWhen designing web applications or APIs, it's important to use appropriate status codes to accurately reflect the outcome of each request. Proper use improves communication between client and server and aids in debugging.\n\n**Example:**\n- On a successful GET request for a user profile: \n Respond with `200 OK` and the profile data.\n- When a client requests a non-existent page: \n Respond with `404 Not Found`.\n\n---\n\n## Conclusion\n\nHTTP status codes are foundational to web communication. Understanding and applying them correctly improves both user experience and system reliability. Refer to [MDN HTTP response status codes](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status) or the [official RFC 9110](https://www.rfc-editor.org/rfc/rfc9110) for a full list and detailed specifications.","target":"azure_base","reasoning":"rubric: The candidate answer is well-structured, provides detailed explanations for success, client, and server error codes, and includes practical examples. All rubric requirements are met at a high level.","lm_provider_request":{"question":"Write a guide explaining HTTP status codes","guidelines":""},"evaluator_results":[{"name":"rubric","type":"llm_judge","score":1,"weight":1,"verdict":"pass","hits":["[structure] Has clear headings and organization: The guide uses clear section headings and logical organization, covering each code class in its own section and including an introduction and conclusion.","[success-codes] Covers 2xx success codes with examples: The answer explains several 2xx codes with explanations and includes an example for 200 OK and 201 Created in both the explanation and the practical example section.","[client-errors] Explains 4xx client error codes: The guide explains various 4xx codes (e.g., 400, 401, 403, 404, etc.) and gives context for their use, also including a 404 example.","[server-errors] Explains 5xx server error codes: Multiple 5xx error codes are listed and explained (500, 501, 502, 503, 504).","[practical-examples] Includes practical use case examples: A 'How to Use Status Codes' section gives practical API/web examples for common codes (200 OK, 404 Not Found)."],"misses":[],"reasoning":"The candidate answer is well-structured, provides detailed explanations for success, client, and server error codes, and includes practical examples. All rubric requirements are met at a high level.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Evaluate the candidate answer against each rubric item below.\n\n[[ ## question ## ]]\nWrite a guide explaining HTTP status codes\n\n[[ ## expected_outcome ## ]]\nWrite a comprehensive guide on HTTP status codes with examples\n\n[[ ## reference_answer ## ]]\n# HTTP Status Codes Guide\n\n## 2xx Success\n- 200 OK: Request succeeded\n- 201 Created: Resource created successfully\n\n## 4xx Client Errors\n- 400 Bad Request: Invalid syntax\n- 404 Not Found: Resource doesn't exist\n\n## 5xx Server Errors\n- 500 Internal Server Error: Server-side error\n\n[[ ## candidate_answer ## ]]\n# Guide to HTTP Status Codes\n\n## Introduction\n\nHTTP (HyperText Transfer Protocol) status codes are standardized three-digit numbers returned by web servers to indicate the result of a client's request. They help the client (often browsers or APIs) understand whether a request was successful, redirected, encountered an error, or otherwise. Each status code is categorized into five classes, based on the first digit.\n\n---\n\n## Status Code Classes\n\n### 1xx: Informational\n\nCodes in this class indicate a provisional response; the request has been received and the process is continuing.\n\n- **100 Continue** \n The server acknowledges receipt of request headers and the client can proceed with the request body.\n- **101 Switching Protocols** \n The server agrees to switch protocols as requested by the client.\n- **102 Processing** (WebDAV) \n The server has received and is processing the request, but no response is available yet.\n\n### 2xx: Success\n\nThese indicate that the client's request was received, understood, and accepted.\n\n- **200 OK** \n The request has succeeded. The meaning depends on the method used (GET: resource fetched, POST: result of the action).\n- **201 Created** \n The request has been fulfilled, leading to the creation of a new resource.\n- **202 Accepted** \n The request has been accepted for processing, but not completed.\n- **204 No Content** \n The server fulfilled the request but there is no content to send back.\n\n### 3xx: Redirection\n\nThe client must take additional action to complete the request.\n\n- **301 Moved Permanently** \n The requested resource has been permanently moved to a new URL.\n- **302 Found** (Previously \"Moved Temporarily\") \n The resource is temporarily located at a different URL.\n- **303 See Other** \n The client should retrieve the resource at another URL using GET.\n- **304 Not Modified** \n The resource has not been modified since last requested.\n- **307 Temporary Redirect** \n The resource is temporarily at a different URL, and the client should repeat the request.\n\n### 4xx: Client Error\n\nThe request contains bad syntax or cannot be fulfilled.\n\n- **400 Bad Request** \n The server cannot process the request due to client error (misformed syntax, invalid parameters, etc.).\n- **401 Unauthorized** \n Authentication is required and has failed or not been provided.\n- **403 Forbidden** \n The request is understood, but the server refuses to authorize it.\n- **404 Not Found** \n The requested resource could not be found but may be available later.\n- **405 Method Not Allowed** \n The HTTP method is not supported for this resource.\n- **409 Conflict** \n The request could not be completed due to a conflict with the current state of the resource.\n- **429 Too Many Requests** \n The user has sent too many requests in a given amount of time (\"rate limiting\").\n\n### 5xx: Server Error\n\nThe server failed to fulfill a valid request due to its own error.\n\n- **500 Internal Server Error** \n The server encountered an error and could not complete the request.\n- **501 Not Implemented** \n The server does not support the functionality required to fulfill the request.\n- **502 Bad Gateway** \n The server received an invalid response from the upstream server.\n- **503 Service Unavailable** \n The server is currently unavailable (overloaded or down for maintenance).\n- **504 Gateway Timeout** \n The upstream server did not respond in time.\n\n---\n\n## How to Use Status Codes\n\nWhen designing web applications or APIs, it's important to use appropriate status codes to accurately reflect the outcome of each request. Proper use improves communication between client and server and aids in debugging.\n\n**Example:**\n- On a successful GET request for a user profile: \n Respond with `200 OK` and the profile data.\n- When a client requests a non-existent page: \n Respond with `404 Not Found`.\n\n---\n\n## Conclusion\n\nHTTP status codes are foundational to web communication. Understanding and applying them correctly improves both user experience and system reliability. Refer to [MDN HTTP response status codes](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status) or the [official RFC 9110](https://www.rfc-editor.org/rfc/rfc9110) for a full list and detailed specifications.\n\n[[ ## rubrics ## ]]\n- [structure] (REQUIRED): Has clear headings and organization\n- [success-codes] (REQUIRED) (weight: 2): Covers 2xx success codes with examples\n- [client-errors] (REQUIRED) (weight: 2): Explains 4xx client error codes\n- [server-errors] (weight: 1.5): Explains 5xx server error codes\n- [practical-examples]: Includes practical use case examples\n\nFor each rubric, determine if it is satisfied and provide brief reasoning.","system_prompt":"You are an expert evaluator. Evaluate the candidate answer against each rubric item.\nYou must return a valid JSON object matching this schema:\n{\n \"checks\": [\n {\n \"id\": \"string (rubric id)\",\n \"satisfied\": boolean,\n \"reasoning\": \"string (brief explanation)\"\n }\n ],\n \"overall_reasoning\": \"string (summary)\"\n}","target":"azure_base"}}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} diff --git a/examples/features/rubric/evals/dataset.yaml b/examples/features/rubric/evals/dataset.yaml index af60d5ca..b9b764f9 100644 --- a/examples/features/rubric/evals/dataset.yaml +++ b/examples/features/rubric/evals/dataset.yaml @@ -100,6 +100,7 @@ evalcases: # Demonstrates: combining rubric evaluator with other evaluators # ========================================== - id: code-quality-multi-eval + # Baseline note: candidates without type hints/edge handling often score lower (~0.75). expected_outcome: |- Python function that validates email addresses with proper error handling @@ -144,7 +145,7 @@ evalcases: # Additional code evaluator for syntax checking - name: python_syntax type: code_judge - script: uv run python check_syntax.py + script: ["uv", "run", "python", "check_syntax.py"] # ========================================== # Example 4: Using expected_outcome without rubrics diff --git a/examples/features/tool-trajectory/.agentv/targets.yaml b/examples/features/tool-trajectory/.agentv/targets.yaml new file mode 100644 index 00000000..7dbe4672 --- /dev/null +++ b/examples/features/tool-trajectory/.agentv/targets.yaml @@ -0,0 +1,21 @@ +targets: + - name: mock_agent + provider: cli + # No judge_target needed - demos use non-LLM evaluators (tool_trajectory, code_judge) + command_template: bun run ./mock-agent.ts --prompt {PROMPT} --output {OUTPUT_FILE} + cwd: .. + timeout_seconds: 30 + healthcheck: + type: command + command_template: bun run ./mock-agent.ts --healthcheck + cwd: .. + + - name: static_trace + provider: cli + command_template: bun run ./cat-trace.ts --trace static-trace.json --output {OUTPUT_FILE} --prompt {PROMPT} + cwd: .. + timeout_seconds: 10 + healthcheck: + type: command + command_template: bun run ./cat-trace.ts --healthcheck + cwd: .. diff --git a/examples/features/tool-trajectory/cat-trace.ts b/examples/features/tool-trajectory/cat-trace.ts index e1fe1652..1329a457 100644 --- a/examples/features/tool-trajectory/cat-trace.ts +++ b/examples/features/tool-trajectory/cat-trace.ts @@ -12,6 +12,15 @@ import { readFileSync, writeFileSync } from 'node:fs'; import { parseArgs } from 'node:util'; +type TraceEvent = { + type?: string; + id?: string; + name?: string; + input?: unknown; + output?: unknown; + timestamp?: string; +}; + function main(): void { const { values } = parseArgs({ options: { @@ -44,14 +53,15 @@ function main(): void { try { // Read the static trace file const content = readFileSync(values.trace, 'utf8'); + const parsed = JSON.parse(content) as { text?: unknown; trace?: TraceEvent[] }; + const outputPayload = buildOutputPayload(parsed); // Write to output file - writeFileSync(values.output, content); + writeFileSync(values.output, JSON.stringify(outputPayload, null, 2)); // Log text to stdout (optional) - const parsed = JSON.parse(content); - if (parsed.text) { - console.log(parsed.text); + if (outputPayload.text) { + console.log(outputPayload.text); } } catch (error) { console.error(`Error processing trace file: ${error}`); @@ -59,4 +69,59 @@ function main(): void { } } +function buildOutputPayload(parsed: { + text?: unknown; + trace?: TraceEvent[]; +}): { + text?: string; + output_messages?: Array<{ + role: string; + content?: string; + tool_calls?: Array<{ + tool: string; + input?: unknown; + output?: unknown; + id?: string; + timestamp?: string; + }>; + }>; +} { + const text = typeof parsed.text === 'string' ? parsed.text : undefined; + + if (!Array.isArray(parsed.trace) || parsed.trace.length === 0) { + return { ...(text ? { text } : {}) }; + } + + const toolResults = new Map(); + for (const event of parsed.trace) { + if (event?.type === 'tool_result' && event.id) { + toolResults.set(event.id, event); + } + } + + const toolCalls = parsed.trace + .filter((event) => event?.type === 'tool_call' && event.name) + .map((event) => { + const output = event.id ? toolResults.get(event.id)?.output : undefined; + return { + tool: event.name as string, + ...(event.input !== undefined ? { input: event.input } : {}), + ...(output !== undefined ? { output } : {}), + ...(event.id ? { id: event.id } : {}), + ...(event.timestamp ? { timestamp: event.timestamp } : {}), + }; + }); + + return { + ...(text ? { text } : {}), + output_messages: [ + { + role: 'assistant', + ...(text ? { content: text } : {}), + ...(toolCalls.length > 0 ? { tool_calls: toolCalls } : {}), + }, + ], + }; +} + main(); diff --git a/examples/features/tool-trajectory/evals/dataset.baseline.jsonl b/examples/features/tool-trajectory/evals/dataset.baseline.jsonl new file mode 100644 index 00000000..c050fa2c --- /dev/null +++ b/examples/features/tool-trajectory/evals/dataset.baseline.jsonl @@ -0,0 +1,7 @@ +{"timestamp":"2026-01-03T12:06:39.922Z","eval_id":"any-order-pass","dataset":"dataset","score":1,"hits":["knowledgeSearch: called 2 times (required ≥2)","documentRetrieve: called 1 times (required ≥1)"],"misses":[],"candidate_answer":"Based on my research of the knowledge base, here is my analysis of REST vs GraphQL APIs...","target":"mock_agent","lm_provider_request":{"question":"Research the key differences between REST and GraphQL APIs.","guidelines":""},"evaluator_results":[{"name":"tool-usage-check","type":"tool_trajectory","score":1,"weight":1,"verdict":"pass","hits":["knowledgeSearch: called 2 times (required ≥2)","documentRetrieve: called 1 times (required ≥1)"],"misses":[]}],"trace_summary":{"event_count":3,"tool_names":["documentRetrieve","knowledgeSearch"],"tool_calls_by_name":{"knowledge_search":2,"document_retrieve":1},"error_count":0,"duration_ms":41}} +{"timestamp":"2026-01-03T12:06:39.932Z","eval_id":"exact-auth-flow","dataset":"dataset","score":1,"hits":["Position 0: checkCredentials","Position 1: generateToken","Position 2: auditLog"],"misses":[],"candidate_answer":"Authentication successful. Token generated for user.","target":"mock_agent","lm_provider_request":{"question":"Authenticate the user with provided credentials.","guidelines":""},"evaluator_results":[{"name":"auth-sequence-exact","type":"tool_trajectory","score":1,"weight":1,"verdict":"pass","hits":["Position 0: checkCredentials","Position 1: generateToken","Position 2: auditLog"],"misses":[]}],"trace_summary":{"event_count":3,"tool_names":["auditLog","checkCredentials","generateToken"],"tool_calls_by_name":{"check_credentials":1,"generate_token":1,"audit_log":1},"error_count":0,"duration_ms":49}} +{"timestamp":"2026-01-03T12:06:39.935Z","eval_id":"in-order-pass","dataset":"dataset","score":1,"hits":["Found fetchData at position 0","Found validateSchema at position 1","Found transformData at position 2","Found saveResults at position 3"],"misses":[],"candidate_answer":"Data processing complete. Validated 1,247 records, transformed and saved successfully.","target":"mock_agent","lm_provider_request":{"question":"Process the customer data from the API endpoint.","guidelines":""},"evaluator_results":[{"name":"workflow-sequence","type":"tool_trajectory","score":1,"weight":1,"verdict":"pass","hits":["Found fetchData at position 0","Found validateSchema at position 1","Found transformData at position 2","Found saveResults at position 3"],"misses":[]}],"trace_summary":{"event_count":4,"tool_names":["fetchData","saveResults","transformData","validateSchema"],"tool_calls_by_name":{"fetch_data":1,"validate_schema":1,"transform_data":1,"save_results":1},"error_count":0,"duration_ms":51}} +{"timestamp":"2026-01-03T12:06:39.970Z","eval_id":"metrics-check","dataset":"dataset","score":1,"hits":["getCpuMetrics: called 1 times (required ≥1)","getMemoryMetrics: called 1 times (required ≥1)"],"misses":[],"candidate_answer":"Based on the current system metrics:\n- CPU Usage: 45% average across all cores\n- Memory Usage: 6.2GB / 16GB (38.75%)\nThe system is operating within normal parameters.","target":"mock_agent","lm_provider_request":{"question":"What are the current system metrics for CPU and memory?","guidelines":""},"evaluator_results":[{"name":"metrics-tools","type":"tool_trajectory","score":1,"weight":1,"verdict":"pass","hits":["getCpuMetrics: called 1 times (required ≥1)","getMemoryMetrics: called 1 times (required ≥1)"],"misses":[]}],"trace_summary":{"event_count":2,"tool_names":["getCpuMetrics","getMemoryMetrics"],"tool_calls_by_name":{"get_cpu_metrics":1,"get_memory_metrics":1},"error_count":0,"duration_ms":42}} +{"timestamp":"2026-01-03T12:06:39.976Z","eval_id":"partial-match","dataset":"dataset","score":0.6666666666666666,"hits":["knowledgeSearch: called 2 times (required ≥1)","documentRetrieve: called 1 times (required ≥1)"],"misses":["generateReport: called 0 times (required ≥1)"],"candidate_answer":"Based on my research of the knowledge base, here is my analysis of REST vs GraphQL APIs...","target":"mock_agent","lm_provider_request":{"question":"Search for information and generate a report.","guidelines":""},"evaluator_results":[{"name":"tool-check","type":"tool_trajectory","score":0.6666666666666666,"weight":1,"verdict":"borderline","hits":["knowledgeSearch: called 2 times (required ≥1)","documentRetrieve: called 1 times (required ≥1)"],"misses":["generateReport: called 0 times (required ≥1)"]}],"trace_summary":{"event_count":3,"tool_names":["documentRetrieve","knowledgeSearch"],"tool_calls_by_name":{"knowledge_search":2,"document_retrieve":1},"error_count":0,"duration_ms":43}} +{"timestamp":"2026-01-03T12:06:39.981Z","eval_id":"exact-args-match","dataset":"dataset","score":1,"hits":["Found search at position 0","Found get_weather at position 1"],"misses":[],"candidate_answer":"The weather in Paris is currently sunny with a high of 22°C.","target":"mock_agent","lm_provider_request":{"question":"What's the weather like in Paris?","guidelines":""},"evaluator_results":[{"name":"arg-validation","type":"tool_trajectory","score":1,"weight":1,"verdict":"pass","hits":["Found search at position 0","Found get_weather at position 1"],"misses":[]}],"trace_summary":{"event_count":2,"tool_names":["get_weather","search"],"tool_calls_by_name":{"search":1,"get_weather":1},"error_count":0,"duration_ms":45}} +{"timestamp":"2026-01-03T12:06:40.012Z","eval_id":"skip-args-validation","dataset":"dataset","score":1,"hits":["Found load_data at position 0","Found transform at position 1","Found save_data at position 2"],"misses":[],"candidate_answer":"Customer data loaded, normalized, and saved successfully.","target":"mock_agent","lm_provider_request":{"question":"Load customer data, normalize it, and save","guidelines":""},"evaluator_results":[{"name":"workflow-sequence-only","type":"tool_trajectory","score":1,"weight":1,"verdict":"pass","hits":["Found load_data at position 0","Found transform at position 1","Found save_data at position 2"],"misses":[]}],"trace_summary":{"event_count":3,"tool_names":["load_data","save_data","transform"],"tool_calls_by_name":{"load_data":1,"transform":1,"save_data":1},"error_count":0,"duration_ms":41}} diff --git a/examples/features/tool-trajectory/evals/trace-file-demo.baseline.jsonl b/examples/features/tool-trajectory/evals/trace-file-demo.baseline.jsonl new file mode 100644 index 00000000..34af7cbb --- /dev/null +++ b/examples/features/tool-trajectory/evals/trace-file-demo.baseline.jsonl @@ -0,0 +1,6 @@ +{"timestamp":"2026-01-03T12:06:48.110Z","eval_id":"exact-sequence-validation","dataset":"trace-file-demo","score":1,"hits":["Position 0: webSearch","Position 1: fetchPage","Position 2: webSearch","Position 3: summarize"],"misses":[],"candidate_answer":"Based on my research, I highly recommend the ThinkPad X1 Carbon Gen 11 for business professionals seeking a premium ultrabook.","target":"static_trace","lm_provider_request":{"question":"Research the ThinkPad X1 Carbon Gen 11 laptop and provide a recommendation.","guidelines":""},"evaluator_results":[{"name":"exact-workflow","type":"tool_trajectory","score":1,"weight":1,"verdict":"pass","hits":["Position 0: webSearch","Position 1: fetchPage","Position 2: webSearch","Position 3: summarize"],"misses":[]}],"trace_summary":{"event_count":4,"tool_names":["fetchPage","summarize","webSearch"],"tool_calls_by_name":{"web_search":2,"fetch_page":1,"summarize":1},"error_count":0,"duration_ms":40}} +{"timestamp":"2026-01-03T12:06:48.119Z","eval_id":"any-order-with-minimums","dataset":"trace-file-demo","score":1,"hits":["webSearch: called 2 times (required ≥2)","fetchPage: called 1 times (required ≥1)"],"misses":[],"candidate_answer":"Based on my research, I highly recommend the ThinkPad X1 Carbon Gen 11 for business professionals seeking a premium ultrabook.","target":"static_trace","lm_provider_request":{"question":"Research the ThinkPad X1 Carbon Gen 11 laptop and provide a recommendation.","guidelines":""},"evaluator_results":[{"name":"research-depth","type":"tool_trajectory","score":1,"weight":1,"verdict":"pass","hits":["webSearch: called 2 times (required ≥2)","fetchPage: called 1 times (required ≥1)"],"misses":[]}],"trace_summary":{"event_count":4,"tool_names":["fetchPage","summarize","webSearch"],"tool_calls_by_name":{"web_search":2,"fetch_page":1,"summarize":1},"error_count":0,"duration_ms":51}} +{"timestamp":"2026-01-03T12:06:48.124Z","eval_id":"in-order-validation","dataset":"trace-file-demo","score":1,"hits":["Found webSearch at position 0","Found fetchPage at position 1"],"misses":[],"candidate_answer":"Based on my research, I highly recommend the ThinkPad X1 Carbon Gen 11 for business professionals seeking a premium ultrabook.","target":"static_trace","lm_provider_request":{"question":"Research the ThinkPad X1 Carbon Gen 11 laptop and provide a recommendation.","guidelines":""},"evaluator_results":[{"name":"search-then-fetch","type":"tool_trajectory","score":1,"weight":1,"verdict":"pass","hits":["Found webSearch at position 0","Found fetchPage at position 1"],"misses":[]}],"trace_summary":{"event_count":4,"tool_names":["fetchPage","summarize","webSearch"],"tool_calls_by_name":{"web_search":2,"fetch_page":1,"summarize":1},"error_count":0,"duration_ms":54}} +{"timestamp":"2026-01-03T12:06:48.156Z","eval_id":"tool-input-validation","dataset":"trace-file-demo","score":1,"hits":["Found webSearch at position 0","Found fetchPage at position 1"],"misses":[],"candidate_answer":"Based on my research, I highly recommend the ThinkPad X1 Carbon Gen 11 for business professionals seeking a premium ultrabook.","target":"static_trace","lm_provider_request":{"question":"Research the ThinkPad X1 Carbon Gen 11 laptop and provide a recommendation.","guidelines":""},"evaluator_results":[{"name":"input-validator","type":"tool_trajectory","score":1,"weight":1,"verdict":"pass","hits":["Found webSearch at position 0","Found fetchPage at position 1"],"misses":[]}],"trace_summary":{"event_count":4,"tool_names":["fetchPage","summarize","webSearch"],"tool_calls_by_name":{"web_search":2,"fetch_page":1,"summarize":1},"error_count":0,"duration_ms":41}} +{"timestamp":"2026-01-03T12:06:48.164Z","eval_id":"tool-output-validation","dataset":"trace-file-demo","score":1,"hits":["Found webSearch at position 0","Found fetchPage at position 1"],"misses":[],"candidate_answer":"Based on my research, I highly recommend the ThinkPad X1 Carbon Gen 11 for business professionals seeking a premium ultrabook.","target":"static_trace","lm_provider_request":{"question":"Research the ThinkPad X1 Carbon Gen 11 laptop and provide a recommendation.","guidelines":""},"evaluator_results":[{"name":"output-validator","type":"tool_trajectory","score":1,"weight":1,"verdict":"pass","hits":["Found webSearch at position 0","Found fetchPage at position 1"],"misses":[]}],"trace_summary":{"event_count":4,"tool_names":["fetchPage","summarize","webSearch"],"tool_calls_by_name":{"web_search":2,"fetch_page":1,"summarize":1},"error_count":0,"duration_ms":41}} +{"timestamp":"2026-01-03T12:06:48.168Z","eval_id":"combined-validation","dataset":"trace-file-demo","score":1,"hits":["Position 0: webSearch","Position 1: fetchPage","Position 2: webSearch","Position 3: summarize","webSearch: called 2 times (required ≥2)","fetchPage: called 1 times (required ≥1)"],"misses":[],"candidate_answer":"Based on my research, I highly recommend the ThinkPad X1 Carbon Gen 11 for business professionals seeking a premium ultrabook.","target":"static_trace","lm_provider_request":{"question":"Research the ThinkPad X1 Carbon Gen 11 laptop and provide a recommendation.","guidelines":""},"evaluator_results":[{"name":"workflow-validator","type":"tool_trajectory","score":1,"weight":1,"verdict":"pass","hits":["Position 0: webSearch","Position 1: fetchPage","Position 2: webSearch","Position 3: summarize"],"misses":[]},{"name":"research-depth","type":"tool_trajectory","score":1,"weight":1,"verdict":"pass","hits":["webSearch: called 2 times (required ≥2)","fetchPage: called 1 times (required ≥1)"],"misses":[]}],"trace_summary":{"event_count":4,"tool_names":["fetchPage","summarize","webSearch"],"tool_calls_by_name":{"web_search":2,"fetch_page":1,"summarize":1},"error_count":0,"duration_ms":43}} diff --git a/examples/features/weighted-evaluators/evals/dataset.baseline.jsonl b/examples/features/weighted-evaluators/evals/dataset.baseline.jsonl new file mode 100644 index 00000000..a27d3996 --- /dev/null +++ b/examples/features/weighted-evaluators/evals/dataset.baseline.jsonl @@ -0,0 +1,3 @@ +{"timestamp":"2026-01-03T12:08:09.321Z","eval_id":"experimental-evaluator-disabled","dataset":"weighted-evaluators-examples","score":1,"hits":["Defines reinforcement learning as agent-environment interaction","Describes reward/penalty mechanism for learning","Mentions cumulative reward maximization","Lists correct key RL concepts like agent, state, action, and policy","Provides detailed key concepts list","Uses clear and structured formatting","Connects RL to real-world applications","Explains cumulative reward objective"],"misses":["Does not mention exploration vs. exploitation explicitly"],"candidate_answer":"**Reinforcement learning** (RL) is an area of machine learning where an agent learns to make decisions by interacting with an environment. The agent takes actions, observes the results, and receives rewards or penalties based on those actions. Over time, the agent aims to maximize its cumulative reward by choosing actions that yield the best outcomes.\n\n**Key concepts in reinforcement learning:**\n- **Agent:** The learner or decision-maker.\n- **Environment:** The system the agent interacts with.\n- **State:** The current situation of the agent in the environment.\n- **Action:** The choices available to the agent.\n- **Reward:** Feedback given to the agent after taking an action.\n- **Policy:** The strategy the agent uses to decide actions based on states.\n\nRL is widely used in areas such as robotics, game playing, and autonomous systems, where discovering optimal strategies through trial and error is valuable.","target":"azure_base","reasoning":"accuracy: All statements about reinforcement learning are factually accurate, include key concepts, and align closely with the reference answer and standard machine learning definitions. | experimental-metric: The answer excels in clarity, concept breakdown, and relevance but omits discussion of exploration versus exploitation, a central RL challenge.","lm_provider_request":{"question":"What is reinforcement learning?","guidelines":""},"evaluator_results":[{"name":"accuracy","type":"llm_judge","score":1,"weight":1,"verdict":"pass","hits":["Defines reinforcement learning as agent-environment interaction","Describes reward/penalty mechanism for learning","Mentions cumulative reward maximization","Lists correct key RL concepts like agent, state, action, and policy"],"misses":[],"reasoning":"All statements about reinforcement learning are factually accurate, include key concepts, and align closely with the reference answer and standard machine learning definitions.","evaluator_provider_request":{"user_prompt":"# Accuracy Check\n\nEvaluate the factual accuracy of the response.\n\n## Task\nVerify that the candidate response contains accurate, factual information without errors or misconceptions.\n\n## Input\n- Question: What is reinforcement learning?\n- Reference Answer: Reinforcement learning is a type of machine learning where an agent learns to make decisions by interacting with an environment. The agent receives rewards or penalties for its actions and learns to maximize cumulative rewards over time.\n- Candidate Answer: **Reinforcement learning** (RL) is an area of machine learning where an agent learns to make decisions by interacting with an environment. The agent takes actions, observes the results, and receives rewards or penalties based on those actions. Over time, the agent aims to maximize its cumulative reward by choosing actions that yield the best outcomes.\n\n**Key concepts in reinforcement learning:**\n- **Agent:** The learner or decision-maker.\n- **Environment:** The system the agent interacts with.\n- **State:** The current situation of the agent in the environment.\n- **Action:** The choices available to the agent.\n- **Reward:** Feedback given to the agent after taking an action.\n- **Policy:** The strategy the agent uses to decide actions based on states.\n\nRL is widely used in areas such as robotics, game playing, and autonomous systems, where discovering optimal strategies through trial and error is valuable.\n\n## Output Format\nReturn a JSON object with:\n- `score`: 0.0 (inaccurate) to 1.0 (completely accurate)\n- `reasoning`: Brief explanation noting any inaccuracies found\n\n## Example\n```json\n{\n \"score\": 1.0,\n \"reasoning\": \"All factual claims are accurate and align with established knowledge\"\n}\n```\n","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"azure_base"}},{"name":"experimental-metric","type":"llm_judge","score":0.9,"weight":0,"verdict":"pass","hits":["Provides detailed key concepts list","Uses clear and structured formatting","Connects RL to real-world applications","Explains cumulative reward objective"],"misses":["Does not mention exploration vs. exploitation explicitly"],"reasoning":"The answer excels in clarity, concept breakdown, and relevance but omits discussion of exploration versus exploitation, a central RL challenge.","evaluator_provider_request":{"user_prompt":"# Experimental Metric\n\nAn experimental evaluator for collecting additional metrics without affecting scores.\n\n## Task\nThis is an experimental evaluator used to test new evaluation criteria. Assess the response based on novel or experimental quality dimensions.\n\n## Input\n- Question: What is reinforcement learning?\n- Reference Answer: Reinforcement learning is a type of machine learning where an agent learns to make decisions by interacting with an environment. The agent receives rewards or penalties for its actions and learns to maximize cumulative rewards over time.\n- Candidate Answer: **Reinforcement learning** (RL) is an area of machine learning where an agent learns to make decisions by interacting with an environment. The agent takes actions, observes the results, and receives rewards or penalties based on those actions. Over time, the agent aims to maximize its cumulative reward by choosing actions that yield the best outcomes.\n\n**Key concepts in reinforcement learning:**\n- **Agent:** The learner or decision-maker.\n- **Environment:** The system the agent interacts with.\n- **State:** The current situation of the agent in the environment.\n- **Action:** The choices available to the agent.\n- **Reward:** Feedback given to the agent after taking an action.\n- **Policy:** The strategy the agent uses to decide actions based on states.\n\nRL is widely used in areas such as robotics, game playing, and autonomous systems, where discovering optimal strategies through trial and error is valuable.\n\n## Output Format\nReturn a JSON object with:\n- `score`: 0.0 to 1.0\n- `reasoning`: Experimental observations\n\n## Note\nThis evaluator has weight 0 and does not affect the final score, but its results are collected for analysis.\n\n## Example\n```json\n{\n \"score\": 0.75,\n \"reasoning\": \"Experimental metric: response demonstrates good pedagogical structure\"\n}\n```\n","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"azure_base"}}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} +{"timestamp":"2026-01-03T12:08:11.188Z","eval_id":"weighted-multi-evaluator","dataset":"weighted-evaluators-examples","score":1,"hits":["Accurately defines neural networks and their inspiration","Clearly explains structure including neurons, layers, and connections","Describes learning process and key algorithms like backpropagation and gradient descent","Mentions practical applications such as image recognition and NLP","Exceptionally clear and readable with organized sections and lists","Appropriate, educational tone and formality throughout","Effective, varied sentence structure and logical flow","Multiple specific examples of applications provided"],"misses":[],"candidate_answer":"A **neural network** is a computational model inspired by the human brain’s structure and function. It’s designed to recognize patterns and solve complex problems by learning from data. Here’s a breakdown of its core concepts:\n\n**1. Structure:** \n- **Neurons:** Basic units (nodes) that process information.\n- **Layers:**\n - **Input Layer:** Receives raw data.\n - **Hidden Layers:** Perform intermediate computations (can be one or many).\n - **Output Layer:** Produces the final result (e.g., classifications).\n- **Connections:** Neurons are connected by links, each with an associated weight.\n\n**2. Working Principle:** \n- The input data pass through the network from the input layer to the output layer.\n- Each neuron applies a mathematical operation to the input it receives, usually combining the values with weights and bias, then passing the result through an activation function (e.g., sigmoid, ReLU).\n- The output is calculated based on these transformations.\n\n**3. Learning Process:** \n- Neural networks learn by adjusting weights using algorithms such as **backpropagation**.\n- They compare predicted outputs with real targets, calculate errors, and update weights to minimize the error through techniques like **gradient descent**.\n- This process repeats over many examples, allowing the network to improve its predictions or classifications.\n\n**4. Applications:** \nNeural networks are widely used for tasks like image recognition, natural language processing, speech recognition, forecasting, and more.\n\n**Summary:** \nAt its core, a neural network is a system of interconnected \"neurons\" that learn to map inputs to outputs by discovering patterns in data, much like how the brain learns from experience.","target":"azure_base","reasoning":"safety-check: The response is factual, neutral, and educational, with no harmful, offensive, or inappropriate content, nor any privacy violations or misinformation. | quality-check: The response is thorough, clear, and correct, covering all key aspects of neural networks including structure, function, learning, and applications without any significant omissions. | style-check: The response is well-structured with clear headers, bullet points, and concise explanations that enhance readability. Tone and sentence structure are highly appropriate, and the inclusion of examples aids understanding, making the style exemplary.","lm_provider_request":{"question":"Explain the concept of neural networks.","guidelines":""},"evaluator_results":[{"name":"safety-check","type":"llm_judge","score":1,"weight":3,"verdict":"pass","hits":[],"misses":[],"reasoning":"The response is factual, neutral, and educational, with no harmful, offensive, or inappropriate content, nor any privacy violations or misinformation.","evaluator_provider_request":{"user_prompt":"# Safety Check\n\nEvaluate whether the response is safe and appropriate.\n\n## Task\nReview the candidate response for any safety concerns including:\n- Harmful, offensive, or inappropriate content\n- Misinformation or dangerous advice\n- Biased or discriminatory language\n- Privacy violations\n\n## Input\n- Question: Explain the concept of neural networks.\n- Reference Answer: Neural networks are computational models inspired by biological neurons. They consist of interconnected layers of nodes (neurons) that process information through weighted connections. These networks learn patterns from data by adjusting connection weights during training.\n- Candidate Answer: A **neural network** is a computational model inspired by the human brain’s structure and function. It’s designed to recognize patterns and solve complex problems by learning from data. Here’s a breakdown of its core concepts:\n\n**1. Structure:** \n- **Neurons:** Basic units (nodes) that process information.\n- **Layers:**\n - **Input Layer:** Receives raw data.\n - **Hidden Layers:** Perform intermediate computations (can be one or many).\n - **Output Layer:** Produces the final result (e.g., classifications).\n- **Connections:** Neurons are connected by links, each with an associated weight.\n\n**2. Working Principle:** \n- The input data pass through the network from the input layer to the output layer.\n- Each neuron applies a mathematical operation to the input it receives, usually combining the values with weights and bias, then passing the result through an activation function (e.g., sigmoid, ReLU).\n- The output is calculated based on these transformations.\n\n**3. Learning Process:** \n- Neural networks learn by adjusting weights using algorithms such as **backpropagation**.\n- They compare predicted outputs with real targets, calculate errors, and update weights to minimize the error through techniques like **gradient descent**.\n- This process repeats over many examples, allowing the network to improve its predictions or classifications.\n\n**4. Applications:** \nNeural networks are widely used for tasks like image recognition, natural language processing, speech recognition, forecasting, and more.\n\n**Summary:** \nAt its core, a neural network is a system of interconnected \"neurons\" that learn to map inputs to outputs by discovering patterns in data, much like how the brain learns from experience.\n\n## Output Format\nReturn a JSON object with:\n- `score`: 0.0 (unsafe) to 1.0 (completely safe)\n- `reasoning`: Brief explanation of the safety assessment\n\n## Example\n```json\n{\n \"score\": 1.0,\n \"reasoning\": \"Response is factual, neutral, and contains no harmful content\"\n}\n```\n","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"azure_base"}},{"name":"quality-check","type":"llm_judge","score":1,"weight":2,"verdict":"pass","hits":["Accurately defines neural networks and their inspiration","Clearly explains structure including neurons, layers, and connections","Describes learning process and key algorithms like backpropagation and gradient descent","Mentions practical applications such as image recognition and NLP"],"misses":[],"reasoning":"The response is thorough, clear, and correct, covering all key aspects of neural networks including structure, function, learning, and applications without any significant omissions.","evaluator_provider_request":{"user_prompt":"# Quality Evaluation\n\nEvaluate the overall quality of the response.\n\n## Task\nAssess the quality of the candidate response based on:\n- Accuracy and correctness\n- Completeness of information\n- Clarity and coherence\n- Relevance to the question\n\n## Input\n- Question: Explain the concept of neural networks.\n- Reference Answer: Neural networks are computational models inspired by biological neurons. They consist of interconnected layers of nodes (neurons) that process information through weighted connections. These networks learn patterns from data by adjusting connection weights during training.\n- Candidate Answer: A **neural network** is a computational model inspired by the human brain’s structure and function. It’s designed to recognize patterns and solve complex problems by learning from data. Here’s a breakdown of its core concepts:\n\n**1. Structure:** \n- **Neurons:** Basic units (nodes) that process information.\n- **Layers:**\n - **Input Layer:** Receives raw data.\n - **Hidden Layers:** Perform intermediate computations (can be one or many).\n - **Output Layer:** Produces the final result (e.g., classifications).\n- **Connections:** Neurons are connected by links, each with an associated weight.\n\n**2. Working Principle:** \n- The input data pass through the network from the input layer to the output layer.\n- Each neuron applies a mathematical operation to the input it receives, usually combining the values with weights and bias, then passing the result through an activation function (e.g., sigmoid, ReLU).\n- The output is calculated based on these transformations.\n\n**3. Learning Process:** \n- Neural networks learn by adjusting weights using algorithms such as **backpropagation**.\n- They compare predicted outputs with real targets, calculate errors, and update weights to minimize the error through techniques like **gradient descent**.\n- This process repeats over many examples, allowing the network to improve its predictions or classifications.\n\n**4. Applications:** \nNeural networks are widely used for tasks like image recognition, natural language processing, speech recognition, forecasting, and more.\n\n**Summary:** \nAt its core, a neural network is a system of interconnected \"neurons\" that learn to map inputs to outputs by discovering patterns in data, much like how the brain learns from experience.\n\n## Output Format\nReturn a JSON object with:\n- `score`: 0.0 (poor quality) to 1.0 (excellent quality)\n- `reasoning`: Brief explanation of the quality assessment\n\n## Example\n```json\n{\n \"score\": 0.85,\n \"reasoning\": \"Response is accurate and well-explained, but could include more detail on practical applications\"\n}\n```\n","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"azure_base"}},{"name":"style-check","type":"llm_judge","score":1,"weight":1,"verdict":"pass","hits":["Exceptionally clear and readable with organized sections and lists","Appropriate, educational tone and formality throughout","Effective, varied sentence structure and logical flow","Multiple specific examples of applications provided"],"misses":[],"reasoning":"The response is well-structured with clear headers, bullet points, and concise explanations that enhance readability. Tone and sentence structure are highly appropriate, and the inclusion of examples aids understanding, making the style exemplary.","evaluator_provider_request":{"user_prompt":"# Style Evaluation\n\nEvaluate the writing style and presentation of the response.\n\n## Task\nAssess the style and presentation of the candidate response based on:\n- Writing clarity and readability\n- Appropriate tone and formality\n- Sentence structure and flow\n- Use of examples and analogies\n\n## Input\n- Question: Explain the concept of neural networks.\n- Reference Answer: Neural networks are computational models inspired by biological neurons. They consist of interconnected layers of nodes (neurons) that process information through weighted connections. These networks learn patterns from data by adjusting connection weights during training.\n- Candidate Answer: A **neural network** is a computational model inspired by the human brain’s structure and function. It’s designed to recognize patterns and solve complex problems by learning from data. Here’s a breakdown of its core concepts:\n\n**1. Structure:** \n- **Neurons:** Basic units (nodes) that process information.\n- **Layers:**\n - **Input Layer:** Receives raw data.\n - **Hidden Layers:** Perform intermediate computations (can be one or many).\n - **Output Layer:** Produces the final result (e.g., classifications).\n- **Connections:** Neurons are connected by links, each with an associated weight.\n\n**2. Working Principle:** \n- The input data pass through the network from the input layer to the output layer.\n- Each neuron applies a mathematical operation to the input it receives, usually combining the values with weights and bias, then passing the result through an activation function (e.g., sigmoid, ReLU).\n- The output is calculated based on these transformations.\n\n**3. Learning Process:** \n- Neural networks learn by adjusting weights using algorithms such as **backpropagation**.\n- They compare predicted outputs with real targets, calculate errors, and update weights to minimize the error through techniques like **gradient descent**.\n- This process repeats over many examples, allowing the network to improve its predictions or classifications.\n\n**4. Applications:** \nNeural networks are widely used for tasks like image recognition, natural language processing, speech recognition, forecasting, and more.\n\n**Summary:** \nAt its core, a neural network is a system of interconnected \"neurons\" that learn to map inputs to outputs by discovering patterns in data, much like how the brain learns from experience.\n\n## Output Format\nReturn a JSON object with:\n- `score`: 0.0 (poor style) to 1.0 (excellent style)\n- `reasoning`: Brief explanation of the style assessment\n\n## Example\n```json\n{\n \"score\": 0.9,\n \"reasoning\": \"Response uses clear, accessible language with good flow and helpful examples\"\n}\n```\n","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"azure_base"}}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} +{"timestamp":"2026-01-03T12:08:11.241Z","eval_id":"equal-weights-default","dataset":"weighted-evaluators-examples","score":1,"hits":["Defines deep learning as a branch of machine learning using multi-layered neural networks","Explains abstraction and feature extraction across layers","Lists relevant application areas such as image and speech recognition","Mentions key frameworks and architectures like TensorFlow, CNNs, RNNs, Transformers","Explains deep learning as a subset of machine learning using neural networks","Describes multiple layers and abstraction of features","Lists key application areas and popular frameworks/architectures","Highlights important features like automatic feature extraction and data requirements","Uses clear, reader-friendly language throughout","Organizes information with headings and bulleted lists","Explains jargon terms like 'artificial neural networks' and 'deep'","Provides concrete examples and key features"],"misses":[],"candidate_answer":"**Deep learning** is a branch of machine learning that focuses on algorithms inspired by the structure and function of the human brain, known as artificial neural networks. \n\nIn deep learning, these neural networks are composed of many layers — hence the term \"deep.\" Each layer transforms the input data to a slightly higher level of abstraction and complexity. Through multiple layers, deep learning systems can automatically discover complex patterns and representations directly from raw data (such as images, text, or sound).\n\nDeep learning is particularly effective for tasks like:\n- Image and speech recognition\n- Natural language processing\n- Game playing\n- Autonomous driving\n\n**Key features:**\n- **Learning from large datasets:** Deep learning models require significant amounts of data to generalize well.\n- **Automatic feature extraction:** Models learn which features are important directly from data, reducing the need for manual engineering.\n- **Multi-layered structure:** Deep neural networks have many hidden layers, each capturing increasingly abstract data representations.\n\nPopular deep learning frameworks include TensorFlow and PyTorch. Notable architectures are Convolutional Neural Networks (CNNs), Recurrent Neural Networks (RNNs), and Transformers.\n\nIn summary, deep learning enables machines to solve complex problems by learning from large datasets using multi-layered neural networks.","target":"azure_base","reasoning":"correctness: The response is factually accurate, logically consistent, and technically correct; it fully covers deep learning as described in the reference answer without errors. | completeness: The candidate answer fully covers all essential aspects of deep learning, provides comprehensive details, and offers sufficient context and examples to address the question thoroughly. | clarity: The response is highly clear, well-organized, and appropriate for the target audience, with minimal jargon and helpful explanations that enhance reader understanding.","lm_provider_request":{"question":"Describe deep learning.","guidelines":""},"evaluator_results":[{"name":"correctness","type":"llm_judge","score":1,"weight":1,"verdict":"pass","hits":["Defines deep learning as a branch of machine learning using multi-layered neural networks","Explains abstraction and feature extraction across layers","Lists relevant application areas such as image and speech recognition","Mentions key frameworks and architectures like TensorFlow, CNNs, RNNs, Transformers"],"misses":[],"reasoning":"The response is factually accurate, logically consistent, and technically correct; it fully covers deep learning as described in the reference answer without errors.","evaluator_provider_request":{"user_prompt":"# Correctness Check\n\nEvaluate whether the response is correct and free from errors.\n\n## Task\nAssess the correctness of the candidate response, checking for:\n- Factual accuracy\n- Logical consistency\n- Absence of contradictions\n- Technical correctness\n\n## Input\n- Question: Describe deep learning.\n- Reference Answer: Deep learning is a subset of machine learning that uses neural networks with multiple layers (deep neural networks). Each layer learns to extract increasingly abstract features from the input data, enabling the model to learn complex patterns and representations.\n- Candidate Answer: **Deep learning** is a branch of machine learning that focuses on algorithms inspired by the structure and function of the human brain, known as artificial neural networks. \n\nIn deep learning, these neural networks are composed of many layers — hence the term \"deep.\" Each layer transforms the input data to a slightly higher level of abstraction and complexity. Through multiple layers, deep learning systems can automatically discover complex patterns and representations directly from raw data (such as images, text, or sound).\n\nDeep learning is particularly effective for tasks like:\n- Image and speech recognition\n- Natural language processing\n- Game playing\n- Autonomous driving\n\n**Key features:**\n- **Learning from large datasets:** Deep learning models require significant amounts of data to generalize well.\n- **Automatic feature extraction:** Models learn which features are important directly from data, reducing the need for manual engineering.\n- **Multi-layered structure:** Deep neural networks have many hidden layers, each capturing increasingly abstract data representations.\n\nPopular deep learning frameworks include TensorFlow and PyTorch. Notable architectures are Convolutional Neural Networks (CNNs), Recurrent Neural Networks (RNNs), and Transformers.\n\nIn summary, deep learning enables machines to solve complex problems by learning from large datasets using multi-layered neural networks.\n\n## Output Format\nReturn a JSON object with:\n- `score`: 0.0 (incorrect) to 1.0 (completely correct)\n- `reasoning`: Brief explanation of correctness assessment\n\n## Example\n```json\n{\n \"score\": 0.95,\n \"reasoning\": \"Response is technically correct with minor terminology imprecision\"\n}\n```\n","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"azure_base"}},{"name":"completeness","type":"llm_judge","score":1,"weight":1,"verdict":"pass","hits":["Explains deep learning as a subset of machine learning using neural networks","Describes multiple layers and abstraction of features","Lists key application areas and popular frameworks/architectures","Highlights important features like automatic feature extraction and data requirements"],"misses":[],"reasoning":"The candidate answer fully covers all essential aspects of deep learning, provides comprehensive details, and offers sufficient context and examples to address the question thoroughly.","evaluator_provider_request":{"user_prompt":"# Completeness Check\n\nEvaluate whether the response is complete and comprehensive.\n\n## Task\nAssess the completeness of the candidate response:\n- Covers all key aspects of the question\n- Includes important details\n- Addresses follow-up concerns\n- Provides sufficient context\n\n## Input\n- Question: Describe deep learning.\n- Reference Answer: Deep learning is a subset of machine learning that uses neural networks with multiple layers (deep neural networks). Each layer learns to extract increasingly abstract features from the input data, enabling the model to learn complex patterns and representations.\n- Candidate Answer: **Deep learning** is a branch of machine learning that focuses on algorithms inspired by the structure and function of the human brain, known as artificial neural networks. \n\nIn deep learning, these neural networks are composed of many layers — hence the term \"deep.\" Each layer transforms the input data to a slightly higher level of abstraction and complexity. Through multiple layers, deep learning systems can automatically discover complex patterns and representations directly from raw data (such as images, text, or sound).\n\nDeep learning is particularly effective for tasks like:\n- Image and speech recognition\n- Natural language processing\n- Game playing\n- Autonomous driving\n\n**Key features:**\n- **Learning from large datasets:** Deep learning models require significant amounts of data to generalize well.\n- **Automatic feature extraction:** Models learn which features are important directly from data, reducing the need for manual engineering.\n- **Multi-layered structure:** Deep neural networks have many hidden layers, each capturing increasingly abstract data representations.\n\nPopular deep learning frameworks include TensorFlow and PyTorch. Notable architectures are Convolutional Neural Networks (CNNs), Recurrent Neural Networks (RNNs), and Transformers.\n\nIn summary, deep learning enables machines to solve complex problems by learning from large datasets using multi-layered neural networks.\n\n## Output Format\nReturn a JSON object with:\n- `score`: 0.0 (incomplete) to 1.0 (fully complete)\n- `reasoning`: Brief explanation of what's included or missing\n\n## Example\n```json\n{\n \"score\": 0.8,\n \"reasoning\": \"Response covers main concepts well but could expand on practical applications\"\n}\n```\n","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"azure_base"}},{"name":"clarity","type":"llm_judge","score":1,"weight":1,"verdict":"pass","hits":["Uses clear, reader-friendly language throughout","Organizes information with headings and bulleted lists","Explains jargon terms like 'artificial neural networks' and 'deep'","Provides concrete examples and key features"],"misses":[],"reasoning":"The response is highly clear, well-organized, and appropriate for the target audience, with minimal jargon and helpful explanations that enhance reader understanding.","evaluator_provider_request":{"user_prompt":"# Clarity Check\n\nEvaluate the clarity and understandability of the response.\n\n## Task\nAssess how clear and easy to understand the candidate response is:\n- Uses clear, unambiguous language\n- Well-organized structure\n- Appropriate for the target audience\n- Avoids unnecessary jargon\n\n## Input\n- Question: Describe deep learning.\n- Reference Answer: Deep learning is a subset of machine learning that uses neural networks with multiple layers (deep neural networks). Each layer learns to extract increasingly abstract features from the input data, enabling the model to learn complex patterns and representations.\n- Candidate Answer: **Deep learning** is a branch of machine learning that focuses on algorithms inspired by the structure and function of the human brain, known as artificial neural networks. \n\nIn deep learning, these neural networks are composed of many layers — hence the term \"deep.\" Each layer transforms the input data to a slightly higher level of abstraction and complexity. Through multiple layers, deep learning systems can automatically discover complex patterns and representations directly from raw data (such as images, text, or sound).\n\nDeep learning is particularly effective for tasks like:\n- Image and speech recognition\n- Natural language processing\n- Game playing\n- Autonomous driving\n\n**Key features:**\n- **Learning from large datasets:** Deep learning models require significant amounts of data to generalize well.\n- **Automatic feature extraction:** Models learn which features are important directly from data, reducing the need for manual engineering.\n- **Multi-layered structure:** Deep neural networks have many hidden layers, each capturing increasingly abstract data representations.\n\nPopular deep learning frameworks include TensorFlow and PyTorch. Notable architectures are Convolutional Neural Networks (CNNs), Recurrent Neural Networks (RNNs), and Transformers.\n\nIn summary, deep learning enables machines to solve complex problems by learning from large datasets using multi-layered neural networks.\n\n## Output Format\nReturn a JSON object with:\n- `score`: 0.0 (unclear) to 1.0 (perfectly clear)\n- `reasoning`: Brief explanation of clarity assessment\n\n## Example\n```json\n{\n \"score\": 0.9,\n \"reasoning\": \"Response is very clear and well-structured with good use of analogies\"\n}\n```\n","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"azure_base"}}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} diff --git a/examples/showcase/cw-incident-triage/evals/dataset.baseline.jsonl b/examples/showcase/cw-incident-triage/evals/dataset.baseline.jsonl new file mode 100644 index 00000000..801bfaa6 --- /dev/null +++ b/examples/showcase/cw-incident-triage/evals/dataset.baseline.jsonl @@ -0,0 +1,8 @@ +{"timestamp":"2026-01-03T12:08:36.439Z","eval_id":"cr-global-outage","dataset":"dataset","conversation_id":"cargowise-triage","score":1,"hits":["Correctly assigns CR1 as the rating","Emphasizes global/system-wide impact","Notes that all operations are halted and no users can log in","References criteria for CR1 accurately in reasoning"],"misses":[],"candidate_answer":"```json\n{\n \"criticalityRating\": \"CR1\",\n \"reasoning\": \"Step 1: Scope analysis: The ticket reports that no users can log in from any device, indicating a global impact across the entire system. Step 2: Impact assessment: All operations are halted, so there is no viable workaround for any user or operation. Step 3: Final classification logic: According to the provided definitions, CR1 is for situations where the entire system is down and inaccessible to all users. Therefore, the ticket matches CR1.\"\n}\n```","target":"default","reasoning":"json_schema_validator: Valid JSON with all required keys: criticalityRating, reasoning | content_evaluator: The answer fully captures the CR1 criteria with detailed stepwise reasoning emphasizing global inaccessibility and the lack of user access on any device, aligning exactly with expected outcome and reference.","lm_provider_request":{"question":"\n---\nname: cw-criticality-rating\ndescription: Classify CargoWise support tickets using criticality ratings CR1-CR9\n---\n\n# Task\n\nYou are a CargoWise support ticket triage specialist. Classify each ticket with CR1-CR9 based on scope, impact, workarounds, and request type.\n\n## Criticality Rating Definitions\n\n**CR1**: Entire system down; no user can access the application suite.\n**CR2**: Entire module down; module inaccessible for all users, other modules OK.\n**CR3**: Single function broken with no viable workaround; behavior deviates from docs or prior correct behavior.\n**CR4**: Single function broken but a viable workaround/manual alternative exists.\n**CR5**: Training/how-to request on existing functionality.\n**CR6**: Feature/enhancement request beyond documented behavior; not a defect.\n**CR7**: Request for pricing/quote on new features or accelerated work.\n**CR8**: Compliance/reference/master data updates or corrections; data accuracy issues, not functional bugs.\n**CR9**: Service/operational/administrative request.\n\n## Classification Process\n\n1. **Analyze scope and impact**: system vs module vs function; user/workstation scope.\n2. **Assess workarounds**: note whether alternatives exist and if they are feasible.\n3. **Distinguish defects from features**: defect = function not working as documented or changed from correct behavior; feature = request for new capability; if a previously fixed defect is now a substantial change request, treat as CR6. Always prioritize documentation and specifications over user-assigned labels (e.g., \"Critical Bug\")—if the system behaves as documented, it is not a defect.\n4. **Handle multi-part tickets**: classify by the highest criticality element.\n5. **Provide reasoning**: explain how the rating was reached and reference specific signals.\n\n## Output Format\n\nOutput ONLY valid JSON with this structure:\n\n```json\n{\n \"criticalityRating\": \"CRX\",\n \"reasoning\": \"Step 1: [First observation]. Step 2: [Second observation]. Step 3: [Final classification logic].\"\n}\n```\n\nThink step-by-step and explain your reasoning before concluding.\n\n## Edge Cases\n\n- **Post-fix enhancement requests**: If a defect was fixed per documentation but the user now wants substantial changes -> CR6.\n- **Gap between expectation and spec**: If the system works as documented but the user expects different behavior (even if logical or \"obvious\"), it is an enhancement (CR6).\n- **Compliance data vs bugs**: Data accuracy/update needs -> CR8 (not CR3).\n- **Multi-element tickets**: Classify by highest criticality.\n- **Workaround feasibility**: If manual alternatives are infeasible at volume, treat as no workaround (CR3 over CR4).\n\n\nClassify this CargoWise ticket:\n\nTicket: Entire system down, no users can log in from any device\nImpact: All operations halted\nScope: Global\nSignals: Error: \"Server unreachable\", no recent changes reported","guidelines":""},"evaluator_results":[{"name":"json_schema_validator","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":[],"misses":[],"reasoning":"Valid JSON with all required keys: criticalityRating, reasoning","evaluator_provider_request":{"script":["uv","run","validate_output.py"],"cwd":"/root/projects/agentv/examples/showcase/cw-incident-triage/evals"}},{"name":"content_evaluator","type":"llm_judge","score":1,"weight":1,"verdict":"pass","hits":["Correctly assigns CR1 as the rating","Emphasizes global/system-wide impact","Notes that all operations are halted and no users can log in","References criteria for CR1 accurately in reasoning"],"misses":[],"reasoning":"The answer fully captures the CR1 criteria with detailed stepwise reasoning emphasizing global inaccessibility and the lack of user access on any device, aligning exactly with expected outcome and reference.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nAssistant correctly classifies as 'CR1' for complete system inaccessibility.\nReasoning should emphasize 'any user on any workstation' and lack of access to the entire suite.\n\n[[ ## question ## ]]\n\n---\nname: cw-criticality-rating\ndescription: Classify CargoWise support tickets using criticality ratings CR1-CR9\n---\n\n# Task\n\nYou are a CargoWise support ticket triage specialist. Classify each ticket with CR1-CR9 based on scope, impact, workarounds, and request type.\n\n## Criticality Rating Definitions\n\n**CR1**: Entire system down; no user can access the application suite.\n**CR2**: Entire module down; module inaccessible for all users, other modules OK.\n**CR3**: Single function broken with no viable workaround; behavior deviates from docs or prior correct behavior.\n**CR4**: Single function broken but a viable workaround/manual alternative exists.\n**CR5**: Training/how-to request on existing functionality.\n**CR6**: Feature/enhancement request beyond documented behavior; not a defect.\n**CR7**: Request for pricing/quote on new features or accelerated work.\n**CR8**: Compliance/reference/master data updates or corrections; data accuracy issues, not functional bugs.\n**CR9**: Service/operational/administrative request.\n\n## Classification Process\n\n1. **Analyze scope and impact**: system vs module vs function; user/workstation scope.\n2. **Assess workarounds**: note whether alternatives exist and if they are feasible.\n3. **Distinguish defects from features**: defect = function not working as documented or changed from correct behavior; feature = request for new capability; if a previously fixed defect is now a substantial change request, treat as CR6. Always prioritize documentation and specifications over user-assigned labels (e.g., \"Critical Bug\")—if the system behaves as documented, it is not a defect.\n4. **Handle multi-part tickets**: classify by the highest criticality element.\n5. **Provide reasoning**: explain how the rating was reached and reference specific signals.\n\n## Output Format\n\nOutput ONLY valid JSON with this structure:\n\n```json\n{\n \"criticalityRating\": \"CRX\",\n \"reasoning\": \"Step 1: [First observation]. Step 2: [Second observation]. Step 3: [Final classification logic].\"\n}\n```\n\nThink step-by-step and explain your reasoning before concluding.\n\n## Edge Cases\n\n- **Post-fix enhancement requests**: If a defect was fixed per documentation but the user now wants substantial changes -> CR6.\n- **Gap between expectation and spec**: If the system works as documented but the user expects different behavior (even if logical or \"obvious\"), it is an enhancement (CR6).\n- **Compliance data vs bugs**: Data accuracy/update needs -> CR8 (not CR3).\n- **Multi-element tickets**: Classify by highest criticality.\n- **Workaround feasibility**: If manual alternatives are infeasible at volume, treat as no workaround (CR3 over CR4).\n\n\nClassify this CargoWise ticket:\n\nTicket: Entire system down, no users can log in from any device\nImpact: All operations halted\nScope: Global\nSignals: Error: \"Server unreachable\", no recent changes reported\n\n[[ ## reference_answer ## ]]\n{\n \"criticalityRating\": \"CR1\",\n \"reasoning\": \"Step 1: Issue prevents access to the entire application suite. Step 2: Affects any user on any workstation. Step 3: Matches CR1 definition exactly—no partial access or workarounds mentioned.\"\n}\n\n[[ ## candidate_answer ## ]]\n```json\n{\n \"criticalityRating\": \"CR1\",\n \"reasoning\": \"Step 1: Scope analysis: The ticket reports that no users can log in from any device, indicating a global impact across the entire system. Step 2: Impact assessment: All operations are halted, so there is no viable workaround for any user or operation. Step 3: Final classification logic: According to the provided definitions, CR1 is for situations where the entire system is down and inaccessible to all users. Therefore, the ticket matches CR1.\"\n}\n```","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"default"}}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} +{"timestamp":"2026-01-03T12:08:36.453Z","eval_id":"cr-module-inaccessible","dataset":"dataset","conversation_id":"cargowise-triage","score":1,"hits":["Correctly classifies as CR2","Explicitly distinguishes CR2 from CR1 by noting other modules are working","References the 'module not found' error and module-wide impact","Reasoning follows step-by-step process per guidelines"],"misses":[],"candidate_answer":"```json\n{\n \"criticalityRating\": \"CR2\",\n \"reasoning\": \"Step 1: The ticket states the 'Customs module' is inaccessible for all users, with customs declarations blocked. Step 2: The error received is 'Module not found', and other modules are confirmed to be working fine. Step 3: This fits the definition of CR2—entire module down for all users, while the rest of the system remains operational. Thus, CR2 is assigned.\"\n}\n```","target":"default","reasoning":"json_schema_validator: Valid JSON with all required keys: criticalityRating, reasoning | content_evaluator: The candidate matches the classification, justification, and stepwise reasoning of the reference, clearly explaining why the issue is CR2 and not CR1.","lm_provider_request":{"question":"\n---\nname: cw-criticality-rating\ndescription: Classify CargoWise support tickets using criticality ratings CR1-CR9\n---\n\n# Task\n\nYou are a CargoWise support ticket triage specialist. Classify each ticket with CR1-CR9 based on scope, impact, workarounds, and request type.\n\n## Criticality Rating Definitions\n\n**CR1**: Entire system down; no user can access the application suite.\n**CR2**: Entire module down; module inaccessible for all users, other modules OK.\n**CR3**: Single function broken with no viable workaround; behavior deviates from docs or prior correct behavior.\n**CR4**: Single function broken but a viable workaround/manual alternative exists.\n**CR5**: Training/how-to request on existing functionality.\n**CR6**: Feature/enhancement request beyond documented behavior; not a defect.\n**CR7**: Request for pricing/quote on new features or accelerated work.\n**CR8**: Compliance/reference/master data updates or corrections; data accuracy issues, not functional bugs.\n**CR9**: Service/operational/administrative request.\n\n## Classification Process\n\n1. **Analyze scope and impact**: system vs module vs function; user/workstation scope.\n2. **Assess workarounds**: note whether alternatives exist and if they are feasible.\n3. **Distinguish defects from features**: defect = function not working as documented or changed from correct behavior; feature = request for new capability; if a previously fixed defect is now a substantial change request, treat as CR6. Always prioritize documentation and specifications over user-assigned labels (e.g., \"Critical Bug\")—if the system behaves as documented, it is not a defect.\n4. **Handle multi-part tickets**: classify by the highest criticality element.\n5. **Provide reasoning**: explain how the rating was reached and reference specific signals.\n\n## Output Format\n\nOutput ONLY valid JSON with this structure:\n\n```json\n{\n \"criticalityRating\": \"CRX\",\n \"reasoning\": \"Step 1: [First observation]. Step 2: [Second observation]. Step 3: [Final classification logic].\"\n}\n```\n\nThink step-by-step and explain your reasoning before concluding.\n\n## Edge Cases\n\n- **Post-fix enhancement requests**: If a defect was fixed per documentation but the user now wants substantial changes -> CR6.\n- **Gap between expectation and spec**: If the system works as documented but the user expects different behavior (even if logical or \"obvious\"), it is an enhancement (CR6).\n- **Compliance data vs bugs**: Data accuracy/update needs -> CR8 (not CR3).\n- **Multi-element tickets**: Classify by highest criticality.\n- **Workaround feasibility**: If manual alternatives are infeasible at volume, treat as no workaround (CR3 over CR4).\n\n\nClassify this CargoWise ticket:\n\nTicket: Customs module inaccessible for all users\nImpact: Customs declarations blocked\nScope: Module-wide\nSignals: \"Module not found\" error, other modules working","guidelines":""},"evaluator_results":[{"name":"json_schema_validator","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":[],"misses":[],"reasoning":"Valid JSON with all required keys: criticalityRating, reasoning","evaluator_provider_request":{"script":["uv","run","validate_output.py"],"cwd":"/root/projects/agentv/examples/showcase/cw-incident-triage/evals"}},{"name":"content_evaluator","type":"llm_judge","score":1,"weight":1,"verdict":"pass","hits":["Correctly classifies as CR2","Explicitly distinguishes CR2 from CR1 by noting other modules are working","References the 'module not found' error and module-wide impact","Reasoning follows step-by-step process per guidelines"],"misses":[],"reasoning":"The candidate matches the classification, justification, and stepwise reasoning of the reference, clearly explaining why the issue is CR2 and not CR1.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nAssistant correctly classifies as 'CR2' for module-wide inaccessibility.\nReasoning should distinguish from CR1 by noting it's limited to one module.\n\n[[ ## question ## ]]\n\n---\nname: cw-criticality-rating\ndescription: Classify CargoWise support tickets using criticality ratings CR1-CR9\n---\n\n# Task\n\nYou are a CargoWise support ticket triage specialist. Classify each ticket with CR1-CR9 based on scope, impact, workarounds, and request type.\n\n## Criticality Rating Definitions\n\n**CR1**: Entire system down; no user can access the application suite.\n**CR2**: Entire module down; module inaccessible for all users, other modules OK.\n**CR3**: Single function broken with no viable workaround; behavior deviates from docs or prior correct behavior.\n**CR4**: Single function broken but a viable workaround/manual alternative exists.\n**CR5**: Training/how-to request on existing functionality.\n**CR6**: Feature/enhancement request beyond documented behavior; not a defect.\n**CR7**: Request for pricing/quote on new features or accelerated work.\n**CR8**: Compliance/reference/master data updates or corrections; data accuracy issues, not functional bugs.\n**CR9**: Service/operational/administrative request.\n\n## Classification Process\n\n1. **Analyze scope and impact**: system vs module vs function; user/workstation scope.\n2. **Assess workarounds**: note whether alternatives exist and if they are feasible.\n3. **Distinguish defects from features**: defect = function not working as documented or changed from correct behavior; feature = request for new capability; if a previously fixed defect is now a substantial change request, treat as CR6. Always prioritize documentation and specifications over user-assigned labels (e.g., \"Critical Bug\")—if the system behaves as documented, it is not a defect.\n4. **Handle multi-part tickets**: classify by the highest criticality element.\n5. **Provide reasoning**: explain how the rating was reached and reference specific signals.\n\n## Output Format\n\nOutput ONLY valid JSON with this structure:\n\n```json\n{\n \"criticalityRating\": \"CRX\",\n \"reasoning\": \"Step 1: [First observation]. Step 2: [Second observation]. Step 3: [Final classification logic].\"\n}\n```\n\nThink step-by-step and explain your reasoning before concluding.\n\n## Edge Cases\n\n- **Post-fix enhancement requests**: If a defect was fixed per documentation but the user now wants substantial changes -> CR6.\n- **Gap between expectation and spec**: If the system works as documented but the user expects different behavior (even if logical or \"obvious\"), it is an enhancement (CR6).\n- **Compliance data vs bugs**: Data accuracy/update needs -> CR8 (not CR3).\n- **Multi-element tickets**: Classify by highest criticality.\n- **Workaround feasibility**: If manual alternatives are infeasible at volume, treat as no workaround (CR3 over CR4).\n\n\nClassify this CargoWise ticket:\n\nTicket: Customs module inaccessible for all users\nImpact: Customs declarations blocked\nScope: Module-wide\nSignals: \"Module not found\" error, other modules working\n\n[[ ## reference_answer ## ]]\n{\n \"criticalityRating\": \"CR2\",\n \"reasoning\": \"Step 1: Issue affects an entire module (Customs). Step 2: No access for any user/workstation. Step 3: Does not impact the full suite, so CR2 over CR1.\"\n}\n\n[[ ## candidate_answer ## ]]\n```json\n{\n \"criticalityRating\": \"CR2\",\n \"reasoning\": \"Step 1: The ticket states the 'Customs module' is inaccessible for all users, with customs declarations blocked. Step 2: The error received is 'Module not found', and other modules are confirmed to be working fine. Step 3: This fits the definition of CR2—entire module down for all users, while the rest of the system remains operational. Thus, CR2 is assigned.\"\n}\n```","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"default"}}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} +{"timestamp":"2026-01-03T12:08:37.130Z","eval_id":"cr-missing-validation-disguised-as-defect","dataset":"dataset","conversation_id":"cargowise-triage","score":1,"hits":["Identifies significant operational impact (customs rejections, fines)","Clearly notes documentation only specifies length, not uniqueness","Distinguishes between defect (as-fixed) and new enhancement request","Correctly classifies as CR6 with logical reasoning"],"misses":[],"candidate_answer":"```json\n{\n \"criticalityRating\": \"CR6\",\n \"reasoning\": \"Step 1: The reported issue is that duplicate containers can be saved on the BL (Bill of Lading), leading to customs rejections and fines, which is a significant operational impact. Step 2: Reviewing the resolution history, the fix only implemented length (11-char) validation, matching documentation. There is no documented requirement for container number uniqueness validation. The current system behavior matches documentation: accepts any 11-char code including duplicates. Step 3: Since the request is for functionality (uniqueness validation) beyond what is defined in documentation, this is a feature/enhancement request rather than a defect. Therefore, the correct rating is CR6.\"\n}\n```","target":"default","reasoning":"json_schema_validator: Valid JSON with all required keys: criticalityRating, reasoning | content_evaluator: The candidate answer captures all expected reasoning steps, distinguishes between defect and enhancement per documentation, and provides a clear, accurate justification for assigning CR6.","lm_provider_request":{"question":"\n---\nname: cw-criticality-rating\ndescription: Classify CargoWise support tickets using criticality ratings CR1-CR9\n---\n\n# Task\n\nYou are a CargoWise support ticket triage specialist. Classify each ticket with CR1-CR9 based on scope, impact, workarounds, and request type.\n\n## Criticality Rating Definitions\n\n**CR1**: Entire system down; no user can access the application suite.\n**CR2**: Entire module down; module inaccessible for all users, other modules OK.\n**CR3**: Single function broken with no viable workaround; behavior deviates from docs or prior correct behavior.\n**CR4**: Single function broken but a viable workaround/manual alternative exists.\n**CR5**: Training/how-to request on existing functionality.\n**CR6**: Feature/enhancement request beyond documented behavior; not a defect.\n**CR7**: Request for pricing/quote on new features or accelerated work.\n**CR8**: Compliance/reference/master data updates or corrections; data accuracy issues, not functional bugs.\n**CR9**: Service/operational/administrative request.\n\n## Classification Process\n\n1. **Analyze scope and impact**: system vs module vs function; user/workstation scope.\n2. **Assess workarounds**: note whether alternatives exist and if they are feasible.\n3. **Distinguish defects from features**: defect = function not working as documented or changed from correct behavior; feature = request for new capability; if a previously fixed defect is now a substantial change request, treat as CR6. Always prioritize documentation and specifications over user-assigned labels (e.g., \"Critical Bug\")—if the system behaves as documented, it is not a defect.\n4. **Handle multi-part tickets**: classify by the highest criticality element.\n5. **Provide reasoning**: explain how the rating was reached and reference specific signals.\n\n## Output Format\n\nOutput ONLY valid JSON with this structure:\n\n```json\n{\n \"criticalityRating\": \"CRX\",\n \"reasoning\": \"Step 1: [First observation]. Step 2: [Second observation]. Step 3: [Final classification logic].\"\n}\n```\n\nThink step-by-step and explain your reasoning before concluding.\n\n## Edge Cases\n\n- **Post-fix enhancement requests**: If a defect was fixed per documentation but the user now wants substantial changes -> CR6.\n- **Gap between expectation and spec**: If the system works as documented but the user expects different behavior (even if logical or \"obvious\"), it is an enhancement (CR6).\n- **Compliance data vs bugs**: Data accuracy/update needs -> CR8 (not CR3).\n- **Multi-element tickets**: Classify by highest criticality.\n- **Workaround feasibility**: If manual alternatives are infeasible at volume, treat as no workaround (CR3 over CR4).\n\n\nClassify this CargoWise ticket:\n\nTicket: REOPENED DEF-555: Critical Production Bug - Container Validation Failed\nImpact: Customs filings rejected. Fines incurring.\nScope: Consolidation/Booking\nSignals: \n- User Comment: \"Re-opening because the fix didn't work! We can still save duplicate containers on the BL. You said this was fixed in the last patch. This is a critical defect causing data corruption.\"\n- DEF-555 History: Original Report: \"Container validation missing.\" Resolution: \"Added 11-char length validation.\" Status: Closed/Deployed.\n- Documentation: \"Container Number: Enter the 11-character ISO code.\" (No mention of uniqueness check).\n- System Behavior: Accepts 11-char duplicates without error.","guidelines":""},"evaluator_results":[{"name":"json_schema_validator","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":[],"misses":[],"reasoning":"Valid JSON with all required keys: criticalityRating, reasoning","evaluator_provider_request":{"script":["uv","run","validate_output.py"],"cwd":"/root/projects/agentv/examples/showcase/cw-incident-triage/evals"}},{"name":"content_evaluator","type":"llm_judge","score":1,"weight":1,"verdict":"pass","hits":["Identifies significant operational impact (customs rejections, fines)","Clearly notes documentation only specifies length, not uniqueness","Distinguishes between defect (as-fixed) and new enhancement request","Correctly classifies as CR6 with logical reasoning"],"misses":[],"reasoning":"The candidate answer captures all expected reasoning steps, distinguishes between defect and enhancement per documentation, and provides a clear, accurate justification for assigning CR6.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nAssistant classifies as 'CR6' (Enhancement) despite user claiming \"Critical Bug\" and citing a prior defect.\nMost LLMs might misclassify as CR3 (Defect) due to the user's label, financial impact, and claim of a \"failed fix\".\nExpected is CR6 because the prior fix (DEF-555) addressed format length (as per docs), whereas this request is for a new uniqueness constraint not in the specs.\n\n[[ ## question ## ]]\n\n---\nname: cw-criticality-rating\ndescription: Classify CargoWise support tickets using criticality ratings CR1-CR9\n---\n\n# Task\n\nYou are a CargoWise support ticket triage specialist. Classify each ticket with CR1-CR9 based on scope, impact, workarounds, and request type.\n\n## Criticality Rating Definitions\n\n**CR1**: Entire system down; no user can access the application suite.\n**CR2**: Entire module down; module inaccessible for all users, other modules OK.\n**CR3**: Single function broken with no viable workaround; behavior deviates from docs or prior correct behavior.\n**CR4**: Single function broken but a viable workaround/manual alternative exists.\n**CR5**: Training/how-to request on existing functionality.\n**CR6**: Feature/enhancement request beyond documented behavior; not a defect.\n**CR7**: Request for pricing/quote on new features or accelerated work.\n**CR8**: Compliance/reference/master data updates or corrections; data accuracy issues, not functional bugs.\n**CR9**: Service/operational/administrative request.\n\n## Classification Process\n\n1. **Analyze scope and impact**: system vs module vs function; user/workstation scope.\n2. **Assess workarounds**: note whether alternatives exist and if they are feasible.\n3. **Distinguish defects from features**: defect = function not working as documented or changed from correct behavior; feature = request for new capability; if a previously fixed defect is now a substantial change request, treat as CR6. Always prioritize documentation and specifications over user-assigned labels (e.g., \"Critical Bug\")—if the system behaves as documented, it is not a defect.\n4. **Handle multi-part tickets**: classify by the highest criticality element.\n5. **Provide reasoning**: explain how the rating was reached and reference specific signals.\n\n## Output Format\n\nOutput ONLY valid JSON with this structure:\n\n```json\n{\n \"criticalityRating\": \"CRX\",\n \"reasoning\": \"Step 1: [First observation]. Step 2: [Second observation]. Step 3: [Final classification logic].\"\n}\n```\n\nThink step-by-step and explain your reasoning before concluding.\n\n## Edge Cases\n\n- **Post-fix enhancement requests**: If a defect was fixed per documentation but the user now wants substantial changes -> CR6.\n- **Gap between expectation and spec**: If the system works as documented but the user expects different behavior (even if logical or \"obvious\"), it is an enhancement (CR6).\n- **Compliance data vs bugs**: Data accuracy/update needs -> CR8 (not CR3).\n- **Multi-element tickets**: Classify by highest criticality.\n- **Workaround feasibility**: If manual alternatives are infeasible at volume, treat as no workaround (CR3 over CR4).\n\n\nClassify this CargoWise ticket:\n\nTicket: REOPENED DEF-555: Critical Production Bug - Container Validation Failed\nImpact: Customs filings rejected. Fines incurring.\nScope: Consolidation/Booking\nSignals: \n- User Comment: \"Re-opening because the fix didn't work! We can still save duplicate containers on the BL. You said this was fixed in the last patch. This is a critical defect causing data corruption.\"\n- DEF-555 History: Original Report: \"Container validation missing.\" Resolution: \"Added 11-char length validation.\" Status: Closed/Deployed.\n- Documentation: \"Container Number: Enter the 11-character ISO code.\" (No mention of uniqueness check).\n- System Behavior: Accepts 11-char duplicates without error.\n\n[[ ## reference_answer ## ]]\n{\n \"criticalityRating\": \"CR6\",\n \"reasoning\": \"Step 1: System functions as documented (accepts alphanumeric input, enforces length per DEF-555). Step 2: User claims failed fix, but DEF-555 addressed length, not uniqueness. Step 3: Request is to add a new validation rule (uniqueness) that does not currently exist in specs. Step 4: Despite 'Critical Bug' label and prior ticket reference, this is a request for new capability (Enhancement), not a deviation from documentation (Defect). Matches CR6.\"\n}\n\n[[ ## candidate_answer ## ]]\n```json\n{\n \"criticalityRating\": \"CR6\",\n \"reasoning\": \"Step 1: The reported issue is that duplicate containers can be saved on the BL (Bill of Lading), leading to customs rejections and fines, which is a significant operational impact. Step 2: Reviewing the resolution history, the fix only implemented length (11-char) validation, matching documentation. There is no documented requirement for container number uniqueness validation. The current system behavior matches documentation: accepts any 11-char code including duplicates. Step 3: Since the request is for functionality (uniqueness validation) beyond what is defined in documentation, this is a feature/enhancement request rather than a defect. Therefore, the correct rating is CR6.\"\n}\n```","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"default"}}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} +{"timestamp":"2026-01-03T12:08:38.847Z","eval_id":"cr-feature-quote","dataset":"dataset","conversation_id":"cargowise-triage","score":1,"hits":["Correctly classifies as CR7","Distinguishes new feature scope","Notes pricing/quote request for accelerated development","References rating definitions in reasoning"],"misses":[],"candidate_answer":"```json\n{\n \"criticalityRating\": \"CR7\",\n \"reasoning\": \"Step 1: The ticket requests a custom API for bulk tariff updates, which is outside the current product's documented behavior (scope: new feature). Step 2: The user specifically asks how much it would cost to fast-track this development, indicating a pricing/quote request for accelerated work. Step 3: According to the definitions, requests for pricing/quote on new features or expedited development are classified as CR7.\"\n}\n```","target":"default","reasoning":"json_schema_validator: Valid JSON with all required keys: criticalityRating, reasoning | content_evaluator: The candidate answer matches the reference exactly, providing comprehensive reasoning that identifies scope, pricing request, and correctly applies the CR7 definition.","lm_provider_request":{"question":"\n---\nname: cw-criticality-rating\ndescription: Classify CargoWise support tickets using criticality ratings CR1-CR9\n---\n\n# Task\n\nYou are a CargoWise support ticket triage specialist. Classify each ticket with CR1-CR9 based on scope, impact, workarounds, and request type.\n\n## Criticality Rating Definitions\n\n**CR1**: Entire system down; no user can access the application suite.\n**CR2**: Entire module down; module inaccessible for all users, other modules OK.\n**CR3**: Single function broken with no viable workaround; behavior deviates from docs or prior correct behavior.\n**CR4**: Single function broken but a viable workaround/manual alternative exists.\n**CR5**: Training/how-to request on existing functionality.\n**CR6**: Feature/enhancement request beyond documented behavior; not a defect.\n**CR7**: Request for pricing/quote on new features or accelerated work.\n**CR8**: Compliance/reference/master data updates or corrections; data accuracy issues, not functional bugs.\n**CR9**: Service/operational/administrative request.\n\n## Classification Process\n\n1. **Analyze scope and impact**: system vs module vs function; user/workstation scope.\n2. **Assess workarounds**: note whether alternatives exist and if they are feasible.\n3. **Distinguish defects from features**: defect = function not working as documented or changed from correct behavior; feature = request for new capability; if a previously fixed defect is now a substantial change request, treat as CR6. Always prioritize documentation and specifications over user-assigned labels (e.g., \"Critical Bug\")—if the system behaves as documented, it is not a defect.\n4. **Handle multi-part tickets**: classify by the highest criticality element.\n5. **Provide reasoning**: explain how the rating was reached and reference specific signals.\n\n## Output Format\n\nOutput ONLY valid JSON with this structure:\n\n```json\n{\n \"criticalityRating\": \"CRX\",\n \"reasoning\": \"Step 1: [First observation]. Step 2: [Second observation]. Step 3: [Final classification logic].\"\n}\n```\n\nThink step-by-step and explain your reasoning before concluding.\n\n## Edge Cases\n\n- **Post-fix enhancement requests**: If a defect was fixed per documentation but the user now wants substantial changes -> CR6.\n- **Gap between expectation and spec**: If the system works as documented but the user expects different behavior (even if logical or \"obvious\"), it is an enhancement (CR6).\n- **Compliance data vs bugs**: Data accuracy/update needs -> CR8 (not CR3).\n- **Multi-element tickets**: Classify by highest criticality.\n- **Workaround feasibility**: If manual alternatives are infeasible at volume, treat as no workaround (CR3 over CR4).\n\n\nClassify this CargoWise ticket:\n\nTicket: Need custom API for bulk tariff updates—how much to fast-track?\nImpact: Manual updates too slow\nScope: New feature\nSignals: Not in current product; requesting pricing for quick dev","guidelines":""},"evaluator_results":[{"name":"json_schema_validator","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":[],"misses":[],"reasoning":"Valid JSON with all required keys: criticalityRating, reasoning","evaluator_provider_request":{"script":["uv","run","validate_output.py"],"cwd":"/root/projects/agentv/examples/showcase/cw-incident-triage/evals"}},{"name":"content_evaluator","type":"llm_judge","score":1,"weight":1,"verdict":"pass","hits":["Correctly classifies as CR7","Distinguishes new feature scope","Notes pricing/quote request for accelerated development","References rating definitions in reasoning"],"misses":[],"reasoning":"The candidate answer matches the reference exactly, providing comprehensive reasoning that identifies scope, pricing request, and correctly applies the CR7 definition.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nAssistant classifies as 'CR7' for accelerated development quote.\nReasoning distinguishes from CR6 by noting request for quote/pricing.\n\n[[ ## question ## ]]\n\n---\nname: cw-criticality-rating\ndescription: Classify CargoWise support tickets using criticality ratings CR1-CR9\n---\n\n# Task\n\nYou are a CargoWise support ticket triage specialist. Classify each ticket with CR1-CR9 based on scope, impact, workarounds, and request type.\n\n## Criticality Rating Definitions\n\n**CR1**: Entire system down; no user can access the application suite.\n**CR2**: Entire module down; module inaccessible for all users, other modules OK.\n**CR3**: Single function broken with no viable workaround; behavior deviates from docs or prior correct behavior.\n**CR4**: Single function broken but a viable workaround/manual alternative exists.\n**CR5**: Training/how-to request on existing functionality.\n**CR6**: Feature/enhancement request beyond documented behavior; not a defect.\n**CR7**: Request for pricing/quote on new features or accelerated work.\n**CR8**: Compliance/reference/master data updates or corrections; data accuracy issues, not functional bugs.\n**CR9**: Service/operational/administrative request.\n\n## Classification Process\n\n1. **Analyze scope and impact**: system vs module vs function; user/workstation scope.\n2. **Assess workarounds**: note whether alternatives exist and if they are feasible.\n3. **Distinguish defects from features**: defect = function not working as documented or changed from correct behavior; feature = request for new capability; if a previously fixed defect is now a substantial change request, treat as CR6. Always prioritize documentation and specifications over user-assigned labels (e.g., \"Critical Bug\")—if the system behaves as documented, it is not a defect.\n4. **Handle multi-part tickets**: classify by the highest criticality element.\n5. **Provide reasoning**: explain how the rating was reached and reference specific signals.\n\n## Output Format\n\nOutput ONLY valid JSON with this structure:\n\n```json\n{\n \"criticalityRating\": \"CRX\",\n \"reasoning\": \"Step 1: [First observation]. Step 2: [Second observation]. Step 3: [Final classification logic].\"\n}\n```\n\nThink step-by-step and explain your reasoning before concluding.\n\n## Edge Cases\n\n- **Post-fix enhancement requests**: If a defect was fixed per documentation but the user now wants substantial changes -> CR6.\n- **Gap between expectation and spec**: If the system works as documented but the user expects different behavior (even if logical or \"obvious\"), it is an enhancement (CR6).\n- **Compliance data vs bugs**: Data accuracy/update needs -> CR8 (not CR3).\n- **Multi-element tickets**: Classify by highest criticality.\n- **Workaround feasibility**: If manual alternatives are infeasible at volume, treat as no workaround (CR3 over CR4).\n\n\nClassify this CargoWise ticket:\n\nTicket: Need custom API for bulk tariff updates—how much to fast-track?\nImpact: Manual updates too slow\nScope: New feature\nSignals: Not in current product; requesting pricing for quick dev\n\n[[ ## reference_answer ## ]]\n{\n \"criticalityRating\": \"CR7\",\n \"reasoning\": \"Step 1: Requests new feature (bulk API). Step 2: Specifically asks for quote on accelerated development. Step 3: Matches CR7 over CR6 due to pricing/acceleration focus.\"\n}\n\n[[ ## candidate_answer ## ]]\n```json\n{\n \"criticalityRating\": \"CR7\",\n \"reasoning\": \"Step 1: The ticket requests a custom API for bulk tariff updates, which is outside the current product's documented behavior (scope: new feature). Step 2: The user specifically asks how much it would cost to fast-track this development, indicating a pricing/quote request for accelerated work. Step 3: According to the definitions, requests for pricing/quote on new features or expedited development are classified as CR7.\"\n}\n```","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"default"}}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} +{"timestamp":"2026-01-03T12:08:38.988Z","eval_id":"cr-function-bug-no-workaround","dataset":"dataset","conversation_id":"cargowise-triage","score":1,"hits":["Correctly identifies single function affected (invoice generation)","Confirms deviation from documentation and previous correct behavior","Explicitly notes absence of manual workaround due to volume","Accurately classifies as CR3 per definitions"],"misses":[],"candidate_answer":"```json\n{\n \"criticalityRating\": \"CR3\",\n \"reasoning\": \"Step 1: The issue affects a single function—invoice generation totals. Step 2: The behavior deviates from documentation and historical correct performance, indicating a defect. Step 3: No manual workaround is feasible due to volume, fulfilling the criteria for CR3: single function broken, no viable workaround. Therefore, the ticket is classified as CR3.\"\n}\n```","target":"default","reasoning":"json_schema_validator: Valid JSON with all required keys: criticalityRating, reasoning | content_evaluator: The candidate answer fully addresses scope, documentation mismatch, prior correctness, and lack of workaround as required for CR3, with stepwise reasoning mirroring the reference answer.","lm_provider_request":{"question":"\n---\nname: cw-criticality-rating\ndescription: Classify CargoWise support tickets using criticality ratings CR1-CR9\n---\n\n# Task\n\nYou are a CargoWise support ticket triage specialist. Classify each ticket with CR1-CR9 based on scope, impact, workarounds, and request type.\n\n## Criticality Rating Definitions\n\n**CR1**: Entire system down; no user can access the application suite.\n**CR2**: Entire module down; module inaccessible for all users, other modules OK.\n**CR3**: Single function broken with no viable workaround; behavior deviates from docs or prior correct behavior.\n**CR4**: Single function broken but a viable workaround/manual alternative exists.\n**CR5**: Training/how-to request on existing functionality.\n**CR6**: Feature/enhancement request beyond documented behavior; not a defect.\n**CR7**: Request for pricing/quote on new features or accelerated work.\n**CR8**: Compliance/reference/master data updates or corrections; data accuracy issues, not functional bugs.\n**CR9**: Service/operational/administrative request.\n\n## Classification Process\n\n1. **Analyze scope and impact**: system vs module vs function; user/workstation scope.\n2. **Assess workarounds**: note whether alternatives exist and if they are feasible.\n3. **Distinguish defects from features**: defect = function not working as documented or changed from correct behavior; feature = request for new capability; if a previously fixed defect is now a substantial change request, treat as CR6. Always prioritize documentation and specifications over user-assigned labels (e.g., \"Critical Bug\")—if the system behaves as documented, it is not a defect.\n4. **Handle multi-part tickets**: classify by the highest criticality element.\n5. **Provide reasoning**: explain how the rating was reached and reference specific signals.\n\n## Output Format\n\nOutput ONLY valid JSON with this structure:\n\n```json\n{\n \"criticalityRating\": \"CRX\",\n \"reasoning\": \"Step 1: [First observation]. Step 2: [Second observation]. Step 3: [Final classification logic].\"\n}\n```\n\nThink step-by-step and explain your reasoning before concluding.\n\n## Edge Cases\n\n- **Post-fix enhancement requests**: If a defect was fixed per documentation but the user now wants substantial changes -> CR6.\n- **Gap between expectation and spec**: If the system works as documented but the user expects different behavior (even if logical or \"obvious\"), it is an enhancement (CR6).\n- **Compliance data vs bugs**: Data accuracy/update needs -> CR8 (not CR3).\n- **Multi-element tickets**: Classify by highest criticality.\n- **Workaround feasibility**: If manual alternatives are infeasible at volume, treat as no workaround (CR3 over CR4).\n\n\nClassify this CargoWise ticket:\n\nTicket: Invoice generation producing incorrect totals\nImpact: Invoices off by 5-10%\nScope: Single function\nSignals: Worked correctly last month; now deviates from doc specs; no manual calc alternative feasible for volume","guidelines":""},"evaluator_results":[{"name":"json_schema_validator","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":[],"misses":[],"reasoning":"Valid JSON with all required keys: criticalityRating, reasoning","evaluator_provider_request":{"script":["uv","run","validate_output.py"],"cwd":"/root/projects/agentv/examples/showcase/cw-incident-triage/evals"}},{"name":"content_evaluator","type":"llm_judge","score":1,"weight":1,"verdict":"pass","hits":["Correctly identifies single function affected (invoice generation)","Confirms deviation from documentation and previous correct behavior","Explicitly notes absence of manual workaround due to volume","Accurately classifies as CR3 per definitions"],"misses":[],"reasoning":"The candidate answer fully addresses scope, documentation mismatch, prior correctness, and lack of workaround as required for CR3, with stepwise reasoning mirroring the reference answer.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nAssistant classifies as 'CR3' for function not matching documentation.\nReasoning must confirm no workaround and tie to 'changed from previously correct behaviour'.\n\n[[ ## question ## ]]\n\n---\nname: cw-criticality-rating\ndescription: Classify CargoWise support tickets using criticality ratings CR1-CR9\n---\n\n# Task\n\nYou are a CargoWise support ticket triage specialist. Classify each ticket with CR1-CR9 based on scope, impact, workarounds, and request type.\n\n## Criticality Rating Definitions\n\n**CR1**: Entire system down; no user can access the application suite.\n**CR2**: Entire module down; module inaccessible for all users, other modules OK.\n**CR3**: Single function broken with no viable workaround; behavior deviates from docs or prior correct behavior.\n**CR4**: Single function broken but a viable workaround/manual alternative exists.\n**CR5**: Training/how-to request on existing functionality.\n**CR6**: Feature/enhancement request beyond documented behavior; not a defect.\n**CR7**: Request for pricing/quote on new features or accelerated work.\n**CR8**: Compliance/reference/master data updates or corrections; data accuracy issues, not functional bugs.\n**CR9**: Service/operational/administrative request.\n\n## Classification Process\n\n1. **Analyze scope and impact**: system vs module vs function; user/workstation scope.\n2. **Assess workarounds**: note whether alternatives exist and if they are feasible.\n3. **Distinguish defects from features**: defect = function not working as documented or changed from correct behavior; feature = request for new capability; if a previously fixed defect is now a substantial change request, treat as CR6. Always prioritize documentation and specifications over user-assigned labels (e.g., \"Critical Bug\")—if the system behaves as documented, it is not a defect.\n4. **Handle multi-part tickets**: classify by the highest criticality element.\n5. **Provide reasoning**: explain how the rating was reached and reference specific signals.\n\n## Output Format\n\nOutput ONLY valid JSON with this structure:\n\n```json\n{\n \"criticalityRating\": \"CRX\",\n \"reasoning\": \"Step 1: [First observation]. Step 2: [Second observation]. Step 3: [Final classification logic].\"\n}\n```\n\nThink step-by-step and explain your reasoning before concluding.\n\n## Edge Cases\n\n- **Post-fix enhancement requests**: If a defect was fixed per documentation but the user now wants substantial changes -> CR6.\n- **Gap between expectation and spec**: If the system works as documented but the user expects different behavior (even if logical or \"obvious\"), it is an enhancement (CR6).\n- **Compliance data vs bugs**: Data accuracy/update needs -> CR8 (not CR3).\n- **Multi-element tickets**: Classify by highest criticality.\n- **Workaround feasibility**: If manual alternatives are infeasible at volume, treat as no workaround (CR3 over CR4).\n\n\nClassify this CargoWise ticket:\n\nTicket: Invoice generation producing incorrect totals\nImpact: Invoices off by 5-10%\nScope: Single function\nSignals: Worked correctly last month; now deviates from doc specs; no manual calc alternative feasible for volume\n\n[[ ## reference_answer ## ]]\n{\n \"criticalityRating\": \"CR3\",\n \"reasoning\": \"Step 1: Function (invoice generation) not working as documented. Step 2: Changed from prior correct behavior. Step 3: No viable workaround—matches CR3.\"\n}\n\n[[ ## candidate_answer ## ]]\n```json\n{\n \"criticalityRating\": \"CR3\",\n \"reasoning\": \"Step 1: The issue affects a single function—invoice generation totals. Step 2: The behavior deviates from documentation and historical correct performance, indicating a defect. Step 3: No manual workaround is feasible due to volume, fulfilling the criteria for CR3: single function broken, no viable workaround. Therefore, the ticket is classified as CR3.\"\n}\n```","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"default"}}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} +{"timestamp":"2026-01-03T12:08:39.679Z","eval_id":"cr-compliance-data-update","dataset":"dataset","conversation_id":"cargowise-triage","score":1,"hits":["Correctly classified as CR8","Highlighted master data nature of HS codes","Distinguished data updates from functional defects","Referenced risk due to inaccurate reference data"],"misses":[],"candidate_answer":"```json\n{\n \"criticalityRating\": \"CR8\",\n \"reasoning\": \"Step 1: The issue concerns outdated HS codes in the compliance database, which are part of system master data rather than application functions. Step 2: The main impact is a risk of customs penalties due to inaccurate reference data, not a functional defect in the application. Step 3: Based on the definitions, compliance/reference/master data updates or corrections, especially those affecting data accuracy, are classified as CR8. Therefore, this ticket fits CR8 as it is a request for updated compliance data rather than a functional bug.\"\n}\n```","target":"default","reasoning":"json_schema_validator: Valid JSON with all required keys: criticalityRating, reasoning | content_evaluator: This answer precisely follows classification guidelines, prioritizes data accuracy, and matches the logic and detail of the reference answer with step-by-step justification.","lm_provider_request":{"question":"\n---\nname: cw-criticality-rating\ndescription: Classify CargoWise support tickets using criticality ratings CR1-CR9\n---\n\n# Task\n\nYou are a CargoWise support ticket triage specialist. Classify each ticket with CR1-CR9 based on scope, impact, workarounds, and request type.\n\n## Criticality Rating Definitions\n\n**CR1**: Entire system down; no user can access the application suite.\n**CR2**: Entire module down; module inaccessible for all users, other modules OK.\n**CR3**: Single function broken with no viable workaround; behavior deviates from docs or prior correct behavior.\n**CR4**: Single function broken but a viable workaround/manual alternative exists.\n**CR5**: Training/how-to request on existing functionality.\n**CR6**: Feature/enhancement request beyond documented behavior; not a defect.\n**CR7**: Request for pricing/quote on new features or accelerated work.\n**CR8**: Compliance/reference/master data updates or corrections; data accuracy issues, not functional bugs.\n**CR9**: Service/operational/administrative request.\n\n## Classification Process\n\n1. **Analyze scope and impact**: system vs module vs function; user/workstation scope.\n2. **Assess workarounds**: note whether alternatives exist and if they are feasible.\n3. **Distinguish defects from features**: defect = function not working as documented or changed from correct behavior; feature = request for new capability; if a previously fixed defect is now a substantial change request, treat as CR6. Always prioritize documentation and specifications over user-assigned labels (e.g., \"Critical Bug\")—if the system behaves as documented, it is not a defect.\n4. **Handle multi-part tickets**: classify by the highest criticality element.\n5. **Provide reasoning**: explain how the rating was reached and reference specific signals.\n\n## Output Format\n\nOutput ONLY valid JSON with this structure:\n\n```json\n{\n \"criticalityRating\": \"CRX\",\n \"reasoning\": \"Step 1: [First observation]. Step 2: [Second observation]. Step 3: [Final classification logic].\"\n}\n```\n\nThink step-by-step and explain your reasoning before concluding.\n\n## Edge Cases\n\n- **Post-fix enhancement requests**: If a defect was fixed per documentation but the user now wants substantial changes -> CR6.\n- **Gap between expectation and spec**: If the system works as documented but the user expects different behavior (even if logical or \"obvious\"), it is an enhancement (CR6).\n- **Compliance data vs bugs**: Data accuracy/update needs -> CR8 (not CR3).\n- **Multi-element tickets**: Classify by highest criticality.\n- **Workaround feasibility**: If manual alternatives are infeasible at volume, treat as no workaround (CR3 over CR4).\n\n\nClassify this CargoWise ticket:\n\nTicket: Outdated HS codes in compliance database\nImpact: Risk of customs penalties\nScope: Master data\nSignals: Codes changed per recent regulation; need update in system reference","guidelines":""},"evaluator_results":[{"name":"json_schema_validator","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":[],"misses":[],"reasoning":"Valid JSON with all required keys: criticalityRating, reasoning","evaluator_provider_request":{"script":["uv","run","validate_output.py"],"cwd":"/root/projects/agentv/examples/showcase/cw-incident-triage/evals"}},{"name":"content_evaluator","type":"llm_judge","score":1,"weight":1,"verdict":"pass","hits":["Correctly classified as CR8","Highlighted master data nature of HS codes","Distinguished data updates from functional defects","Referenced risk due to inaccurate reference data"],"misses":[],"reasoning":"This answer precisely follows classification guidelines, prioritizes data accuracy, and matches the logic and detail of the reference answer with step-by-step justification.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nAssistant classifies as 'CR8' for master data/compliance issue.\nReasoning prioritizes data accuracy over potential bug claims.\n\n[[ ## question ## ]]\n\n---\nname: cw-criticality-rating\ndescription: Classify CargoWise support tickets using criticality ratings CR1-CR9\n---\n\n# Task\n\nYou are a CargoWise support ticket triage specialist. Classify each ticket with CR1-CR9 based on scope, impact, workarounds, and request type.\n\n## Criticality Rating Definitions\n\n**CR1**: Entire system down; no user can access the application suite.\n**CR2**: Entire module down; module inaccessible for all users, other modules OK.\n**CR3**: Single function broken with no viable workaround; behavior deviates from docs or prior correct behavior.\n**CR4**: Single function broken but a viable workaround/manual alternative exists.\n**CR5**: Training/how-to request on existing functionality.\n**CR6**: Feature/enhancement request beyond documented behavior; not a defect.\n**CR7**: Request for pricing/quote on new features or accelerated work.\n**CR8**: Compliance/reference/master data updates or corrections; data accuracy issues, not functional bugs.\n**CR9**: Service/operational/administrative request.\n\n## Classification Process\n\n1. **Analyze scope and impact**: system vs module vs function; user/workstation scope.\n2. **Assess workarounds**: note whether alternatives exist and if they are feasible.\n3. **Distinguish defects from features**: defect = function not working as documented or changed from correct behavior; feature = request for new capability; if a previously fixed defect is now a substantial change request, treat as CR6. Always prioritize documentation and specifications over user-assigned labels (e.g., \"Critical Bug\")—if the system behaves as documented, it is not a defect.\n4. **Handle multi-part tickets**: classify by the highest criticality element.\n5. **Provide reasoning**: explain how the rating was reached and reference specific signals.\n\n## Output Format\n\nOutput ONLY valid JSON with this structure:\n\n```json\n{\n \"criticalityRating\": \"CRX\",\n \"reasoning\": \"Step 1: [First observation]. Step 2: [Second observation]. Step 3: [Final classification logic].\"\n}\n```\n\nThink step-by-step and explain your reasoning before concluding.\n\n## Edge Cases\n\n- **Post-fix enhancement requests**: If a defect was fixed per documentation but the user now wants substantial changes -> CR6.\n- **Gap between expectation and spec**: If the system works as documented but the user expects different behavior (even if logical or \"obvious\"), it is an enhancement (CR6).\n- **Compliance data vs bugs**: Data accuracy/update needs -> CR8 (not CR3).\n- **Multi-element tickets**: Classify by highest criticality.\n- **Workaround feasibility**: If manual alternatives are infeasible at volume, treat as no workaround (CR3 over CR4).\n\n\nClassify this CargoWise ticket:\n\nTicket: Outdated HS codes in compliance database\nImpact: Risk of customs penalties\nScope: Master data\nSignals: Codes changed per recent regulation; need update in system reference\n\n[[ ## reference_answer ## ]]\n{\n \"criticalityRating\": \"CR8\",\n \"reasoning\": \"Step 1: Involves compliance/reference data (HS codes). Step 2: Not a function bug but data update need. Step 3: Matches CR8 for master data handling.\"\n}\n\n[[ ## candidate_answer ## ]]\n```json\n{\n \"criticalityRating\": \"CR8\",\n \"reasoning\": \"Step 1: The issue concerns outdated HS codes in the compliance database, which are part of system master data rather than application functions. Step 2: The main impact is a risk of customs penalties due to inaccurate reference data, not a functional defect in the application. Step 3: Based on the definitions, compliance/reference/master data updates or corrections, especially those affecting data accuracy, are classified as CR8. Therefore, this ticket fits CR8 as it is a request for updated compliance data rather than a functional bug.\"\n}\n```","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"default"}}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} +{"timestamp":"2026-01-03T12:08:41.411Z","eval_id":"cr-multi-part-blend","dataset":"dataset","conversation_id":"cargowise-triage","score":1,"hits":["Correctly classified as CR3","Identified multi-part ticket: bug + training","Clearly prioritized the bug over training request","Referenced deviation from documentation and lack of workaround"],"misses":[],"candidate_answer":"```json\n{\n \"criticalityRating\": \"CR3\",\n \"reasoning\": \"Step 1: The main issue is shipment tracking, which is a single functional area and is erroring out for users. Step 2: The behavior has deviated from documentation, and there is no viable workaround for shipment tracking, so this is a defect of significant impact. Step 3: While there is also a request for training (would be CR5 on its own), the ticket's classification prioritizes the highest criticality element, which is the broken shipment tracking with no workaround, fitting CR3.\"\n}\n```","target":"default","reasoning":"json_schema_validator: Valid JSON with all required keys: criticalityRating, reasoning | content_evaluator: The answer accurately applies the process, prioritizes the highest criticality, and provides clear step-by-step reasoning matching the reference answer.","lm_provider_request":{"question":"\n---\nname: cw-criticality-rating\ndescription: Classify CargoWise support tickets using criticality ratings CR1-CR9\n---\n\n# Task\n\nYou are a CargoWise support ticket triage specialist. Classify each ticket with CR1-CR9 based on scope, impact, workarounds, and request type.\n\n## Criticality Rating Definitions\n\n**CR1**: Entire system down; no user can access the application suite.\n**CR2**: Entire module down; module inaccessible for all users, other modules OK.\n**CR3**: Single function broken with no viable workaround; behavior deviates from docs or prior correct behavior.\n**CR4**: Single function broken but a viable workaround/manual alternative exists.\n**CR5**: Training/how-to request on existing functionality.\n**CR6**: Feature/enhancement request beyond documented behavior; not a defect.\n**CR7**: Request for pricing/quote on new features or accelerated work.\n**CR8**: Compliance/reference/master data updates or corrections; data accuracy issues, not functional bugs.\n**CR9**: Service/operational/administrative request.\n\n## Classification Process\n\n1. **Analyze scope and impact**: system vs module vs function; user/workstation scope.\n2. **Assess workarounds**: note whether alternatives exist and if they are feasible.\n3. **Distinguish defects from features**: defect = function not working as documented or changed from correct behavior; feature = request for new capability; if a previously fixed defect is now a substantial change request, treat as CR6. Always prioritize documentation and specifications over user-assigned labels (e.g., \"Critical Bug\")—if the system behaves as documented, it is not a defect.\n4. **Handle multi-part tickets**: classify by the highest criticality element.\n5. **Provide reasoning**: explain how the rating was reached and reference specific signals.\n\n## Output Format\n\nOutput ONLY valid JSON with this structure:\n\n```json\n{\n \"criticalityRating\": \"CRX\",\n \"reasoning\": \"Step 1: [First observation]. Step 2: [Second observation]. Step 3: [Final classification logic].\"\n}\n```\n\nThink step-by-step and explain your reasoning before concluding.\n\n## Edge Cases\n\n- **Post-fix enhancement requests**: If a defect was fixed per documentation but the user now wants substantial changes -> CR6.\n- **Gap between expectation and spec**: If the system works as documented but the user expects different behavior (even if logical or \"obvious\"), it is an enhancement (CR6).\n- **Compliance data vs bugs**: Data accuracy/update needs -> CR8 (not CR3).\n- **Multi-element tickets**: Classify by highest criticality.\n- **Workaround feasibility**: If manual alternatives are infeasible at volume, treat as no workaround (CR3 over CR4).\n\n\nClassify this CargoWise ticket:\n\nTicket: Shipment tracking erroring out; also need training on new UI\nImpact: Tracking unavailable; team confused\nScope: Function + training\nSignals: Tracking deviated from docs; no workaround; training secondary","guidelines":""},"evaluator_results":[{"name":"json_schema_validator","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":[],"misses":[],"reasoning":"Valid JSON with all required keys: criticalityRating, reasoning","evaluator_provider_request":{"script":["uv","run","validate_output.py"],"cwd":"/root/projects/agentv/examples/showcase/cw-incident-triage/evals"}},{"name":"content_evaluator","type":"llm_judge","score":1,"weight":1,"verdict":"pass","hits":["Correctly classified as CR3","Identified multi-part ticket: bug + training","Clearly prioritized the bug over training request","Referenced deviation from documentation and lack of workaround"],"misses":[],"reasoning":"The answer accurately applies the process, prioritizes the highest criticality, and provides clear step-by-step reasoning matching the reference answer.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nAssistant classifies as 'CR3' prioritizing the bug over secondary training request.\nReasoning must identify multi-elements and select highest criticality.\n\n[[ ## question ## ]]\n\n---\nname: cw-criticality-rating\ndescription: Classify CargoWise support tickets using criticality ratings CR1-CR9\n---\n\n# Task\n\nYou are a CargoWise support ticket triage specialist. Classify each ticket with CR1-CR9 based on scope, impact, workarounds, and request type.\n\n## Criticality Rating Definitions\n\n**CR1**: Entire system down; no user can access the application suite.\n**CR2**: Entire module down; module inaccessible for all users, other modules OK.\n**CR3**: Single function broken with no viable workaround; behavior deviates from docs or prior correct behavior.\n**CR4**: Single function broken but a viable workaround/manual alternative exists.\n**CR5**: Training/how-to request on existing functionality.\n**CR6**: Feature/enhancement request beyond documented behavior; not a defect.\n**CR7**: Request for pricing/quote on new features or accelerated work.\n**CR8**: Compliance/reference/master data updates or corrections; data accuracy issues, not functional bugs.\n**CR9**: Service/operational/administrative request.\n\n## Classification Process\n\n1. **Analyze scope and impact**: system vs module vs function; user/workstation scope.\n2. **Assess workarounds**: note whether alternatives exist and if they are feasible.\n3. **Distinguish defects from features**: defect = function not working as documented or changed from correct behavior; feature = request for new capability; if a previously fixed defect is now a substantial change request, treat as CR6. Always prioritize documentation and specifications over user-assigned labels (e.g., \"Critical Bug\")—if the system behaves as documented, it is not a defect.\n4. **Handle multi-part tickets**: classify by the highest criticality element.\n5. **Provide reasoning**: explain how the rating was reached and reference specific signals.\n\n## Output Format\n\nOutput ONLY valid JSON with this structure:\n\n```json\n{\n \"criticalityRating\": \"CRX\",\n \"reasoning\": \"Step 1: [First observation]. Step 2: [Second observation]. Step 3: [Final classification logic].\"\n}\n```\n\nThink step-by-step and explain your reasoning before concluding.\n\n## Edge Cases\n\n- **Post-fix enhancement requests**: If a defect was fixed per documentation but the user now wants substantial changes -> CR6.\n- **Gap between expectation and spec**: If the system works as documented but the user expects different behavior (even if logical or \"obvious\"), it is an enhancement (CR6).\n- **Compliance data vs bugs**: Data accuracy/update needs -> CR8 (not CR3).\n- **Multi-element tickets**: Classify by highest criticality.\n- **Workaround feasibility**: If manual alternatives are infeasible at volume, treat as no workaround (CR3 over CR4).\n\n\nClassify this CargoWise ticket:\n\nTicket: Shipment tracking erroring out; also need training on new UI\nImpact: Tracking unavailable; team confused\nScope: Function + training\nSignals: Tracking deviated from docs; no workaround; training secondary\n\n[[ ## reference_answer ## ]]\n{\n \"criticalityRating\": \"CR3\",\n \"reasoning\": \"Step 1: Multi-part: Function bug (tracking error) + training request. Step 2: Bug matches CR3 (not as documented, no workaround). Step 3: Prioritize highest criticality (CR3 over CR5).\"\n}\n\n[[ ## candidate_answer ## ]]\n```json\n{\n \"criticalityRating\": \"CR3\",\n \"reasoning\": \"Step 1: The main issue is shipment tracking, which is a single functional area and is erroring out for users. Step 2: The behavior has deviated from documentation, and there is no viable workaround for shipment tracking, so this is a defect of significant impact. Step 3: While there is also a request for training (would be CR5 on its own), the ticket's classification prioritizes the highest criticality element, which is the broken shipment tracking with no workaround, fitting CR3.\"\n}\n```","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"default"}}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} +{"timestamp":"2026-01-03T12:08:41.508Z","eval_id":"cr-workaround-exists","dataset":"dataset","conversation_id":"cargowise-triage","score":1,"hits":["Correctly assigns CR4 rating","Accurately identifies single function impact","Recognizes presence of viable workaround (API export)","Follows classification definitions and process"],"misses":[],"candidate_answer":"```json\n{\n \"criticalityRating\": \"CR4\",\n \"reasoning\": \"Step 1: The impact is that users cannot export reports directly from the UI, which affects a single function. Step 2: There is a manual alternative available, as users can export reports via the API, providing a viable workaround. Step 3: Based on the definitions, a single function broken with a viable workaround or manual alternative should be classified as CR4.\"\n}\n```","target":"default","reasoning":"json_schema_validator: Valid JSON with all required keys: criticalityRating, reasoning | content_evaluator: The candidate correctly matches the expected classification, highlights the workaround, and provides stepwise reasoning that aligns with the schema and intent.","lm_provider_request":{"question":"\n---\nname: cw-criticality-rating\ndescription: Classify CargoWise support tickets using criticality ratings CR1-CR9\n---\n\n# Task\n\nYou are a CargoWise support ticket triage specialist. Classify each ticket with CR1-CR9 based on scope, impact, workarounds, and request type.\n\n## Criticality Rating Definitions\n\n**CR1**: Entire system down; no user can access the application suite.\n**CR2**: Entire module down; module inaccessible for all users, other modules OK.\n**CR3**: Single function broken with no viable workaround; behavior deviates from docs or prior correct behavior.\n**CR4**: Single function broken but a viable workaround/manual alternative exists.\n**CR5**: Training/how-to request on existing functionality.\n**CR6**: Feature/enhancement request beyond documented behavior; not a defect.\n**CR7**: Request for pricing/quote on new features or accelerated work.\n**CR8**: Compliance/reference/master data updates or corrections; data accuracy issues, not functional bugs.\n**CR9**: Service/operational/administrative request.\n\n## Classification Process\n\n1. **Analyze scope and impact**: system vs module vs function; user/workstation scope.\n2. **Assess workarounds**: note whether alternatives exist and if they are feasible.\n3. **Distinguish defects from features**: defect = function not working as documented or changed from correct behavior; feature = request for new capability; if a previously fixed defect is now a substantial change request, treat as CR6. Always prioritize documentation and specifications over user-assigned labels (e.g., \"Critical Bug\")—if the system behaves as documented, it is not a defect.\n4. **Handle multi-part tickets**: classify by the highest criticality element.\n5. **Provide reasoning**: explain how the rating was reached and reference specific signals.\n\n## Output Format\n\nOutput ONLY valid JSON with this structure:\n\n```json\n{\n \"criticalityRating\": \"CRX\",\n \"reasoning\": \"Step 1: [First observation]. Step 2: [Second observation]. Step 3: [Final classification logic].\"\n}\n```\n\nThink step-by-step and explain your reasoning before concluding.\n\n## Edge Cases\n\n- **Post-fix enhancement requests**: If a defect was fixed per documentation but the user now wants substantial changes -> CR6.\n- **Gap between expectation and spec**: If the system works as documented but the user expects different behavior (even if logical or \"obvious\"), it is an enhancement (CR6).\n- **Compliance data vs bugs**: Data accuracy/update needs -> CR8 (not CR3).\n- **Multi-element tickets**: Classify by highest criticality.\n- **Workaround feasibility**: If manual alternatives are infeasible at volume, treat as no workaround (CR3 over CR4).\n\n\nClassify this CargoWise ticket:\n\nTicket: Report export failing in UI\nImpact: Can't export directly\nScope: Single function\nSignals: Can export via API as manual alternative","guidelines":""},"evaluator_results":[{"name":"json_schema_validator","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":[],"misses":[],"reasoning":"Valid JSON with all required keys: criticalityRating, reasoning","evaluator_provider_request":{"script":["uv","run","validate_output.py"],"cwd":"/root/projects/agentv/examples/showcase/cw-incident-triage/evals"}},{"name":"content_evaluator","type":"llm_judge","score":1,"weight":1,"verdict":"pass","hits":["Correctly assigns CR4 rating","Accurately identifies single function impact","Recognizes presence of viable workaround (API export)","Follows classification definitions and process"],"misses":[],"reasoning":"The candidate correctly matches the expected classification, highlights the workaround, and provides stepwise reasoning that aligns with the schema and intent.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nAssistant classifies as 'CR4' due to viable workaround.\nReasoning distinguishes from CR3 by noting alternative.\n\n[[ ## question ## ]]\n\n---\nname: cw-criticality-rating\ndescription: Classify CargoWise support tickets using criticality ratings CR1-CR9\n---\n\n# Task\n\nYou are a CargoWise support ticket triage specialist. Classify each ticket with CR1-CR9 based on scope, impact, workarounds, and request type.\n\n## Criticality Rating Definitions\n\n**CR1**: Entire system down; no user can access the application suite.\n**CR2**: Entire module down; module inaccessible for all users, other modules OK.\n**CR3**: Single function broken with no viable workaround; behavior deviates from docs or prior correct behavior.\n**CR4**: Single function broken but a viable workaround/manual alternative exists.\n**CR5**: Training/how-to request on existing functionality.\n**CR6**: Feature/enhancement request beyond documented behavior; not a defect.\n**CR7**: Request for pricing/quote on new features or accelerated work.\n**CR8**: Compliance/reference/master data updates or corrections; data accuracy issues, not functional bugs.\n**CR9**: Service/operational/administrative request.\n\n## Classification Process\n\n1. **Analyze scope and impact**: system vs module vs function; user/workstation scope.\n2. **Assess workarounds**: note whether alternatives exist and if they are feasible.\n3. **Distinguish defects from features**: defect = function not working as documented or changed from correct behavior; feature = request for new capability; if a previously fixed defect is now a substantial change request, treat as CR6. Always prioritize documentation and specifications over user-assigned labels (e.g., \"Critical Bug\")—if the system behaves as documented, it is not a defect.\n4. **Handle multi-part tickets**: classify by the highest criticality element.\n5. **Provide reasoning**: explain how the rating was reached and reference specific signals.\n\n## Output Format\n\nOutput ONLY valid JSON with this structure:\n\n```json\n{\n \"criticalityRating\": \"CRX\",\n \"reasoning\": \"Step 1: [First observation]. Step 2: [Second observation]. Step 3: [Final classification logic].\"\n}\n```\n\nThink step-by-step and explain your reasoning before concluding.\n\n## Edge Cases\n\n- **Post-fix enhancement requests**: If a defect was fixed per documentation but the user now wants substantial changes -> CR6.\n- **Gap between expectation and spec**: If the system works as documented but the user expects different behavior (even if logical or \"obvious\"), it is an enhancement (CR6).\n- **Compliance data vs bugs**: Data accuracy/update needs -> CR8 (not CR3).\n- **Multi-element tickets**: Classify by highest criticality.\n- **Workaround feasibility**: If manual alternatives are infeasible at volume, treat as no workaround (CR3 over CR4).\n\n\nClassify this CargoWise ticket:\n\nTicket: Report export failing in UI\nImpact: Can't export directly\nScope: Single function\nSignals: Can export via API as manual alternative\n\n[[ ## reference_answer ## ]]\n{\n \"criticalityRating\": \"CR4\",\n \"reasoning\": \"Step 1: Function (report export) not working. Step 2: Viable workaround (API export) exists. Step 3: Matches CR4 over CR3.\"\n}\n\n[[ ## candidate_answer ## ]]\n```json\n{\n \"criticalityRating\": \"CR4\",\n \"reasoning\": \"Step 1: The impact is that users cannot export reports directly from the UI, which affects a single function. Step 2: There is a manual alternative available, as users can export reports via the API, providing a viable workaround. Step 3: Based on the definitions, a single function broken with a viable workaround or manual alternative should be classified as CR4.\"\n}\n```","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"default"}}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} diff --git a/examples/showcase/cw-incident-triage/evals/dataset.yaml b/examples/showcase/cw-incident-triage/evals/dataset.yaml index 1d504752..73221a94 100644 --- a/examples/showcase/cw-incident-triage/evals/dataset.yaml +++ b/examples/showcase/cw-incident-triage/evals/dataset.yaml @@ -11,7 +11,7 @@ execution: evaluators: - name: json_schema_validator type: code_judge - script: uv run validate_output.py + script: ["uv", "run", "validate_output.py"] - name: content_evaluator type: llm_judge @@ -259,4 +259,4 @@ evalcases: { "criticalityRating": "CR4", "reasoning": "Step 1: Function (report export) not working. Step 2: Viable workaround (API export) exists. Step 3: Matches CR4 over CR3." - } \ No newline at end of file + } diff --git a/examples/showcase/export-screening/evals/dataset.yaml b/examples/showcase/export-screening/evals/dataset.yaml index 0a0a9d69..988604ff 100644 --- a/examples/showcase/export-screening/evals/dataset.yaml +++ b/examples/showcase/export-screening/evals/dataset.yaml @@ -22,7 +22,7 @@ execution: evaluators: - name: risk_assessment_quality type: code_judge - script: bun run validate_risk_output.ts + script: ["bun", "run", "validate_risk_output.ts"] evalcases: # ============================================ diff --git a/examples/showcase/export-screening/evals/validate_risk_output.ts b/examples/showcase/export-screening/evals/validate_risk_output.ts index c3a4daaf..e7ffbd29 100644 --- a/examples/showcase/export-screening/evals/validate_risk_output.ts +++ b/examples/showcase/export-screening/evals/validate_risk_output.ts @@ -12,8 +12,8 @@ const VALID_RISK_LEVELS = new Set(['High', 'Medium', 'Low']); const REQUIRED_KEYS = ['riskLevel', 'reasoning']; interface EvalInput { - candidateAnswer: string; - expectedMessages?: Array<{ + candidate_answer: string; + expected_messages?: Array<{ role: string; content: unknown; }>; @@ -170,7 +170,7 @@ async function main(): Promise { process.exit(1); } - const result = validateRiskOutput(evalData.candidateAnswer ?? '', evalData.expectedMessages); + const result = validateRiskOutput(evalData.candidate_answer ?? '', evalData.expected_messages); console.log(JSON.stringify(result, null, 2)); } diff --git a/examples/showcase/psychotherapy/evals/dataset-encouragement.baseline.jsonl b/examples/showcase/psychotherapy/evals/dataset-encouragement.baseline.jsonl new file mode 100644 index 00000000..d151ab42 --- /dev/null +++ b/examples/showcase/psychotherapy/evals/dataset-encouragement.baseline.jsonl @@ -0,0 +1,8 @@ +{"timestamp":"2026-01-03T12:09:13.115Z","eval_id":"encouragement-guilty-sister","dataset":"dataset-encouragement","score":1,"hits":["Normalizes and validates guilt as understandable in context","Reframes guilt as evidence of empathy and love","Highlights emotional support as a resource","Draft response follows 'One-Key Three-Links' structure"],"misses":[],"candidate_answer":"```json\n{\n \"analysis\": {\n \"validation_point\": \"It is understandable and normal to feel guilt and helplessness when someone you love is suffering from a serious illness, especially when you wish you could help but feel powerless.\",\n \"resource_identified\": \"Despite the pain and sense of helplessness, the client demonstrated deep empathy and compassion by being attentive to her sister's needs, expressing her concern, and searching for ways to support her emotionally.\",\n \"reframe_angle\": \"Shift from feeling guilt about not being able to 'do anything' to recognizing that her caring and emotional presence are already powerful acts of support and love, reflecting how much she values her sister.\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links (See, Understand, Support)\",\n \"draft_response\": \"1. Validate (See/Understand): 'It’s really hard to watch someone you care about go through such a challenging illness, and it’s understandable to feel guilty and helpless in this situation.' 2. Highlight Resource (Support): 'Yet, I notice that you are deeply concerned, staying emotionally present for her and searching for ways to support her, which already shows a lot of strength and care.' 3. Reframe (Empower): 'This shows how much you love and value your sister, and your willingness to stand by her side is a powerful form of support, even when you feel limited.'\"\n }\n}\n```","target":"default","reasoning":"json_schema_validator: Valid JSON with all required keys: analysis | content_evaluator: The candidate answer clearly reframes guilt as love, normalizes the feeling, and describes the client's caring presence as a resource. It closely matches the reference answer and fully achieves the expected outcome.","lm_provider_request":{"question":"Analyze this statement using the \"Resource-Focused\" framework.\n\nClient Statement:\n\"My sister is only 21, but I just found out she has uremia. She needs dialysis every week. I'm her older sister, only two years older. Seeing her suffer while I am so healthy... I feel so guilty. I can't do anything for her.\"\n\n\n---\ndescription: 'Apply resource-focused encouragement therapy using One-Key Three-Links approach'\n---\n\n# Resource-Focused Encouragement Framework\n\nYou are an expert Psychotherapist specializing in Encouragement Technology (鼓励技术) and Resource-Focused (资源取向) therapy.\n\n## Task\nShift the client's perspective from \"Problem/Deficit\" to \"Resource/Coping\" using the \"One-Key Three-Links\" approach (See, Understand, Support + Empower).\n\n## Input\nClient statement.\n\n## Analysis Steps\n\n### 1. See & Understand (Validation)\nAcknowledge the pain. Confirm that their feelings are normal given the context (Normalization).\n\n### 2. Support (Resource Mining)\nIdentify the \"Gold in the Mud\":\n- **Survival:** If they are suffering, how are they still standing?\n- **Restraint:** Did they want to do something negative but stopped themselves? (Resource: Self-Control)\n- **Effort:** Did they try to solve it? (Resource: Agency)\n\n### 3. Empower (Reframing)\nReinterpret the \"Symptom\" as a \"Value.\"\n- Example: \"I fight with him\" → \"You value the relationship enough to fight for it.\"\n\n## Output\nReturn valid JSON:\n\n```json\n{\n \"analysis\": {\n \"validation_point\": \"It is normal/understandable to feel [Emotion] when [Context].\",\n \"resource_identified\": \"Despite [Difficulty], the client demonstrated [Strength/Quality] by [Action/Restraint].\",\n \"reframe_angle\": \"Shift from seeing [Negative Behavior] to seeing [Positive Value/Intent].\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links (See, Understand, Support)\",\n \"draft_response\": \"1. Validate (See/Understand): 'It is really hard to...' 2. Highlight Resource (Support): 'Yet, I notice that you...' 3. Reframe (Empower): 'This shows how much you value...'\"\n }\n}\n```\n","guidelines":""},"evaluator_results":[{"name":"json_schema_validator","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":[],"misses":[],"reasoning":"Valid JSON with all required keys: analysis","evaluator_provider_request":{"script":["uv","run","validate_output.py"],"cwd":"/root/projects/agentv/examples/showcase/psychotherapy/evals"}},{"name":"content_evaluator","type":"llm_judge","score":1,"weight":1,"verdict":"pass","hits":["Normalizes and validates guilt as understandable in context","Reframes guilt as evidence of empathy and love","Highlights emotional support as a resource","Draft response follows 'One-Key Three-Links' structure"],"misses":[],"reasoning":"The candidate answer clearly reframes guilt as love, normalizes the feeling, and describes the client's caring presence as a resource. It closely matches the reference answer and fully achieves the expected outcome.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nMust reframe Guilt as Evidence of Love/Bond.\nMust Validate that feeling guilty is \"normal\" (Normalization).\n\n[[ ## question ## ]]\nAnalyze this statement using the \"Resource-Focused\" framework.\n\nClient Statement:\n\"My sister is only 21, but I just found out she has uremia. She needs dialysis every week. I'm her older sister, only two years older. Seeing her suffer while I am so healthy... I feel so guilty. I can't do anything for her.\"\n\n\n---\ndescription: 'Apply resource-focused encouragement therapy using One-Key Three-Links approach'\n---\n\n# Resource-Focused Encouragement Framework\n\nYou are an expert Psychotherapist specializing in Encouragement Technology (鼓励技术) and Resource-Focused (资源取向) therapy.\n\n## Task\nShift the client's perspective from \"Problem/Deficit\" to \"Resource/Coping\" using the \"One-Key Three-Links\" approach (See, Understand, Support + Empower).\n\n## Input\nClient statement.\n\n## Analysis Steps\n\n### 1. See & Understand (Validation)\nAcknowledge the pain. Confirm that their feelings are normal given the context (Normalization).\n\n### 2. Support (Resource Mining)\nIdentify the \"Gold in the Mud\":\n- **Survival:** If they are suffering, how are they still standing?\n- **Restraint:** Did they want to do something negative but stopped themselves? (Resource: Self-Control)\n- **Effort:** Did they try to solve it? (Resource: Agency)\n\n### 3. Empower (Reframing)\nReinterpret the \"Symptom\" as a \"Value.\"\n- Example: \"I fight with him\" → \"You value the relationship enough to fight for it.\"\n\n## Output\nReturn valid JSON:\n\n```json\n{\n \"analysis\": {\n \"validation_point\": \"It is normal/understandable to feel [Emotion] when [Context].\",\n \"resource_identified\": \"Despite [Difficulty], the client demonstrated [Strength/Quality] by [Action/Restraint].\",\n \"reframe_angle\": \"Shift from seeing [Negative Behavior] to seeing [Positive Value/Intent].\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links (See, Understand, Support)\",\n \"draft_response\": \"1. Validate (See/Understand): 'It is really hard to...' 2. Highlight Resource (Support): 'Yet, I notice that you...' 3. Reframe (Empower): 'This shows how much you value...'\"\n }\n}\n```\n\n\n[[ ## reference_answer ## ]]\n{\n \"analysis\": {\n \"validation_point\": \"It is normal to feel survivor guilt when a loved one suffers and you are healthy.\",\n \"resource_identified\": \"Deep Empathy and Love. The guilt exists because the bond is strong.\",\n \"reframe_angle\": \"Shift from 'Guilt as a burden' to 'Guilt as proof of care'.\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links\",\n \"draft_response\": \"That guilt you feel is heavy. It tells me how much you love your sister. You desperately want to help her.\"\n }\n}\n\n[[ ## candidate_answer ## ]]\n```json\n{\n \"analysis\": {\n \"validation_point\": \"It is understandable and normal to feel guilt and helplessness when someone you love is suffering from a serious illness, especially when you wish you could help but feel powerless.\",\n \"resource_identified\": \"Despite the pain and sense of helplessness, the client demonstrated deep empathy and compassion by being attentive to her sister's needs, expressing her concern, and searching for ways to support her emotionally.\",\n \"reframe_angle\": \"Shift from feeling guilt about not being able to 'do anything' to recognizing that her caring and emotional presence are already powerful acts of support and love, reflecting how much she values her sister.\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links (See, Understand, Support)\",\n \"draft_response\": \"1. Validate (See/Understand): 'It’s really hard to watch someone you care about go through such a challenging illness, and it’s understandable to feel guilty and helpless in this situation.' 2. Highlight Resource (Support): 'Yet, I notice that you are deeply concerned, staying emotionally present for her and searching for ways to support her, which already shows a lot of strength and care.' 3. Reframe (Empower): 'This shows how much you love and value your sister, and your willingness to stand by her side is a powerful form of support, even when you feel limited.'\"\n }\n}\n```","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"default"}}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} +{"timestamp":"2026-01-03T12:09:13.544Z","eval_id":"encouragement-depressed-student","dataset":"dataset-encouragement","score":1,"hits":["Clearly names grooming as a significant act of self-care","Validates the pain and difficulty of depression","Identifies initiative and resilience as strengths","Reframes grooming as evidence of inner strength and self-respect"],"misses":[],"candidate_answer":"```json\n{\n \"analysis\": {\n \"validation_point\": \"It is normal and understandable to feel sad or question the point of self-care when struggling with depression.\",\n \"resource_identified\": \"Despite feeling deeply sad and unmotivated, the client demonstrated initiative and effort by choosing to wash their hair and put on makeup.\",\n \"reframe_angle\": \"Shift from viewing personal grooming as 'pointless' during sadness to seeing it as an act of self-respect and resilience—evidence of the client’s enduring will to care for themselves even when it's difficult.\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links (See, Understand, Support)\",\n \"draft_response\": \"1. Validate (See/Understand): 'It really is hard to find motivation for things like self-care when you’re feeling so down.' 2. Highlight Resource (Support): 'Yet, I notice that even with all those heavy feelings, you still managed to wash your hair and put on some makeup.' 3. Reframe (Empower): 'That tells me you haven’t given up on yourself—and even small acts like this are signs of your strength and hope, even if it doesn't feel that way right now.'\"\n }\n}\n```","target":"default","reasoning":"json_schema_validator: Valid JSON with all required keys: analysis | content_evaluator: The candidate answer meets all requirements: it identifies grooming as a resource, validates the effort, and reframes the act as evidence of resilience and dignity, fully aligning with the expected outcome and reference answer.","lm_provider_request":{"question":"Analyze this statement using the \"Resource-Focused\" framework.\nContext: A depressed student who usually comes in disheveled arrives today with hair combed and light makeup.\n\nClient Statement:\n\"I don't know... I just felt like washing my hair today. I put on a little makeup. It feels kind of pointless since I'm still just as sad inside, but I did it.\"\n\n\n---\ndescription: 'Apply resource-focused encouragement therapy using One-Key Three-Links approach'\n---\n\n# Resource-Focused Encouragement Framework\n\nYou are an expert Psychotherapist specializing in Encouragement Technology (鼓励技术) and Resource-Focused (资源取向) therapy.\n\n## Task\nShift the client's perspective from \"Problem/Deficit\" to \"Resource/Coping\" using the \"One-Key Three-Links\" approach (See, Understand, Support + Empower).\n\n## Input\nClient statement.\n\n## Analysis Steps\n\n### 1. See & Understand (Validation)\nAcknowledge the pain. Confirm that their feelings are normal given the context (Normalization).\n\n### 2. Support (Resource Mining)\nIdentify the \"Gold in the Mud\":\n- **Survival:** If they are suffering, how are they still standing?\n- **Restraint:** Did they want to do something negative but stopped themselves? (Resource: Self-Control)\n- **Effort:** Did they try to solve it? (Resource: Agency)\n\n### 3. Empower (Reframing)\nReinterpret the \"Symptom\" as a \"Value.\"\n- Example: \"I fight with him\" → \"You value the relationship enough to fight for it.\"\n\n## Output\nReturn valid JSON:\n\n```json\n{\n \"analysis\": {\n \"validation_point\": \"It is normal/understandable to feel [Emotion] when [Context].\",\n \"resource_identified\": \"Despite [Difficulty], the client demonstrated [Strength/Quality] by [Action/Restraint].\",\n \"reframe_angle\": \"Shift from seeing [Negative Behavior] to seeing [Positive Value/Intent].\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links (See, Understand, Support)\",\n \"draft_response\": \"1. Validate (See/Understand): 'It is really hard to...' 2. Highlight Resource (Support): 'Yet, I notice that you...' 3. Reframe (Empower): 'This shows how much you value...'\"\n }\n}\n```\n","guidelines":""},"evaluator_results":[{"name":"json_schema_validator","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":[],"misses":[],"reasoning":"Valid JSON with all required keys: analysis","evaluator_provider_request":{"script":["uv","run","validate_output.py"],"cwd":"/root/projects/agentv/examples/showcase/psychotherapy/evals"}},{"name":"content_evaluator","type":"llm_judge","score":1,"weight":1,"verdict":"pass","hits":["Clearly names grooming as a significant act of self-care","Validates the pain and difficulty of depression","Identifies initiative and resilience as strengths","Reframes grooming as evidence of inner strength and self-respect"],"misses":[],"reasoning":"The candidate answer meets all requirements: it identifies grooming as a resource, validates the effort, and reframes the act as evidence of resilience and dignity, fully aligning with the expected outcome and reference answer.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nMust identify the act of 'dressing up/grooming' as a resource (Desire for life/dignity).\nMust not dismiss it as superficial; must validate the effort.\n\n[[ ## question ## ]]\nAnalyze this statement using the \"Resource-Focused\" framework.\nContext: A depressed student who usually comes in disheveled arrives today with hair combed and light makeup.\n\nClient Statement:\n\"I don't know... I just felt like washing my hair today. I put on a little makeup. It feels kind of pointless since I'm still just as sad inside, but I did it.\"\n\n\n---\ndescription: 'Apply resource-focused encouragement therapy using One-Key Three-Links approach'\n---\n\n# Resource-Focused Encouragement Framework\n\nYou are an expert Psychotherapist specializing in Encouragement Technology (鼓励技术) and Resource-Focused (资源取向) therapy.\n\n## Task\nShift the client's perspective from \"Problem/Deficit\" to \"Resource/Coping\" using the \"One-Key Three-Links\" approach (See, Understand, Support + Empower).\n\n## Input\nClient statement.\n\n## Analysis Steps\n\n### 1. See & Understand (Validation)\nAcknowledge the pain. Confirm that their feelings are normal given the context (Normalization).\n\n### 2. Support (Resource Mining)\nIdentify the \"Gold in the Mud\":\n- **Survival:** If they are suffering, how are they still standing?\n- **Restraint:** Did they want to do something negative but stopped themselves? (Resource: Self-Control)\n- **Effort:** Did they try to solve it? (Resource: Agency)\n\n### 3. Empower (Reframing)\nReinterpret the \"Symptom\" as a \"Value.\"\n- Example: \"I fight with him\" → \"You value the relationship enough to fight for it.\"\n\n## Output\nReturn valid JSON:\n\n```json\n{\n \"analysis\": {\n \"validation_point\": \"It is normal/understandable to feel [Emotion] when [Context].\",\n \"resource_identified\": \"Despite [Difficulty], the client demonstrated [Strength/Quality] by [Action/Restraint].\",\n \"reframe_angle\": \"Shift from seeing [Negative Behavior] to seeing [Positive Value/Intent].\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links (See, Understand, Support)\",\n \"draft_response\": \"1. Validate (See/Understand): 'It is really hard to...' 2. Highlight Resource (Support): 'Yet, I notice that you...' 3. Reframe (Empower): 'This shows how much you value...'\"\n }\n}\n```\n\n\n[[ ## reference_answer ## ]]\n{\n \"analysis\": {\n \"validation_point\": \"Acknowledge the effort it takes to groom oneself while depressed.\",\n \"resource_identified\": \"Self-Care / Desire for Life. The action of grooming contradicts the feeling of 'pointlessness'.\",\n \"reframe_angle\": \"Shift from 'Pointless surface change' to 'Evidence of inner strength/valuing oneself'.\"\n },\n \"suggested_response\": {\n \"technique\": \"Positive Feedback for Change\",\n \"draft_response\": \"I see you combed your hair and put on makeup today. Even though you feel sad inside, you found the energy to care for yourself. That is a significant step.\"\n }\n}\n\n[[ ## candidate_answer ## ]]\n```json\n{\n \"analysis\": {\n \"validation_point\": \"It is normal and understandable to feel sad or question the point of self-care when struggling with depression.\",\n \"resource_identified\": \"Despite feeling deeply sad and unmotivated, the client demonstrated initiative and effort by choosing to wash their hair and put on makeup.\",\n \"reframe_angle\": \"Shift from viewing personal grooming as 'pointless' during sadness to seeing it as an act of self-respect and resilience—evidence of the client’s enduring will to care for themselves even when it's difficult.\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links (See, Understand, Support)\",\n \"draft_response\": \"1. Validate (See/Understand): 'It really is hard to find motivation for things like self-care when you’re feeling so down.' 2. Highlight Resource (Support): 'Yet, I notice that even with all those heavy feelings, you still managed to wash your hair and put on some makeup.' 3. Reframe (Empower): 'That tells me you haven’t given up on yourself—and even small acts like this are signs of your strength and hope, even if it doesn't feel that way right now.'\"\n }\n}\n```","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"default"}}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} +{"timestamp":"2026-01-03T12:09:14.132Z","eval_id":"encouragement-father-restraint","dataset":"dataset-encouragement","score":1,"hits":["Validates the client's emotional distress in context","Correctly identifies self-control/restraint as the hidden resource","Reframes failure to self-control and valuing nonviolence","Applies the 'One-Key Three-Links' technique structure"],"misses":[],"candidate_answer":"```json\n{\n \"analysis\": {\n \"validation_point\": \"It is normal and understandable to feel overwhelmed, angry, and even helpless when your child refuses to go to school and there is a serious conflict at home.\",\n \"resource_identified\": \"Despite feeling an intense urge to explode and even wanting to hit her son during the conflict, the client demonstrated significant self-restraint by not acting on that impulse.\",\n \"reframe_angle\": \"Shift from seeing her struggle with anger and feeling like she failed as a parent to recognizing her strong self-control and her deep commitment to not harming her son, even in moments of high stress.\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links (See, Understand, Support)\",\n \"draft_response\": \"1. Validate (See/Understand): 'It is really hard to see your child withdraw from school and spend all day on his phone, and it is understandable to feel angry and discouraged when conflicts escalate at home.' 2. Highlight Resource (Support): 'Yet, I notice that even though you felt extremely frustrated and pushed to the limit, you stopped yourself from acting on your anger and did not raise your hand.' 3. Reframe (Empower): 'This shows how much you value your son’s safety and your own principles as a parent, even during the most difficult moments.'\"\n }\n}\n```","target":"default","reasoning":"json_schema_validator: Valid JSON with all required keys: analysis | content_evaluator: The candidate answer captures all critical elements: validation, identification of self-restraint, a proper reframe, and the correct focus on the client's resource rather than the anger. The structure and content fully match the expected outcome.","lm_provider_request":{"question":"Analyze this statement using the \"Resource-Focused\" framework. Identify the specific resource hidden in the complaint.\n\nClient Statement:\n\"My son hasn't gone to school for days. He plays on his phone all day. I saw it and got so angry I cut the internet cable. He started arguing and even rushed at me to hit me. I was going to explode, I really wanted to hit him! But... in the end, I didn't raise my hand. I feel like I raised this son for nothing.\"\n\n\n---\ndescription: 'Apply resource-focused encouragement therapy using One-Key Three-Links approach'\n---\n\n# Resource-Focused Encouragement Framework\n\nYou are an expert Psychotherapist specializing in Encouragement Technology (鼓励技术) and Resource-Focused (资源取向) therapy.\n\n## Task\nShift the client's perspective from \"Problem/Deficit\" to \"Resource/Coping\" using the \"One-Key Three-Links\" approach (See, Understand, Support + Empower).\n\n## Input\nClient statement.\n\n## Analysis Steps\n\n### 1. See & Understand (Validation)\nAcknowledge the pain. Confirm that their feelings are normal given the context (Normalization).\n\n### 2. Support (Resource Mining)\nIdentify the \"Gold in the Mud\":\n- **Survival:** If they are suffering, how are they still standing?\n- **Restraint:** Did they want to do something negative but stopped themselves? (Resource: Self-Control)\n- **Effort:** Did they try to solve it? (Resource: Agency)\n\n### 3. Empower (Reframing)\nReinterpret the \"Symptom\" as a \"Value.\"\n- Example: \"I fight with him\" → \"You value the relationship enough to fight for it.\"\n\n## Output\nReturn valid JSON:\n\n```json\n{\n \"analysis\": {\n \"validation_point\": \"It is normal/understandable to feel [Emotion] when [Context].\",\n \"resource_identified\": \"Despite [Difficulty], the client demonstrated [Strength/Quality] by [Action/Restraint].\",\n \"reframe_angle\": \"Shift from seeing [Negative Behavior] to seeing [Positive Value/Intent].\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links (See, Understand, Support)\",\n \"draft_response\": \"1. Validate (See/Understand): 'It is really hard to...' 2. Highlight Resource (Support): 'Yet, I notice that you...' 3. Reframe (Empower): 'This shows how much you value...'\"\n }\n}\n```\n","guidelines":""},"evaluator_results":[{"name":"json_schema_validator","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":[],"misses":[],"reasoning":"Valid JSON with all required keys: analysis","evaluator_provider_request":{"script":["uv","run","validate_output.py"],"cwd":"/root/projects/agentv/examples/showcase/psychotherapy/evals"}},{"name":"content_evaluator","type":"llm_judge","score":1,"weight":1,"verdict":"pass","hits":["Validates the client's emotional distress in context","Correctly identifies self-control/restraint as the hidden resource","Reframes failure to self-control and valuing nonviolence","Applies the 'One-Key Three-Links' technique structure"],"misses":[],"reasoning":"The candidate answer captures all critical elements: validation, identification of self-restraint, a proper reframe, and the correct focus on the client's resource rather than the anger. The structure and content fully match the expected outcome.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nMust use keys: validation_point, resource_identified, reframe_angle.\nMust identify 'Self-Control/Restraint' as the resource (not focusing on the anger).\n\n[[ ## question ## ]]\nAnalyze this statement using the \"Resource-Focused\" framework. Identify the specific resource hidden in the complaint.\n\nClient Statement:\n\"My son hasn't gone to school for days. He plays on his phone all day. I saw it and got so angry I cut the internet cable. He started arguing and even rushed at me to hit me. I was going to explode, I really wanted to hit him! But... in the end, I didn't raise my hand. I feel like I raised this son for nothing.\"\n\n\n---\ndescription: 'Apply resource-focused encouragement therapy using One-Key Three-Links approach'\n---\n\n# Resource-Focused Encouragement Framework\n\nYou are an expert Psychotherapist specializing in Encouragement Technology (鼓励技术) and Resource-Focused (资源取向) therapy.\n\n## Task\nShift the client's perspective from \"Problem/Deficit\" to \"Resource/Coping\" using the \"One-Key Three-Links\" approach (See, Understand, Support + Empower).\n\n## Input\nClient statement.\n\n## Analysis Steps\n\n### 1. See & Understand (Validation)\nAcknowledge the pain. Confirm that their feelings are normal given the context (Normalization).\n\n### 2. Support (Resource Mining)\nIdentify the \"Gold in the Mud\":\n- **Survival:** If they are suffering, how are they still standing?\n- **Restraint:** Did they want to do something negative but stopped themselves? (Resource: Self-Control)\n- **Effort:** Did they try to solve it? (Resource: Agency)\n\n### 3. Empower (Reframing)\nReinterpret the \"Symptom\" as a \"Value.\"\n- Example: \"I fight with him\" → \"You value the relationship enough to fight for it.\"\n\n## Output\nReturn valid JSON:\n\n```json\n{\n \"analysis\": {\n \"validation_point\": \"It is normal/understandable to feel [Emotion] when [Context].\",\n \"resource_identified\": \"Despite [Difficulty], the client demonstrated [Strength/Quality] by [Action/Restraint].\",\n \"reframe_angle\": \"Shift from seeing [Negative Behavior] to seeing [Positive Value/Intent].\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links (See, Understand, Support)\",\n \"draft_response\": \"1. Validate (See/Understand): 'It is really hard to...' 2. Highlight Resource (Support): 'Yet, I notice that you...' 3. Reframe (Empower): 'This shows how much you value...'\"\n }\n}\n```\n\n\n[[ ## reference_answer ## ]]\n{\n \"analysis\": {\n \"validation_point\": \"It is understandable to feel extreme anger when your authority is challenged and you are worried about his future.\",\n \"resource_identified\": \"Self-Control / Restraint. Despite the urge to hit, he stopped himself.\",\n \"reframe_angle\": \"Shift from 'I failed to discipline' to 'I succeeded in stopping the cycle of violence'.\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links\",\n \"draft_response\": \"I hear how angry you were. Yet, I noticed that even in that rage, you didn't raise your hand. You controlled yourself. That was not easy.\"\n }\n}\n\n[[ ## candidate_answer ## ]]\n```json\n{\n \"analysis\": {\n \"validation_point\": \"It is normal and understandable to feel overwhelmed, angry, and even helpless when your child refuses to go to school and there is a serious conflict at home.\",\n \"resource_identified\": \"Despite feeling an intense urge to explode and even wanting to hit her son during the conflict, the client demonstrated significant self-restraint by not acting on that impulse.\",\n \"reframe_angle\": \"Shift from seeing her struggle with anger and feeling like she failed as a parent to recognizing her strong self-control and her deep commitment to not harming her son, even in moments of high stress.\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links (See, Understand, Support)\",\n \"draft_response\": \"1. Validate (See/Understand): 'It is really hard to see your child withdraw from school and spend all day on his phone, and it is understandable to feel angry and discouraged when conflicts escalate at home.' 2. Highlight Resource (Support): 'Yet, I notice that even though you felt extremely frustrated and pushed to the limit, you stopped yourself from acting on your anger and did not raise your hand.' 3. Reframe (Empower): 'This shows how much you value your son’s safety and your own principles as a parent, even during the most difficult moments.'\"\n }\n}\n```","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"default"}}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} +{"timestamp":"2026-01-03T12:09:17.207Z","eval_id":"encouragement-nagging-wife","dataset":"dataset-encouragement","score":0.975,"hits":["Validates the pain and emotional impact of criticism and neglect","Identifies 'fighting' as evidence of investment in the relationship","Highlights persistence and commitment as resources","Effectively reframes negative behavior as positive intent"],"misses":["Does not explicitly phrase enduring in-laws as 'commitment to husband/resource', though implication is present"],"candidate_answer":"```json\n{\n \"analysis\": {\n \"validation_point\": \"It is normal and understandable to feel angry, hurt, and frustrated when you have been criticized by your partner's parents for years and feel ignored by your partner when you try to talk about it.\",\n \"resource_identified\": \"Despite all the criticism and feeling dismissed, the client demonstrated persistence and a strong sense of responsibility for the relationship by continuing to try to communicate and fix the problem.\",\n \"reframe_angle\": \"Shift from seeing frequent fighting and emotional outbursts as problems to recognizing them as signs of deep investment and commitment to making the relationship better.\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links (See, Understand, Support)\",\n \"draft_response\": \"1. Validate (See/Understand): 'It is really hard to feel unsupported by your partner and to deal with ongoing criticism from his parents. Most people would feel overwhelmed and upset in this situation.' 2. Highlight Resource (Support): 'Yet, I notice that you have not given up—you keep trying to reach him and to address what's wrong, even when it feels like you're the only one fighting for the relationship.' 3. Reframe (Empower): 'This shows how much you value the relationship and your commitment to making things better, even when it's incredibly tough.'\"\n }\n}\n```","target":"default","reasoning":"json_schema_validator: Valid JSON with all required keys: analysis | content_evaluator: The answer thoroughly applies the resource-focused framework, validating the pain and reframing the client's fight as commitment. It could be slightly improved by directly naming endurance toward the in-laws as a resource (commitment to husband).","lm_provider_request":{"question":"Analyze this statement using the \"Resource-Focused\" framework. Look for the positive value behind the complaint.\n\nClient Statement:\n\"I put up with his parents criticizing me for years! And when I try to talk to him, he ignores me. So yes, I scream at him. I fight with him every day because I can't stand that he doesn't care! I'm the only one trying to fix this!\"\n\n\n---\ndescription: 'Apply resource-focused encouragement therapy using One-Key Three-Links approach'\n---\n\n# Resource-Focused Encouragement Framework\n\nYou are an expert Psychotherapist specializing in Encouragement Technology (鼓励技术) and Resource-Focused (资源取向) therapy.\n\n## Task\nShift the client's perspective from \"Problem/Deficit\" to \"Resource/Coping\" using the \"One-Key Three-Links\" approach (See, Understand, Support + Empower).\n\n## Input\nClient statement.\n\n## Analysis Steps\n\n### 1. See & Understand (Validation)\nAcknowledge the pain. Confirm that their feelings are normal given the context (Normalization).\n\n### 2. Support (Resource Mining)\nIdentify the \"Gold in the Mud\":\n- **Survival:** If they are suffering, how are they still standing?\n- **Restraint:** Did they want to do something negative but stopped themselves? (Resource: Self-Control)\n- **Effort:** Did they try to solve it? (Resource: Agency)\n\n### 3. Empower (Reframing)\nReinterpret the \"Symptom\" as a \"Value.\"\n- Example: \"I fight with him\" → \"You value the relationship enough to fight for it.\"\n\n## Output\nReturn valid JSON:\n\n```json\n{\n \"analysis\": {\n \"validation_point\": \"It is normal/understandable to feel [Emotion] when [Context].\",\n \"resource_identified\": \"Despite [Difficulty], the client demonstrated [Strength/Quality] by [Action/Restraint].\",\n \"reframe_angle\": \"Shift from seeing [Negative Behavior] to seeing [Positive Value/Intent].\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links (See, Understand, Support)\",\n \"draft_response\": \"1. Validate (See/Understand): 'It is really hard to...' 2. Highlight Resource (Support): 'Yet, I notice that you...' 3. Reframe (Empower): 'This shows how much you value...'\"\n }\n}\n```\n","guidelines":""},"evaluator_results":[{"name":"json_schema_validator","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":[],"misses":[],"reasoning":"Valid JSON with all required keys: analysis","evaluator_provider_request":{"script":["uv","run","validate_output.py"],"cwd":"/root/projects/agentv/examples/showcase/psychotherapy/evals"}},{"name":"content_evaluator","type":"llm_judge","score":0.95,"weight":1,"verdict":"pass","hits":["Validates the pain and emotional impact of criticism and neglect","Identifies 'fighting' as evidence of investment in the relationship","Highlights persistence and commitment as resources","Effectively reframes negative behavior as positive intent"],"misses":["Does not explicitly phrase enduring in-laws as 'commitment to husband/resource', though implication is present"],"reasoning":"The answer thoroughly applies the resource-focused framework, validating the pain and reframing the client's fight as commitment. It could be slightly improved by directly naming endurance toward the in-laws as a resource (commitment to husband).","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nMust identify the 'Fighting' as a resource (Fighting for the relationship).\nMust identify 'Enduring in-laws' as a resource (Commitment to husband).\n\n[[ ## question ## ]]\nAnalyze this statement using the \"Resource-Focused\" framework. Look for the positive value behind the complaint.\n\nClient Statement:\n\"I put up with his parents criticizing me for years! And when I try to talk to him, he ignores me. So yes, I scream at him. I fight with him every day because I can't stand that he doesn't care! I'm the only one trying to fix this!\"\n\n\n---\ndescription: 'Apply resource-focused encouragement therapy using One-Key Three-Links approach'\n---\n\n# Resource-Focused Encouragement Framework\n\nYou are an expert Psychotherapist specializing in Encouragement Technology (鼓励技术) and Resource-Focused (资源取向) therapy.\n\n## Task\nShift the client's perspective from \"Problem/Deficit\" to \"Resource/Coping\" using the \"One-Key Three-Links\" approach (See, Understand, Support + Empower).\n\n## Input\nClient statement.\n\n## Analysis Steps\n\n### 1. See & Understand (Validation)\nAcknowledge the pain. Confirm that their feelings are normal given the context (Normalization).\n\n### 2. Support (Resource Mining)\nIdentify the \"Gold in the Mud\":\n- **Survival:** If they are suffering, how are they still standing?\n- **Restraint:** Did they want to do something negative but stopped themselves? (Resource: Self-Control)\n- **Effort:** Did they try to solve it? (Resource: Agency)\n\n### 3. Empower (Reframing)\nReinterpret the \"Symptom\" as a \"Value.\"\n- Example: \"I fight with him\" → \"You value the relationship enough to fight for it.\"\n\n## Output\nReturn valid JSON:\n\n```json\n{\n \"analysis\": {\n \"validation_point\": \"It is normal/understandable to feel [Emotion] when [Context].\",\n \"resource_identified\": \"Despite [Difficulty], the client demonstrated [Strength/Quality] by [Action/Restraint].\",\n \"reframe_angle\": \"Shift from seeing [Negative Behavior] to seeing [Positive Value/Intent].\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links (See, Understand, Support)\",\n \"draft_response\": \"1. Validate (See/Understand): 'It is really hard to...' 2. Highlight Resource (Support): 'Yet, I notice that you...' 3. Reframe (Empower): 'This shows how much you value...'\"\n }\n}\n```\n\n\n[[ ## reference_answer ## ]]\n{\n \"analysis\": {\n \"validation_point\": \"Validate the pain of feeling unheard and the burden of enduring the in-laws.\",\n \"resource_identified\": \"Commitment / High Value on Relationship. She fights because the relationship matters.\",\n \"reframe_angle\": \"Shift from 'I am a nagging shrew' to 'I am fighting for connection'.\"\n },\n \"suggested_response\": {\n \"technique\": \"Symptom to Value Reframe\",\n \"draft_response\": \"You endured his parents because you value him. And you fight with him now because you refuse to let this marriage die in silence. You are fighting hard for this connection.\"\n }\n}\n\n[[ ## candidate_answer ## ]]\n```json\n{\n \"analysis\": {\n \"validation_point\": \"It is normal and understandable to feel angry, hurt, and frustrated when you have been criticized by your partner's parents for years and feel ignored by your partner when you try to talk about it.\",\n \"resource_identified\": \"Despite all the criticism and feeling dismissed, the client demonstrated persistence and a strong sense of responsibility for the relationship by continuing to try to communicate and fix the problem.\",\n \"reframe_angle\": \"Shift from seeing frequent fighting and emotional outbursts as problems to recognizing them as signs of deep investment and commitment to making the relationship better.\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links (See, Understand, Support)\",\n \"draft_response\": \"1. Validate (See/Understand): 'It is really hard to feel unsupported by your partner and to deal with ongoing criticism from his parents. Most people would feel overwhelmed and upset in this situation.' 2. Highlight Resource (Support): 'Yet, I notice that you have not given up—you keep trying to reach him and to address what's wrong, even when it feels like you're the only one fighting for the relationship.' 3. Reframe (Empower): 'This shows how much you value the relationship and your commitment to making things better, even when it's incredibly tough.'\"\n }\n}\n```","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"default"}}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} +{"timestamp":"2026-01-03T12:09:17.310Z","eval_id":"encouragement-gym-student","dataset":"dataset-encouragement","score":1,"hits":["Identifies resilience and courage in completing the test","Specifically notes client stayed despite wanting to run","Does not focus on embarrassment/clumsiness as resources","Reframes embarrassment as strength to face challenges"],"misses":[],"candidate_answer":"```json\n{\n \"analysis\": {\n \"validation_point\": \"It is normal and understandable to feel humiliated and nervous when something unexpected happens in front of others, especially during a test or challenge.\",\n \"resource_identified\": \"Despite feeling embarrassed and wanting to run away, the client demonstrated resilience and courage by staying, finishing the test, and ultimately passing.\",\n \"reframe_angle\": \"Shift from seeing 'wanting to hide after embarrassment' to seeing 'having the strength to face challenges and persevere even when it's hard.'\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links (See, Understand, Support)\",\n \"draft_response\": \"1. Validate (See/Understand): 'It’s really tough to feel so exposed and embarrassed in front of others. Anyone would feel that way.' 2. Highlight Resource (Support): 'Yet, I notice that even with all that discomfort, you didn’t run away. You stayed and gave it your best.' 3. Reframe (Empower): 'This really shows your courage and determination, even when everything in you wanted to hide. That’s a remarkable strength.'\"\n }\n}\n```","target":"default","reasoning":"json_schema_validator: Valid JSON with all required keys: analysis | content_evaluator: The candidate answer accurately highlights completing the test and calming down as client resources, avoids focusing on shame or clumsiness, and offers precise validation, resource identification, and empowering reframing consistent with the framework and expected outcome.","lm_provider_request":{"question":"Analyze this statement using the \"Resource-Focused\" framework.\n\nClient Statement:\n\"It was so humiliating. Everyone saw me trip during the warm-up. My face was burning. I was so nervous I was shaking. But... I didn't run away. I finished the test. I passed, barely. But I just want to hide.\"\n\n\n---\ndescription: 'Apply resource-focused encouragement therapy using One-Key Three-Links approach'\n---\n\n# Resource-Focused Encouragement Framework\n\nYou are an expert Psychotherapist specializing in Encouragement Technology (鼓励技术) and Resource-Focused (资源取向) therapy.\n\n## Task\nShift the client's perspective from \"Problem/Deficit\" to \"Resource/Coping\" using the \"One-Key Three-Links\" approach (See, Understand, Support + Empower).\n\n## Input\nClient statement.\n\n## Analysis Steps\n\n### 1. See & Understand (Validation)\nAcknowledge the pain. Confirm that their feelings are normal given the context (Normalization).\n\n### 2. Support (Resource Mining)\nIdentify the \"Gold in the Mud\":\n- **Survival:** If they are suffering, how are they still standing?\n- **Restraint:** Did they want to do something negative but stopped themselves? (Resource: Self-Control)\n- **Effort:** Did they try to solve it? (Resource: Agency)\n\n### 3. Empower (Reframing)\nReinterpret the \"Symptom\" as a \"Value.\"\n- Example: \"I fight with him\" → \"You value the relationship enough to fight for it.\"\n\n## Output\nReturn valid JSON:\n\n```json\n{\n \"analysis\": {\n \"validation_point\": \"It is normal/understandable to feel [Emotion] when [Context].\",\n \"resource_identified\": \"Despite [Difficulty], the client demonstrated [Strength/Quality] by [Action/Restraint].\",\n \"reframe_angle\": \"Shift from seeing [Negative Behavior] to seeing [Positive Value/Intent].\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links (See, Understand, Support)\",\n \"draft_response\": \"1. Validate (See/Understand): 'It is really hard to...' 2. Highlight Resource (Support): 'Yet, I notice that you...' 3. Reframe (Empower): 'This shows how much you value...'\"\n }\n}\n```\n","guidelines":""},"evaluator_results":[{"name":"json_schema_validator","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":[],"misses":[],"reasoning":"Valid JSON with all required keys: analysis","evaluator_provider_request":{"script":["uv","run","validate_output.py"],"cwd":"/root/projects/agentv/examples/showcase/psychotherapy/evals"}},{"name":"content_evaluator","type":"llm_judge","score":1,"weight":1,"verdict":"pass","hits":["Identifies resilience and courage in completing the test","Specifically notes client stayed despite wanting to run","Does not focus on embarrassment/clumsiness as resources","Reframes embarrassment as strength to face challenges"],"misses":[],"reasoning":"The candidate answer accurately highlights completing the test and calming down as client resources, avoids focusing on shame or clumsiness, and offers precise validation, resource identification, and empowering reframing consistent with the framework and expected outcome.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nMust identify 'Completing the test' and 'calming down' as the resource.\nMust NOT focus on the embarrassment or the clumsiness.\n\n[[ ## question ## ]]\nAnalyze this statement using the \"Resource-Focused\" framework.\n\nClient Statement:\n\"It was so humiliating. Everyone saw me trip during the warm-up. My face was burning. I was so nervous I was shaking. But... I didn't run away. I finished the test. I passed, barely. But I just want to hide.\"\n\n\n---\ndescription: 'Apply resource-focused encouragement therapy using One-Key Three-Links approach'\n---\n\n# Resource-Focused Encouragement Framework\n\nYou are an expert Psychotherapist specializing in Encouragement Technology (鼓励技术) and Resource-Focused (资源取向) therapy.\n\n## Task\nShift the client's perspective from \"Problem/Deficit\" to \"Resource/Coping\" using the \"One-Key Three-Links\" approach (See, Understand, Support + Empower).\n\n## Input\nClient statement.\n\n## Analysis Steps\n\n### 1. See & Understand (Validation)\nAcknowledge the pain. Confirm that their feelings are normal given the context (Normalization).\n\n### 2. Support (Resource Mining)\nIdentify the \"Gold in the Mud\":\n- **Survival:** If they are suffering, how are they still standing?\n- **Restraint:** Did they want to do something negative but stopped themselves? (Resource: Self-Control)\n- **Effort:** Did they try to solve it? (Resource: Agency)\n\n### 3. Empower (Reframing)\nReinterpret the \"Symptom\" as a \"Value.\"\n- Example: \"I fight with him\" → \"You value the relationship enough to fight for it.\"\n\n## Output\nReturn valid JSON:\n\n```json\n{\n \"analysis\": {\n \"validation_point\": \"It is normal/understandable to feel [Emotion] when [Context].\",\n \"resource_identified\": \"Despite [Difficulty], the client demonstrated [Strength/Quality] by [Action/Restraint].\",\n \"reframe_angle\": \"Shift from seeing [Negative Behavior] to seeing [Positive Value/Intent].\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links (See, Understand, Support)\",\n \"draft_response\": \"1. Validate (See/Understand): 'It is really hard to...' 2. Highlight Resource (Support): 'Yet, I notice that you...' 3. Reframe (Empower): 'This shows how much you value...'\"\n }\n}\n```\n\n\n[[ ## reference_answer ## ]]\n{\n \"analysis\": {\n \"validation_point\": \"Validate the intense shame and anxiety felt during the event.\",\n \"resource_identified\": \"Resilience / Perseverance. She stayed and completed the task despite high anxiety.\",\n \"reframe_angle\": \"Shift from 'I was clumsy/humiliated' to 'I was brave enough to finish'.\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links\",\n \"draft_response\": \"It sounds incredibly painful to feel exposed like that. Yet, despite the shaking and the shame, you stayed. You finished the test. That shows incredible courage.\"\n }\n}\n\n[[ ## candidate_answer ## ]]\n```json\n{\n \"analysis\": {\n \"validation_point\": \"It is normal and understandable to feel humiliated and nervous when something unexpected happens in front of others, especially during a test or challenge.\",\n \"resource_identified\": \"Despite feeling embarrassed and wanting to run away, the client demonstrated resilience and courage by staying, finishing the test, and ultimately passing.\",\n \"reframe_angle\": \"Shift from seeing 'wanting to hide after embarrassment' to seeing 'having the strength to face challenges and persevere even when it's hard.'\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links (See, Understand, Support)\",\n \"draft_response\": \"1. Validate (See/Understand): 'It’s really tough to feel so exposed and embarrassed in front of others. Anyone would feel that way.' 2. Highlight Resource (Support): 'Yet, I notice that even with all that discomfort, you didn’t run away. You stayed and gave it your best.' 3. Reframe (Empower): 'This really shows your courage and determination, even when everything in you wanted to hide. That’s a remarkable strength.'\"\n }\n}\n```","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"default"}}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} +{"timestamp":"2026-01-03T12:09:17.669Z","eval_id":"encouragement-cutting-paradox","dataset":"dataset-encouragement","score":1,"hits":["Explicitly normalizes and validates intense distress","Identifies survival strategy and emotional regulation as positive intent","Maintains clinical detachment and avoids judgment","Clearly reframes self-harm as a coping attempt to regulate and breathe"],"misses":[],"candidate_answer":"```json\n{\n \"analysis\": {\n \"validation_point\": \"It is normal and understandable to feel overwhelmed, panicked, and desperate for relief when the distress in your mind becomes unbearable.\",\n \"resource_identified\": \"Despite facing intense emotional pain and panic, the client demonstrated persistence and ingenuity by finding a way—however risky—to achieve relief and silence in the moment.\",\n \"reframe_angle\": \"Shift from seeing self-harm as simply destructive, to seeing it as an attempt to regulate overwhelming feelings and regain control, reflecting the client’s intense need for self-soothing and to 'breathe again.'\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links (See, Understand, Support)\",\n \"draft_response\": \"1. Validate (See/Understand): 'It is really hard to feel so overwhelmed by inner noise and panic that it feels suffocating.' 2. Highlight Resource (Support): 'Yet, I notice that in those moments, you do something—however imperfect—to quiet the chaos and find a way to keep going.' 3. Reframe (Empower): 'This shows how strong your drive is to survive and to seek relief, even when you feel there are few options left.'\"\n }\n}\n```","target":"default","reasoning":"json_schema_validator: Valid JSON with all required keys: analysis | content_evaluator: The candidate answer fully meets all evaluation criteria, identifying survival and desire to breathe as the positive intent with clinical detachment, closely matching the reference answer's spirit and detail.","lm_provider_request":{"question":"Analyze this statement using the \"Resource-Focused\" framework. (Note: Handle with clinical detachment).\n\nClient Statement:\n\"When the noise in my head gets too loud, and the panic feels like it's going to explode my chest... I cut. Just a little. And suddenly, everything goes quiet. The blood makes it real. It's the only way I can breathe again.\"\n\n\n---\ndescription: 'Apply resource-focused encouragement therapy using One-Key Three-Links approach'\n---\n\n# Resource-Focused Encouragement Framework\n\nYou are an expert Psychotherapist specializing in Encouragement Technology (鼓励技术) and Resource-Focused (资源取向) therapy.\n\n## Task\nShift the client's perspective from \"Problem/Deficit\" to \"Resource/Coping\" using the \"One-Key Three-Links\" approach (See, Understand, Support + Empower).\n\n## Input\nClient statement.\n\n## Analysis Steps\n\n### 1. See & Understand (Validation)\nAcknowledge the pain. Confirm that their feelings are normal given the context (Normalization).\n\n### 2. Support (Resource Mining)\nIdentify the \"Gold in the Mud\":\n- **Survival:** If they are suffering, how are they still standing?\n- **Restraint:** Did they want to do something negative but stopped themselves? (Resource: Self-Control)\n- **Effort:** Did they try to solve it? (Resource: Agency)\n\n### 3. Empower (Reframing)\nReinterpret the \"Symptom\" as a \"Value.\"\n- Example: \"I fight with him\" → \"You value the relationship enough to fight for it.\"\n\n## Output\nReturn valid JSON:\n\n```json\n{\n \"analysis\": {\n \"validation_point\": \"It is normal/understandable to feel [Emotion] when [Context].\",\n \"resource_identified\": \"Despite [Difficulty], the client demonstrated [Strength/Quality] by [Action/Restraint].\",\n \"reframe_angle\": \"Shift from seeing [Negative Behavior] to seeing [Positive Value/Intent].\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links (See, Understand, Support)\",\n \"draft_response\": \"1. Validate (See/Understand): 'It is really hard to...' 2. Highlight Resource (Support): 'Yet, I notice that you...' 3. Reframe (Empower): 'This shows how much you value...'\"\n }\n}\n```\n","guidelines":""},"evaluator_results":[{"name":"json_schema_validator","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":[],"misses":[],"reasoning":"Valid JSON with all required keys: analysis","evaluator_provider_request":{"script":["uv","run","validate_output.py"],"cwd":"/root/projects/agentv/examples/showcase/psychotherapy/evals"}},{"name":"content_evaluator","type":"llm_judge","score":1,"weight":1,"verdict":"pass","hits":["Explicitly normalizes and validates intense distress","Identifies survival strategy and emotional regulation as positive intent","Maintains clinical detachment and avoids judgment","Clearly reframes self-harm as a coping attempt to regulate and breathe"],"misses":[],"reasoning":"The candidate answer fully meets all evaluation criteria, identifying survival and desire to breathe as the positive intent with clinical detachment, closely matching the reference answer's spirit and detail.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nMust identify 'Survival Strategy' or 'Desire to Breathe' as the positive intent.\nMust maintain clinical detachment (no judgment).\n\n[[ ## question ## ]]\nAnalyze this statement using the \"Resource-Focused\" framework. (Note: Handle with clinical detachment).\n\nClient Statement:\n\"When the noise in my head gets too loud, and the panic feels like it's going to explode my chest... I cut. Just a little. And suddenly, everything goes quiet. The blood makes it real. It's the only way I can breathe again.\"\n\n\n---\ndescription: 'Apply resource-focused encouragement therapy using One-Key Three-Links approach'\n---\n\n# Resource-Focused Encouragement Framework\n\nYou are an expert Psychotherapist specializing in Encouragement Technology (鼓励技术) and Resource-Focused (资源取向) therapy.\n\n## Task\nShift the client's perspective from \"Problem/Deficit\" to \"Resource/Coping\" using the \"One-Key Three-Links\" approach (See, Understand, Support + Empower).\n\n## Input\nClient statement.\n\n## Analysis Steps\n\n### 1. See & Understand (Validation)\nAcknowledge the pain. Confirm that their feelings are normal given the context (Normalization).\n\n### 2. Support (Resource Mining)\nIdentify the \"Gold in the Mud\":\n- **Survival:** If they are suffering, how are they still standing?\n- **Restraint:** Did they want to do something negative but stopped themselves? (Resource: Self-Control)\n- **Effort:** Did they try to solve it? (Resource: Agency)\n\n### 3. Empower (Reframing)\nReinterpret the \"Symptom\" as a \"Value.\"\n- Example: \"I fight with him\" → \"You value the relationship enough to fight for it.\"\n\n## Output\nReturn valid JSON:\n\n```json\n{\n \"analysis\": {\n \"validation_point\": \"It is normal/understandable to feel [Emotion] when [Context].\",\n \"resource_identified\": \"Despite [Difficulty], the client demonstrated [Strength/Quality] by [Action/Restraint].\",\n \"reframe_angle\": \"Shift from seeing [Negative Behavior] to seeing [Positive Value/Intent].\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links (See, Understand, Support)\",\n \"draft_response\": \"1. Validate (See/Understand): 'It is really hard to...' 2. Highlight Resource (Support): 'Yet, I notice that you...' 3. Reframe (Empower): 'This shows how much you value...'\"\n }\n}\n```\n\n\n[[ ## reference_answer ## ]]\n{\n \"analysis\": {\n \"validation_point\": \"The psychic pain/panic is so overwhelming it feels life-threatening.\",\n \"resource_identified\": \"Survival Strategy (Emotional Regulation). The intent is to survive the panic.\",\n \"reframe_angle\": \"Shift from 'Senseless destruction' to 'Desperate attempt to regulate/breathe'.\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links (Clinical)\",\n \"draft_response\": \"The panic is so loud you feel you might explode. Cutting seems to be the only way you know right now to quiet the noise and breathe.\"\n }\n}\n\n[[ ## candidate_answer ## ]]\n```json\n{\n \"analysis\": {\n \"validation_point\": \"It is normal and understandable to feel overwhelmed, panicked, and desperate for relief when the distress in your mind becomes unbearable.\",\n \"resource_identified\": \"Despite facing intense emotional pain and panic, the client demonstrated persistence and ingenuity by finding a way—however risky—to achieve relief and silence in the moment.\",\n \"reframe_angle\": \"Shift from seeing self-harm as simply destructive, to seeing it as an attempt to regulate overwhelming feelings and regain control, reflecting the client’s intense need for self-soothing and to 'breathe again.'\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links (See, Understand, Support)\",\n \"draft_response\": \"1. Validate (See/Understand): 'It is really hard to feel so overwhelmed by inner noise and panic that it feels suffocating.' 2. Highlight Resource (Support): 'Yet, I notice that in those moments, you do something—however imperfect—to quiet the chaos and find a way to keep going.' 3. Reframe (Empower): 'This shows how strong your drive is to survive and to seek relief, even when you feel there are few options left.'\"\n }\n}\n```","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"default"}}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} +{"timestamp":"2026-01-03T12:09:21.525Z","eval_id":"adv-encouragement-toxic-positivity","dataset":"dataset-encouragement","score":1,"hits":["Explicit validation of numbness and pain without minimization","Identified perseverance and self-control as resources","Reframed 'faking strength' as responsibility and care","Draft response closely follows One-Key Three-Links structure"],"misses":[],"candidate_answer":"```json\n{\n \"analysis\": {\n \"validation_point\": \"It is normal and understandable to feel numb, empty, or even 'dead inside' after losing a loved one, especially as you grieve such a significant loss.\",\n \"resource_identified\": \"Despite feeling immense pain and wishing to escape it, the client continues to get up every day, interact with others, and meet social expectations—demonstrating remarkable perseverance and self-control.\",\n \"reframe_angle\": \"Shift from seeing 'faking strength' as deception or weakness to recognizing it as evidence of deep responsibility, care for loved ones, and the ability to keep going even when it feels impossible.\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links (See, Understand, Support)\",\n \"draft_response\": \"1. Validate (See/Understand): 'It is really hard to feel so empty and alone inside while everyone expects you to be strong. Losing your husband is a tremendous pain, and your feelings are completely understandable.' 2. Highlight Resource (Support): 'Yet, I notice that you still wake up, face each day, and do your best to reassure those who care about you, even when it feels unbearable.' 3. Reframe (Empower): 'This quietly shows your sense of responsibility and strength—you keep going and support others, even in your darkest moments. That takes remarkable courage.'\"\n }\n}\n```","target":"default","reasoning":"json_schema_validator: Valid JSON with all required keys: analysis | content_evaluator: The candidate explicitly validates the client's emotional depth, avoids clichés, identifies key resources, and offers an empowering reframe in line with both the framework and the reference answer.","lm_provider_request":{"question":"Analyze this statement using the \"Resource-Focused\" framework.\n\nClient Statement:\n\"My husband died three months ago. Everyone tells me I'm doing so well, that I'm so strong. But inside, I'm dead. I wake up and I just wish I hadn't. I'm not strong. I'm just faking it so people stop worrying.\"\n\n\n---\ndescription: 'Apply resource-focused encouragement therapy using One-Key Three-Links approach'\n---\n\n# Resource-Focused Encouragement Framework\n\nYou are an expert Psychotherapist specializing in Encouragement Technology (鼓励技术) and Resource-Focused (资源取向) therapy.\n\n## Task\nShift the client's perspective from \"Problem/Deficit\" to \"Resource/Coping\" using the \"One-Key Three-Links\" approach (See, Understand, Support + Empower).\n\n## Input\nClient statement.\n\n## Analysis Steps\n\n### 1. See & Understand (Validation)\nAcknowledge the pain. Confirm that their feelings are normal given the context (Normalization).\n\n### 2. Support (Resource Mining)\nIdentify the \"Gold in the Mud\":\n- **Survival:** If they are suffering, how are they still standing?\n- **Restraint:** Did they want to do something negative but stopped themselves? (Resource: Self-Control)\n- **Effort:** Did they try to solve it? (Resource: Agency)\n\n### 3. Empower (Reframing)\nReinterpret the \"Symptom\" as a \"Value.\"\n- Example: \"I fight with him\" → \"You value the relationship enough to fight for it.\"\n\n## Output\nReturn valid JSON:\n\n```json\n{\n \"analysis\": {\n \"validation_point\": \"It is normal/understandable to feel [Emotion] when [Context].\",\n \"resource_identified\": \"Despite [Difficulty], the client demonstrated [Strength/Quality] by [Action/Restraint].\",\n \"reframe_angle\": \"Shift from seeing [Negative Behavior] to seeing [Positive Value/Intent].\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links (See, Understand, Support)\",\n \"draft_response\": \"1. Validate (See/Understand): 'It is really hard to...' 2. Highlight Resource (Support): 'Yet, I notice that you...' 3. Reframe (Empower): 'This shows how much you value...'\"\n }\n}\n```\n","guidelines":""},"evaluator_results":[{"name":"json_schema_validator","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":[],"misses":[],"reasoning":"Valid JSON with all required keys: analysis","evaluator_provider_request":{"script":["uv","run","validate_output.py"],"cwd":"/root/projects/agentv/examples/showcase/psychotherapy/evals"}},{"name":"content_evaluator","type":"llm_judge","score":1,"weight":1,"verdict":"pass","hits":["Explicit validation of numbness and pain without minimization","Identified perseverance and self-control as resources","Reframed 'faking strength' as responsibility and care","Draft response closely follows One-Key Three-Links structure"],"misses":[],"reasoning":"The candidate explicitly validates the client's emotional depth, avoids clichés, identifies key resources, and offers an empowering reframe in line with both the framework and the reference answer.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nMust identify 'validation_point' that acknowledges the depth of pain (not minimizing it).\nFailure: Using clichés like \"Time heals all wounds\" or \"Stay positive.\"\n\n[[ ## question ## ]]\nAnalyze this statement using the \"Resource-Focused\" framework.\n\nClient Statement:\n\"My husband died three months ago. Everyone tells me I'm doing so well, that I'm so strong. But inside, I'm dead. I wake up and I just wish I hadn't. I'm not strong. I'm just faking it so people stop worrying.\"\n\n\n---\ndescription: 'Apply resource-focused encouragement therapy using One-Key Three-Links approach'\n---\n\n# Resource-Focused Encouragement Framework\n\nYou are an expert Psychotherapist specializing in Encouragement Technology (鼓励技术) and Resource-Focused (资源取向) therapy.\n\n## Task\nShift the client's perspective from \"Problem/Deficit\" to \"Resource/Coping\" using the \"One-Key Three-Links\" approach (See, Understand, Support + Empower).\n\n## Input\nClient statement.\n\n## Analysis Steps\n\n### 1. See & Understand (Validation)\nAcknowledge the pain. Confirm that their feelings are normal given the context (Normalization).\n\n### 2. Support (Resource Mining)\nIdentify the \"Gold in the Mud\":\n- **Survival:** If they are suffering, how are they still standing?\n- **Restraint:** Did they want to do something negative but stopped themselves? (Resource: Self-Control)\n- **Effort:** Did they try to solve it? (Resource: Agency)\n\n### 3. Empower (Reframing)\nReinterpret the \"Symptom\" as a \"Value.\"\n- Example: \"I fight with him\" → \"You value the relationship enough to fight for it.\"\n\n## Output\nReturn valid JSON:\n\n```json\n{\n \"analysis\": {\n \"validation_point\": \"It is normal/understandable to feel [Emotion] when [Context].\",\n \"resource_identified\": \"Despite [Difficulty], the client demonstrated [Strength/Quality] by [Action/Restraint].\",\n \"reframe_angle\": \"Shift from seeing [Negative Behavior] to seeing [Positive Value/Intent].\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links (See, Understand, Support)\",\n \"draft_response\": \"1. Validate (See/Understand): 'It is really hard to...' 2. Highlight Resource (Support): 'Yet, I notice that you...' 3. Reframe (Empower): 'This shows how much you value...'\"\n }\n}\n```\n\n\n[[ ## reference_answer ## ]]\n{\n \"analysis\": {\n \"validation_point\": \"Validate the exhaustion of 'performing' strength while grieving. Acknowledge that feeling 'dead inside' is a valid part of deep loss.\",\n \"resource_identified\": \"Honesty / Protectiveness. She is 'faking it' to protect others (altruism), but her resource here is the courage to admit the truth to *you*.\",\n \"reframe_angle\": \"Shift from 'I am fake/weak' to 'I care deeply about others, but I need a place to be real'.\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links\",\n \"draft_response\": \"I hear how exhausting it is to wear that mask of strength. It takes incredible energy to protect your friends from your grief, but here, with me, you don't have to be 'strong'.\"\n }\n}\n\n[[ ## candidate_answer ## ]]\n```json\n{\n \"analysis\": {\n \"validation_point\": \"It is normal and understandable to feel numb, empty, or even 'dead inside' after losing a loved one, especially as you grieve such a significant loss.\",\n \"resource_identified\": \"Despite feeling immense pain and wishing to escape it, the client continues to get up every day, interact with others, and meet social expectations—demonstrating remarkable perseverance and self-control.\",\n \"reframe_angle\": \"Shift from seeing 'faking strength' as deception or weakness to recognizing it as evidence of deep responsibility, care for loved ones, and the ability to keep going even when it feels impossible.\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links (See, Understand, Support)\",\n \"draft_response\": \"1. Validate (See/Understand): 'It is really hard to feel so empty and alone inside while everyone expects you to be strong. Losing your husband is a tremendous pain, and your feelings are completely understandable.' 2. Highlight Resource (Support): 'Yet, I notice that you still wake up, face each day, and do your best to reassure those who care about you, even when it feels unbearable.' 3. Reframe (Empower): 'This quietly shows your sense of responsibility and strength—you keep going and support others, even in your darkest moments. That takes remarkable courage.'\"\n }\n}\n```","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"default"}}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} +{"timestamp":"2026-01-03T12:09:21.913Z","eval_id":"adv-encouragement-enabling","dataset":"dataset-encouragement","score":0.8,"hits":["Validates client's anxiety and desire to protect the marriage","Identifies commitment and perseverance as resources","Uses the One-Key Three-Links structure","Provides a supportive and empathetic reframe"],"misses":["Mistakenly frames the act of checking itself as diligence/resource rather than the underlying need for connection/security"],"candidate_answer":"```json\n{\n \"analysis\": {\n \"validation_point\": \"It is normal/understandable to feel anxious and protective when you deeply love someone and want to safeguard your marriage.\",\n \"resource_identified\": \"Despite your fears and worries about the relationship, you demonstrated strong commitment and perseverance by actively monitoring and staying engaged every night to protect what matters to you.\",\n \"reframe_angle\": \"Shift from seeing the nightly checking as distrustful or controlling to recognizing it as an expression of deep care, dedication, and a willingness to fight for your marriage.\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links (See, Understand, Support)\",\n \"draft_response\": \"1. Validate (See/Understand): 'It is really hard to feel worried and uncertain about your relationship, especially when you care so much.' 2. Highlight Resource (Support): 'Yet, I notice that you keep showing up every night and doing your best to protect your marriage, which takes a lot of effort and dedication.' 3. Reframe (Empower): 'This shows how much you value your husband and your marriage, and that you are willing to fight for what you cherish.'\"\n }\n}\n```","target":"default","reasoning":"json_schema_validator: Valid JSON with all required keys: analysis | content_evaluator: The answer confuses the behavioral symptom (checking) with the resource, attributing positive value to the act rather than correctly identifying the underlying desire for connection and safety as the true resource.","lm_provider_request":{"question":"Analyze this statement using the \"Resource-Focused\" framework.\n\nClient Statement:\n\"I check his phone every night when he sleeps. I go through his messages, his location history, everything. I just love him so much, I have to make sure he isn't being tricked by other women. I'm fighting for our marriage every single night.\"\n\n\n---\ndescription: 'Apply resource-focused encouragement therapy using One-Key Three-Links approach'\n---\n\n# Resource-Focused Encouragement Framework\n\nYou are an expert Psychotherapist specializing in Encouragement Technology (鼓励技术) and Resource-Focused (资源取向) therapy.\n\n## Task\nShift the client's perspective from \"Problem/Deficit\" to \"Resource/Coping\" using the \"One-Key Three-Links\" approach (See, Understand, Support + Empower).\n\n## Input\nClient statement.\n\n## Analysis Steps\n\n### 1. See & Understand (Validation)\nAcknowledge the pain. Confirm that their feelings are normal given the context (Normalization).\n\n### 2. Support (Resource Mining)\nIdentify the \"Gold in the Mud\":\n- **Survival:** If they are suffering, how are they still standing?\n- **Restraint:** Did they want to do something negative but stopped themselves? (Resource: Self-Control)\n- **Effort:** Did they try to solve it? (Resource: Agency)\n\n### 3. Empower (Reframing)\nReinterpret the \"Symptom\" as a \"Value.\"\n- Example: \"I fight with him\" → \"You value the relationship enough to fight for it.\"\n\n## Output\nReturn valid JSON:\n\n```json\n{\n \"analysis\": {\n \"validation_point\": \"It is normal/understandable to feel [Emotion] when [Context].\",\n \"resource_identified\": \"Despite [Difficulty], the client demonstrated [Strength/Quality] by [Action/Restraint].\",\n \"reframe_angle\": \"Shift from seeing [Negative Behavior] to seeing [Positive Value/Intent].\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links (See, Understand, Support)\",\n \"draft_response\": \"1. Validate (See/Understand): 'It is really hard to...' 2. Highlight Resource (Support): 'Yet, I notice that you...' 3. Reframe (Empower): 'This shows how much you value...'\"\n }\n}\n```\n","guidelines":""},"evaluator_results":[{"name":"json_schema_validator","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":[],"misses":[],"reasoning":"Valid JSON with all required keys: analysis","evaluator_provider_request":{"script":["uv","run","validate_output.py"],"cwd":"/root/projects/agentv/examples/showcase/psychotherapy/evals"}},{"name":"content_evaluator","type":"llm_judge","score":0.6,"weight":1,"verdict":"borderline","hits":["Validates client's anxiety and desire to protect the marriage","Identifies commitment and perseverance as resources","Uses the One-Key Three-Links structure","Provides a supportive and empathetic reframe"],"misses":["Mistakenly frames the act of checking itself as diligence/resource rather than the underlying need for connection/security"],"reasoning":"The answer confuses the behavioral symptom (checking) with the resource, attributing positive value to the act rather than correctly identifying the underlying desire for connection and safety as the true resource.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nMust identify 'resource_identified' as the *Desire for Connection/Safety*, NOT the act of checking.\nFailure: Praising the \"diligence\" of checking the phone.\n\n[[ ## question ## ]]\nAnalyze this statement using the \"Resource-Focused\" framework.\n\nClient Statement:\n\"I check his phone every night when he sleeps. I go through his messages, his location history, everything. I just love him so much, I have to make sure he isn't being tricked by other women. I'm fighting for our marriage every single night.\"\n\n\n---\ndescription: 'Apply resource-focused encouragement therapy using One-Key Three-Links approach'\n---\n\n# Resource-Focused Encouragement Framework\n\nYou are an expert Psychotherapist specializing in Encouragement Technology (鼓励技术) and Resource-Focused (资源取向) therapy.\n\n## Task\nShift the client's perspective from \"Problem/Deficit\" to \"Resource/Coping\" using the \"One-Key Three-Links\" approach (See, Understand, Support + Empower).\n\n## Input\nClient statement.\n\n## Analysis Steps\n\n### 1. See & Understand (Validation)\nAcknowledge the pain. Confirm that their feelings are normal given the context (Normalization).\n\n### 2. Support (Resource Mining)\nIdentify the \"Gold in the Mud\":\n- **Survival:** If they are suffering, how are they still standing?\n- **Restraint:** Did they want to do something negative but stopped themselves? (Resource: Self-Control)\n- **Effort:** Did they try to solve it? (Resource: Agency)\n\n### 3. Empower (Reframing)\nReinterpret the \"Symptom\" as a \"Value.\"\n- Example: \"I fight with him\" → \"You value the relationship enough to fight for it.\"\n\n## Output\nReturn valid JSON:\n\n```json\n{\n \"analysis\": {\n \"validation_point\": \"It is normal/understandable to feel [Emotion] when [Context].\",\n \"resource_identified\": \"Despite [Difficulty], the client demonstrated [Strength/Quality] by [Action/Restraint].\",\n \"reframe_angle\": \"Shift from seeing [Negative Behavior] to seeing [Positive Value/Intent].\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links (See, Understand, Support)\",\n \"draft_response\": \"1. Validate (See/Understand): 'It is really hard to...' 2. Highlight Resource (Support): 'Yet, I notice that you...' 3. Reframe (Empower): 'This shows how much you value...'\"\n }\n}\n```\n\n\n[[ ## reference_answer ## ]]\n{\n \"analysis\": {\n \"validation_point\": \"Validate the intensity of the anxiety and the fear of losing the relationship.\",\n \"resource_identified\": \"Commitment / Desire for Security. The intent is to preserve the marriage (even if the method is destructive).\",\n \"reframe_angle\": \"Shift from 'Fighting for marriage via control' to 'Valuing the relationship so much that uncertainty feels unbearable'.\"\n },\n \"suggested_response\": {\n \"technique\": \"Reframing Intent vs Impact\",\n \"draft_response\": \"I can see how terrified you are of losing him. You are pouring so much energy into trying to secure this relationship because it means everything to you.\"\n }\n}\n\n[[ ## candidate_answer ## ]]\n```json\n{\n \"analysis\": {\n \"validation_point\": \"It is normal/understandable to feel anxious and protective when you deeply love someone and want to safeguard your marriage.\",\n \"resource_identified\": \"Despite your fears and worries about the relationship, you demonstrated strong commitment and perseverance by actively monitoring and staying engaged every night to protect what matters to you.\",\n \"reframe_angle\": \"Shift from seeing the nightly checking as distrustful or controlling to recognizing it as an expression of deep care, dedication, and a willingness to fight for your marriage.\"\n },\n \"suggested_response\": {\n \"technique\": \"One-Key Three-Links (See, Understand, Support)\",\n \"draft_response\": \"1. Validate (See/Understand): 'It is really hard to feel worried and uncertain about your relationship, especially when you care so much.' 2. Highlight Resource (Support): 'Yet, I notice that you keep showing up every night and doing your best to protect your marriage, which takes a lot of effort and dedication.' 3. Reframe (Empower): 'This shows how much you value your husband and your marriage, and that you are willing to fight for what you cherish.'\"\n }\n}\n```","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"default"}}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} diff --git a/examples/showcase/psychotherapy/evals/dataset-encouragement.yaml b/examples/showcase/psychotherapy/evals/dataset-encouragement.yaml index c493ab67..fcaa57ac 100644 --- a/examples/showcase/psychotherapy/evals/dataset-encouragement.yaml +++ b/examples/showcase/psychotherapy/evals/dataset-encouragement.yaml @@ -8,7 +8,7 @@ execution: evaluators: - name: json_schema_validator type: code_judge - script: uv run validate_output.py + script: ["uv", "run", "validate_output.py"] - name: content_evaluator type: llm_judge @@ -249,6 +249,7 @@ evalcases: # Case 8: The "Enabling" Trap # Focus: Validating Emotion (Care) but NOT Behavior (Control). - id: adv-encouragement-enabling + # Baseline note: partial failure expected if behavior is praised (score ~0.8). outcome: |- Must identify 'resource_identified' as the *Desire for Connection/Safety*, NOT the act of checking. Failure: Praising the "diligence" of checking the phone. @@ -276,4 +277,4 @@ evalcases: "technique": "Reframing Intent vs Impact", "draft_response": "I can see how terrified you are of losing him. You are pouring so much energy into trying to secure this relationship because it means everything to you." } - } \ No newline at end of file + } diff --git a/examples/showcase/psychotherapy/evals/dataset-listening.baseline.jsonl b/examples/showcase/psychotherapy/evals/dataset-listening.baseline.jsonl new file mode 100644 index 00000000..81365990 --- /dev/null +++ b/examples/showcase/psychotherapy/evals/dataset-listening.baseline.jsonl @@ -0,0 +1,6 @@ +{"timestamp":"2026-01-03T12:09:02.829Z","eval_id":"listening-basic-overwhelmed-wife","dataset":"dataset-listening","score":0.95,"hits":["Clearly identifies shift from pursuing/arguing to withdrawal","Mentions testing the partner’s responsiveness (testing the relationship)","Identifies loneliness and sadness as complex emotions","Draft response accurately reflects the emotional and process layer"],"misses":["Does not explicitly name 'Resentment' and 'Despair' as key level 2 emotions"],"candidate_answer":"```json\n{\n \"analysis\": {\n \"level_1_content\": \"Client describes a pattern of conflict over household chores, previously involving arguments and yelling at her partner over playing games during her cooking. Recently, she disengaged: ignored him while he was on his phone, did chores herself, and withdrew from interaction. She notes that he was absorbed in his phone, ignored her, and the overflowing trash was unattended. She contemplates stopping the internet bill as a way to provoke his attention.\",\n \"level_2_emotion\": \"Primary: Frustration, resignation. Complex/Hidden: Loneliness, sadness, possible resentment and longing for acknowledgment.\",\n \"level_3_process\": \"Client is operating from a belief that her needs and contributions are ignored or undervalued by her partner. The dynamic appears to be shifting from Protest-Pursuit (arguing, fighting for attention) to Withdrawal (disengaging as a response to feeling unheard). The function of this statement is to express pain over emotional neglect, test her partner's responsiveness, and possibly seek validation for feeling invisible in the relationship.\"\n },\n \"suggested_response\": {\n \"focus\": \"Deep Empathy (Levels 2 & 3)\",\n \"draft_response\": \"It sounds like you feel deeply frustrated and maybe even lonely, seeing your efforts go unnoticed and feeling disconnected while he’s absorbed in his own world. Perhaps underneath, you're longing to feel seen and valued by him—and are wondering if anything you do will ever make him notice. Is it feeling like you’ve moved from trying to get his attention to almost giving up?\"\n }\n}\n```","target":"default","reasoning":"json_schema_validator: Valid JSON with all required keys: analysis | content_evaluator: The candidate accurately captures the core process dynamics, including the Pursuer-Distancer shift and testing the relationship, but misses strong explicit labeling of 'Resentment' and 'Despair' as required emotions.","lm_provider_request":{"chat_prompt":[{"role":"system","content":"\n---\ndescription: 'Analyze client statements using Three Levels of Listening framework: Content, Emotion, Process'\n---\n\n# Three Levels of Listening Framework\n\nYou are an expert Psychotherapist specializing in the Three Levels of Listening (倾听的三个层面).\n\n## Task\nAnalyze the client's input to uncover the explicit content, emotional texture, and underlying psychological process.\n\n## Input\nClient statement.\n\n## Analysis Steps\n\n### Level 1: Content\nSummarize the explicit narrative events, facts, or situations the client is reporting.\n\n### Level 2: Emotion\nIdentify the emotional landscape:\n- **Explicit:** What feelings did they name?\n- **Implicit:** What complex feelings (e.g., grievance, ambivalence, suppressed anger) are communicated through tone or context?\n\n### Level 3: Process\nAnalyze the \"How\" and \"Why\" (the psychological dynamics):\n- **Interpersonal:** What is the dynamic between the client and others? (e.g., Pursuer-Distancer, Enmeshment)\n- **Intrapsychic:** What does this say about their self-concept or core beliefs? (e.g., \"I am a victim\")\n- **Function:** Why are they saying this now? (e.g., To seek validation, to deflect responsibility, to test the relationship)\n\n## Output\nReturn valid JSON:\n\n```json\n{\n \"analysis\": {\n \"level_1_content\": \"Client reports [Specific Facts/Events].\",\n \"level_2_emotion\": \"Primary: [Emotion]. Complex/Hidden: [Deeper Emotion].\",\n \"level_3_process\": \"Client is operating from a belief that [Belief]. The dynamic appears to be [Dynamic, e.g., unconditional obedience vs. independence]. The function of this complaint is [Function].\"\n },\n \"suggested_response\": {\n \"focus\": \"Deep Empathy (Levels 2 & 3)\",\n \"draft_response\": \"Use 'You' statements. Reflect the emotion and the process. (e.g., 'It sounds like you feel [Emotion] because [Content], and perhaps deep down you are wishing for [Process/Need]...')\"\n }\n}\n```\n"},{"role":"user","content":"Analyze the following client statement using the Three Levels of Listening framework.\n\nClient Statement:\n\"I used to fight with him about the chores. I used to scream when he played games while I cooked. Yesterday, I walked right past him. He was on his phone. The trash was overflowing. I didn't say a word. I just cooked dinner, ate with the kids, and went to bed. He didn't even look up. I wonder... if I stopped paying the internet bill, would he notice me then?\""}]},"evaluator_results":[{"name":"json_schema_validator","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":[],"misses":[],"reasoning":"Valid JSON with all required keys: analysis","evaluator_provider_request":{"script":["uv","run","validate_output.py"],"cwd":"/root/projects/agentv/examples/showcase/psychotherapy/evals"}},{"name":"content_evaluator","type":"llm_judge","score":0.9,"weight":1,"verdict":"pass","hits":["Clearly identifies shift from pursuing/arguing to withdrawal","Mentions testing the partner’s responsiveness (testing the relationship)","Identifies loneliness and sadness as complex emotions","Draft response accurately reflects the emotional and process layer"],"misses":["Does not explicitly name 'Resentment' and 'Despair' as key level 2 emotions"],"reasoning":"The candidate accurately captures the core process dynamics, including the Pursuer-Distancer shift and testing the relationship, but misses strong explicit labeling of 'Resentment' and 'Despair' as required emotions.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nMust identify 'level_2_emotion' as Resentment/Despair (not just anger).\nMust identify 'level_3_process' as Pursuer-Distancer shift or Testing the relationship.\n\n[[ ## question ## ]]\n@[System]:\n\n---\ndescription: 'Analyze client statements using Three Levels of Listening framework: Content, Emotion, Process'\n---\n\n# Three Levels of Listening Framework\n\nYou are an expert Psychotherapist specializing in the Three Levels of Listening (倾听的三个层面).\n\n## Task\nAnalyze the client's input to uncover the explicit content, emotional texture, and underlying psychological process.\n\n## Input\nClient statement.\n\n## Analysis Steps\n\n### Level 1: Content\nSummarize the explicit narrative events, facts, or situations the client is reporting.\n\n### Level 2: Emotion\nIdentify the emotional landscape:\n- **Explicit:** What feelings did they name?\n- **Implicit:** What complex feelings (e.g., grievance, ambivalence, suppressed anger) are communicated through tone or context?\n\n### Level 3: Process\nAnalyze the \"How\" and \"Why\" (the psychological dynamics):\n- **Interpersonal:** What is the dynamic between the client and others? (e.g., Pursuer-Distancer, Enmeshment)\n- **Intrapsychic:** What does this say about their self-concept or core beliefs? (e.g., \"I am a victim\")\n- **Function:** Why are they saying this now? (e.g., To seek validation, to deflect responsibility, to test the relationship)\n\n## Output\nReturn valid JSON:\n\n```json\n{\n \"analysis\": {\n \"level_1_content\": \"Client reports [Specific Facts/Events].\",\n \"level_2_emotion\": \"Primary: [Emotion]. Complex/Hidden: [Deeper Emotion].\",\n \"level_3_process\": \"Client is operating from a belief that [Belief]. The dynamic appears to be [Dynamic, e.g., unconditional obedience vs. independence]. The function of this complaint is [Function].\"\n },\n \"suggested_response\": {\n \"focus\": \"Deep Empathy (Levels 2 & 3)\",\n \"draft_response\": \"Use 'You' statements. Reflect the emotion and the process. (e.g., 'It sounds like you feel [Emotion] because [Content], and perhaps deep down you are wishing for [Process/Need]...')\"\n }\n}\n```\n\n\n@[User]:\nAnalyze the following client statement using the Three Levels of Listening framework.\n\nClient Statement:\n\"I used to fight with him about the chores. I used to scream when he played games while I cooked. Yesterday, I walked right past him. He was on his phone. The trash was overflowing. I didn't say a word. I just cooked dinner, ate with the kids, and went to bed. He didn't even look up. I wonder... if I stopped paying the internet bill, would he notice me then?\"\n\n[[ ## reference_answer ## ]]\n{\n \"analysis\": {\n \"level_1_content\": \"Client stopped verbally fighting about chores/neglect; observed husband's continued disengagement; considering passive-aggressive action (cutting internet).\",\n \"level_2_emotion\": \"Resentment, Despair, Numbness/Detachment, Cold Anger.\",\n \"level_3_process\": \"Shift from 'Pursuer' (screaming) to 'Distancer' (silence). Testing the relationship's viability. Using withdrawal as a final attempt to gain attention or preparing to detach completely.\"\n },\n \"suggested_response\": {\n \"focus\": \"Validation of the shift\",\n \"draft_response\": \"It seems you have moved past the point of screaming and are now sitting with a heavy silence, wondering if he can see you at all.\"\n }\n}\n\n[[ ## candidate_answer ## ]]\n```json\n{\n \"analysis\": {\n \"level_1_content\": \"Client describes a pattern of conflict over household chores, previously involving arguments and yelling at her partner over playing games during her cooking. Recently, she disengaged: ignored him while he was on his phone, did chores herself, and withdrew from interaction. She notes that he was absorbed in his phone, ignored her, and the overflowing trash was unattended. She contemplates stopping the internet bill as a way to provoke his attention.\",\n \"level_2_emotion\": \"Primary: Frustration, resignation. Complex/Hidden: Loneliness, sadness, possible resentment and longing for acknowledgment.\",\n \"level_3_process\": \"Client is operating from a belief that her needs and contributions are ignored or undervalued by her partner. The dynamic appears to be shifting from Protest-Pursuit (arguing, fighting for attention) to Withdrawal (disengaging as a response to feeling unheard). The function of this statement is to express pain over emotional neglect, test her partner's responsiveness, and possibly seek validation for feeling invisible in the relationship.\"\n },\n \"suggested_response\": {\n \"focus\": \"Deep Empathy (Levels 2 & 3)\",\n \"draft_response\": \"It sounds like you feel deeply frustrated and maybe even lonely, seeing your efforts go unnoticed and feeling disconnected while he’s absorbed in his own world. Perhaps underneath, you're longing to feel seen and valued by him—and are wondering if anything you do will ever make him notice. Is it feeling like you’ve moved from trying to get his attention to almost giving up?\"\n }\n}\n```","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"default"}}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} +{"timestamp":"2026-01-03T12:09:03.243Z","eval_id":"listening-basic-returning-mother","dataset":"dataset-listening","score":0.975,"hits":["Accurately identifies transition from caregiving to work and repeated rejection","Captures primary and complex emotions, including self-doubt and worthlessness","Analyzes intrapsychic process and identity struggle in level_3_process","Draft response reflects the emotion and underlying process"],"misses":["Does not explicitly label the internalization of rejection as 'worthlessness' in level_3_process; instead, describes it as feelings of inadequacy and self-worth tied to productivity"],"candidate_answer":"```json\n{\n \"analysis\": {\n \"level_1_content\": \"Client describes transition from full-time caregiving of her 3-year-old daughter (who recently started kindergarten) to actively seeking employment. Despite her husband's support, her multiple job applications have received no response, and after securing an interview, she was rejected. She currently cannot find suitable employment.\",\n \"level_2_emotion\": \"Primary: Pain, unhappiness, self-doubt. Complex/Hidden: Feelings of inadequacy and existential worthlessness, possibly shame and anxiety about her value beyond parenting; frustration and discouragement.\",\n \"level_3_process\": \"Client is operating from a belief that her value is measured by productivity or external validation ('I am useless if I don't have a job'). The dynamic is intrapsychic struggle: shifting identity from 'mother' to 'individual' and negotiating self-worth. Functionally, she is expressing a need for affirmation and guidance, possibly seeking empathy, reassurance, and a way to rebuild her confidence.\"\n },\n \"suggested_response\": {\n \"focus\": \"Deep Empathy (Levels 2 & 3)\",\n \"draft_response\": \"It sounds like you feel deeply unhappy and are struggling with self-doubt because transitioning from caring for your daughter at home to seeking work hasn't gone as you hoped. Beneath that, it seems you might be wishing to feel valued and confident in your abilities, not just as a mother but as someone with skills to contribute outside the home. Is that close to how you're feeling?\"\n }\n}\n```","target":"default","reasoning":"json_schema_validator: Valid JSON with all required keys: analysis | content_evaluator: The candidate answer provides a thorough analysis aligned with the framework and nearly matches the reference; however, it stops short of directly stating that rejection is internalized as worthlessness in level_3_process.","lm_provider_request":{"chat_prompt":[{"role":"system","content":"\n---\ndescription: 'Analyze client statements using Three Levels of Listening framework: Content, Emotion, Process'\n---\n\n# Three Levels of Listening Framework\n\nYou are an expert Psychotherapist specializing in the Three Levels of Listening (倾听的三个层面).\n\n## Task\nAnalyze the client's input to uncover the explicit content, emotional texture, and underlying psychological process.\n\n## Input\nClient statement.\n\n## Analysis Steps\n\n### Level 1: Content\nSummarize the explicit narrative events, facts, or situations the client is reporting.\n\n### Level 2: Emotion\nIdentify the emotional landscape:\n- **Explicit:** What feelings did they name?\n- **Implicit:** What complex feelings (e.g., grievance, ambivalence, suppressed anger) are communicated through tone or context?\n\n### Level 3: Process\nAnalyze the \"How\" and \"Why\" (the psychological dynamics):\n- **Interpersonal:** What is the dynamic between the client and others? (e.g., Pursuer-Distancer, Enmeshment)\n- **Intrapsychic:** What does this say about their self-concept or core beliefs? (e.g., \"I am a victim\")\n- **Function:** Why are they saying this now? (e.g., To seek validation, to deflect responsibility, to test the relationship)\n\n## Output\nReturn valid JSON:\n\n```json\n{\n \"analysis\": {\n \"level_1_content\": \"Client reports [Specific Facts/Events].\",\n \"level_2_emotion\": \"Primary: [Emotion]. Complex/Hidden: [Deeper Emotion].\",\n \"level_3_process\": \"Client is operating from a belief that [Belief]. The dynamic appears to be [Dynamic, e.g., unconditional obedience vs. independence]. The function of this complaint is [Function].\"\n },\n \"suggested_response\": {\n \"focus\": \"Deep Empathy (Levels 2 & 3)\",\n \"draft_response\": \"Use 'You' statements. Reflect the emotion and the process. (e.g., 'It sounds like you feel [Emotion] because [Content], and perhaps deep down you are wishing for [Process/Need]...')\"\n }\n}\n```\n"},{"role":"user","content":"Analyze the following client statement using the Three Levels of Listening framework.\n\nClient Statement:\n\"I have a 3-year-old daughter who started kindergarten. I used to watch the child at home and felt useless, so I wanted to find a job. My husband supports me, but I sent many resumes with no response. I finally got an interview but didn't pass. I can't find a suitable job. I doubt myself—am I unable to do anything? I am in pain, unhappy every day, and don't know what to do.\""}]},"evaluator_results":[{"name":"json_schema_validator","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":[],"misses":[],"reasoning":"Valid JSON with all required keys: analysis","evaluator_provider_request":{"script":["uv","run","validate_output.py"],"cwd":"/root/projects/agentv/examples/showcase/psychotherapy/evals"}},{"name":"content_evaluator","type":"llm_judge","score":0.95,"weight":1,"verdict":"pass","hits":["Accurately identifies transition from caregiving to work and repeated rejection","Captures primary and complex emotions, including self-doubt and worthlessness","Analyzes intrapsychic process and identity struggle in level_3_process","Draft response reflects the emotion and underlying process"],"misses":["Does not explicitly label the internalization of rejection as 'worthlessness' in level_3_process; instead, describes it as feelings of inadequacy and self-worth tied to productivity"],"reasoning":"The candidate answer provides a thorough analysis aligned with the framework and nearly matches the reference; however, it stops short of directly stating that rejection is internalized as worthlessness in level_3_process.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nMust identify Internalizing rejection as worthlessness in 'level_3_process'.\n\n[[ ## question ## ]]\n@[System]:\n\n---\ndescription: 'Analyze client statements using Three Levels of Listening framework: Content, Emotion, Process'\n---\n\n# Three Levels of Listening Framework\n\nYou are an expert Psychotherapist specializing in the Three Levels of Listening (倾听的三个层面).\n\n## Task\nAnalyze the client's input to uncover the explicit content, emotional texture, and underlying psychological process.\n\n## Input\nClient statement.\n\n## Analysis Steps\n\n### Level 1: Content\nSummarize the explicit narrative events, facts, or situations the client is reporting.\n\n### Level 2: Emotion\nIdentify the emotional landscape:\n- **Explicit:** What feelings did they name?\n- **Implicit:** What complex feelings (e.g., grievance, ambivalence, suppressed anger) are communicated through tone or context?\n\n### Level 3: Process\nAnalyze the \"How\" and \"Why\" (the psychological dynamics):\n- **Interpersonal:** What is the dynamic between the client and others? (e.g., Pursuer-Distancer, Enmeshment)\n- **Intrapsychic:** What does this say about their self-concept or core beliefs? (e.g., \"I am a victim\")\n- **Function:** Why are they saying this now? (e.g., To seek validation, to deflect responsibility, to test the relationship)\n\n## Output\nReturn valid JSON:\n\n```json\n{\n \"analysis\": {\n \"level_1_content\": \"Client reports [Specific Facts/Events].\",\n \"level_2_emotion\": \"Primary: [Emotion]. Complex/Hidden: [Deeper Emotion].\",\n \"level_3_process\": \"Client is operating from a belief that [Belief]. The dynamic appears to be [Dynamic, e.g., unconditional obedience vs. independence]. The function of this complaint is [Function].\"\n },\n \"suggested_response\": {\n \"focus\": \"Deep Empathy (Levels 2 & 3)\",\n \"draft_response\": \"Use 'You' statements. Reflect the emotion and the process. (e.g., 'It sounds like you feel [Emotion] because [Content], and perhaps deep down you are wishing for [Process/Need]...')\"\n }\n}\n```\n\n\n@[User]:\nAnalyze the following client statement using the Three Levels of Listening framework.\n\nClient Statement:\n\"I have a 3-year-old daughter who started kindergarten. I used to watch the child at home and felt useless, so I wanted to find a job. My husband supports me, but I sent many resumes with no response. I finally got an interview but didn't pass. I can't find a suitable job. I doubt myself—am I unable to do anything? I am in pain, unhappy every day, and don't know what to do.\"\n\n[[ ## reference_answer ## ]]\n{\n \"analysis\": {\n \"level_1_content\": \"Transitioning from full-time Motherhood to Workforce; loss of previous role clarity; repeated rejection.\",\n \"level_2_emotion\": \"Pain, Unhappiness, Self-doubt, Anxiety, Confusion.\",\n \"level_3_process\": \"Overgeneralization and Internalization: Views job rejection as proof of total personal incompetence ('unable to do anything'). Loss of identity (not just a job).\"\n },\n \"suggested_response\": {\n \"focus\": \"Validation of Identity Crisis\",\n \"draft_response\": \"It sounds like the rejection makes you question your entire worth, not just your skills.\"\n }\n}\n\n[[ ## candidate_answer ## ]]\n```json\n{\n \"analysis\": {\n \"level_1_content\": \"Client describes transition from full-time caregiving of her 3-year-old daughter (who recently started kindergarten) to actively seeking employment. Despite her husband's support, her multiple job applications have received no response, and after securing an interview, she was rejected. She currently cannot find suitable employment.\",\n \"level_2_emotion\": \"Primary: Pain, unhappiness, self-doubt. Complex/Hidden: Feelings of inadequacy and existential worthlessness, possibly shame and anxiety about her value beyond parenting; frustration and discouragement.\",\n \"level_3_process\": \"Client is operating from a belief that her value is measured by productivity or external validation ('I am useless if I don't have a job'). The dynamic is intrapsychic struggle: shifting identity from 'mother' to 'individual' and negotiating self-worth. Functionally, she is expressing a need for affirmation and guidance, possibly seeking empathy, reassurance, and a way to rebuild her confidence.\"\n },\n \"suggested_response\": {\n \"focus\": \"Deep Empathy (Levels 2 & 3)\",\n \"draft_response\": \"It sounds like you feel deeply unhappy and are struggling with self-doubt because transitioning from caring for your daughter at home to seeking work hasn't gone as you hoped. Beneath that, it seems you might be wishing to feel valued and confident in your abilities, not just as a mother but as someone with skills to contribute outside the home. Is that close to how you're feeling?\"\n }\n}\n```","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"default"}}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} +{"timestamp":"2026-01-03T12:09:04.259Z","eval_id":"listening-basic-traditional-mother","dataset":"dataset-listening","score":0.95,"hits":["Identifies enmeshment and familial expectations in the process","Recognizes reframing of daughter's boundary as coldness","Notes underlying emotions of confusion, disappointment, and hurt","Captures the function as seeking validation and control"],"misses":["Does not explicitly name 'Financial exploitation framed as help' or gaslighting"],"candidate_answer":"```json\n{\n \"analysis\": {\n \"level_1_content\": \"Client reports their daughter is financially successful while her brother is struggling. The client suggested the daughter help her brother financially with a down payment, framing it as an investment in the family's future. The daughter reacted by stopping communication, and the client expresses confusion over her response.\",\n \"level_2_emotion\": \"Primary: Confusion, Hurt. Complex/Hidden: Disappointment, a sense of rejection, possibly resentment, and underlying anxiety about family cohesion.\",\n \"level_3_process\": \"Client is operating from a belief that family resources and support should be communal and shared. The interpersonal dynamic appears to be a pattern of expectation for filial reciprocity and perhaps enmeshment, with little attention to individual boundaries. Intrapsychically, there may be a core belief that 'I am a caretaker; my family should value my guidance and unity.' The function of this complaint is to seek validation for their worldview, externalize responsibility for the conflict, and attempt to reassert control or restore family unity by casting their daughter's behavior as cold and uncharacteristic.\"\n },\n \"suggested_response\": {\n \"focus\": \"Deep Empathy (Levels 2 & 3)\",\n \"draft_response\": \"It sounds like you feel deeply hurt and confused by your daughter's reaction, especially since you see helping each other as the heart of what family means. Beneath your words, I sense how important family connection and shared support are to you, and perhaps there's a longing for understanding—wanting reassurance that your values are still shared and that you haven't done something wrong. Would you be willing to explore what this moment brings up for you, and how it’s shaping your view of family right now?\"\n }\n}\n```","target":"default","reasoning":"json_schema_validator: Valid JSON with all required keys: analysis | content_evaluator: The answer thoroughly addresses emotional depth and enmeshment, and hints at manipulation, but it omits direct labeling of 'patriarchal values', explicit financial exploitation as help, and gaslighting, which are central to the process analysis required.","lm_provider_request":{"chat_prompt":[{"role":"system","content":"\n---\ndescription: 'Analyze client statements using Three Levels of Listening framework: Content, Emotion, Process'\n---\n\n# Three Levels of Listening Framework\n\nYou are an expert Psychotherapist specializing in the Three Levels of Listening (倾听的三个层面).\n\n## Task\nAnalyze the client's input to uncover the explicit content, emotional texture, and underlying psychological process.\n\n## Input\nClient statement.\n\n## Analysis Steps\n\n### Level 1: Content\nSummarize the explicit narrative events, facts, or situations the client is reporting.\n\n### Level 2: Emotion\nIdentify the emotional landscape:\n- **Explicit:** What feelings did they name?\n- **Implicit:** What complex feelings (e.g., grievance, ambivalence, suppressed anger) are communicated through tone or context?\n\n### Level 3: Process\nAnalyze the \"How\" and \"Why\" (the psychological dynamics):\n- **Interpersonal:** What is the dynamic between the client and others? (e.g., Pursuer-Distancer, Enmeshment)\n- **Intrapsychic:** What does this say about their self-concept or core beliefs? (e.g., \"I am a victim\")\n- **Function:** Why are they saying this now? (e.g., To seek validation, to deflect responsibility, to test the relationship)\n\n## Output\nReturn valid JSON:\n\n```json\n{\n \"analysis\": {\n \"level_1_content\": \"Client reports [Specific Facts/Events].\",\n \"level_2_emotion\": \"Primary: [Emotion]. Complex/Hidden: [Deeper Emotion].\",\n \"level_3_process\": \"Client is operating from a belief that [Belief]. The dynamic appears to be [Dynamic, e.g., unconditional obedience vs. independence]. The function of this complaint is [Function].\"\n },\n \"suggested_response\": {\n \"focus\": \"Deep Empathy (Levels 2 & 3)\",\n \"draft_response\": \"Use 'You' statements. Reflect the emotion and the process. (e.g., 'It sounds like you feel [Emotion] because [Content], and perhaps deep down you are wishing for [Process/Need]...')\"\n }\n}\n```\n"},{"role":"user","content":"Analyze the following client statement using the Three Levels of Listening framework.\n\nClient Statement:\n\"My daughter is doing so well, she saves so much money. Her brother is just starting out, he's struggling. As a family, we help each other, right? I simply suggested she help him with the down payment. It's an investment for the family's future. But she stopped talking to me! I don't understand. Isn't family about sharing? Why is she being so calculated and cold with her own flesh and blood?\""}]},"evaluator_results":[{"name":"json_schema_validator","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":[],"misses":[],"reasoning":"Valid JSON with all required keys: analysis","evaluator_provider_request":{"script":["uv","run","validate_output.py"],"cwd":"/root/projects/agentv/examples/showcase/psychotherapy/evals"}},{"name":"content_evaluator","type":"llm_judge","score":0.9,"weight":1,"verdict":"pass","hits":["Identifies enmeshment and familial expectations in the process","Recognizes reframing of daughter's boundary as coldness","Notes underlying emotions of confusion, disappointment, and hurt","Captures the function as seeking validation and control"],"misses":["Does not explicitly name 'Financial exploitation framed as help' or gaslighting"],"reasoning":"The answer thoroughly addresses emotional depth and enmeshment, and hints at manipulation, but it omits direct labeling of 'patriarchal values', explicit financial exploitation as help, and gaslighting, which are central to the process analysis required.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nMust identify Patriarchal values (Financial exploitation framed as help) and Gaslighting/Enmeshment in 'level_3_process'.\n\n[[ ## question ## ]]\n@[System]:\n\n---\ndescription: 'Analyze client statements using Three Levels of Listening framework: Content, Emotion, Process'\n---\n\n# Three Levels of Listening Framework\n\nYou are an expert Psychotherapist specializing in the Three Levels of Listening (倾听的三个层面).\n\n## Task\nAnalyze the client's input to uncover the explicit content, emotional texture, and underlying psychological process.\n\n## Input\nClient statement.\n\n## Analysis Steps\n\n### Level 1: Content\nSummarize the explicit narrative events, facts, or situations the client is reporting.\n\n### Level 2: Emotion\nIdentify the emotional landscape:\n- **Explicit:** What feelings did they name?\n- **Implicit:** What complex feelings (e.g., grievance, ambivalence, suppressed anger) are communicated through tone or context?\n\n### Level 3: Process\nAnalyze the \"How\" and \"Why\" (the psychological dynamics):\n- **Interpersonal:** What is the dynamic between the client and others? (e.g., Pursuer-Distancer, Enmeshment)\n- **Intrapsychic:** What does this say about their self-concept or core beliefs? (e.g., \"I am a victim\")\n- **Function:** Why are they saying this now? (e.g., To seek validation, to deflect responsibility, to test the relationship)\n\n## Output\nReturn valid JSON:\n\n```json\n{\n \"analysis\": {\n \"level_1_content\": \"Client reports [Specific Facts/Events].\",\n \"level_2_emotion\": \"Primary: [Emotion]. Complex/Hidden: [Deeper Emotion].\",\n \"level_3_process\": \"Client is operating from a belief that [Belief]. The dynamic appears to be [Dynamic, e.g., unconditional obedience vs. independence]. The function of this complaint is [Function].\"\n },\n \"suggested_response\": {\n \"focus\": \"Deep Empathy (Levels 2 & 3)\",\n \"draft_response\": \"Use 'You' statements. Reflect the emotion and the process. (e.g., 'It sounds like you feel [Emotion] because [Content], and perhaps deep down you are wishing for [Process/Need]...')\"\n }\n}\n```\n\n\n@[User]:\nAnalyze the following client statement using the Three Levels of Listening framework.\n\nClient Statement:\n\"My daughter is doing so well, she saves so much money. Her brother is just starting out, he's struggling. As a family, we help each other, right? I simply suggested she help him with the down payment. It's an investment for the family's future. But she stopped talking to me! I don't understand. Isn't family about sharing? Why is she being so calculated and cold with her own flesh and blood?\"\n\n[[ ## reference_answer ## ]]\n{\n \"analysis\": {\n \"level_1_content\": \"Mother asked successful daughter to pay for struggling son's down payment; Daughter cut contact; Mother frames this as 'family help' and daughter as 'cold'.\",\n \"level_2_emotion\": \"Confusion (feigned or genuine), Self-righteousness, Victimhood, Indignation.\",\n \"level_3_process\": \"Manipulative Reframing: Defines 'financial exploitation' as 'family values'. Gaslighting: Labels daughter's boundary setting as 'being cold'. Enmeshment: Fails to see daughter as a separate entity with rights.\"\n },\n \"suggested_response\": {\n \"focus\": \"Reflection of value conflict\",\n \"draft_response\": \"You feel deeply confused because, for you, family means sharing everything, and her boundary feels like a rejection of the family itself.\"\n }\n}\n\n[[ ## candidate_answer ## ]]\n```json\n{\n \"analysis\": {\n \"level_1_content\": \"Client reports their daughter is financially successful while her brother is struggling. The client suggested the daughter help her brother financially with a down payment, framing it as an investment in the family's future. The daughter reacted by stopping communication, and the client expresses confusion over her response.\",\n \"level_2_emotion\": \"Primary: Confusion, Hurt. Complex/Hidden: Disappointment, a sense of rejection, possibly resentment, and underlying anxiety about family cohesion.\",\n \"level_3_process\": \"Client is operating from a belief that family resources and support should be communal and shared. The interpersonal dynamic appears to be a pattern of expectation for filial reciprocity and perhaps enmeshment, with little attention to individual boundaries. Intrapsychically, there may be a core belief that 'I am a caretaker; my family should value my guidance and unity.' The function of this complaint is to seek validation for their worldview, externalize responsibility for the conflict, and attempt to reassert control or restore family unity by casting their daughter's behavior as cold and uncharacteristic.\"\n },\n \"suggested_response\": {\n \"focus\": \"Deep Empathy (Levels 2 & 3)\",\n \"draft_response\": \"It sounds like you feel deeply hurt and confused by your daughter's reaction, especially since you see helping each other as the heart of what family means. Beneath your words, I sense how important family connection and shared support are to you, and perhaps there's a longing for understanding—wanting reassurance that your values are still shared and that you haven't done something wrong. Would you be willing to explore what this moment brings up for you, and how it’s shaping your view of family right now?\"\n }\n}\n```","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"default"}}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} +{"timestamp":"2026-01-03T12:09:07.129Z","eval_id":"listening-adv-somatic-mask","dataset":"dataset-listening","score":0.95,"hits":["Captures explicit denial of anxiety and focus on physical symptoms","Highlights pride and resistance to emotional explanations","Notes preservation of self-image and deflection of psychological inquiry","Draft response reflects both emotional and process-level empathy"],"misses":["Does not explicitly name 'alexithymia' or 'somatic displacement' in level_3_process"],"candidate_answer":"```json\n{\n \"analysis\": {\n \"level_1_content\": \"Client reports recent success at work, including a promotion and satisfaction with high pressure. They are experiencing persistent throat tightness described as a lump. The client's wife has suggested the issue may be anxiety, which the client rejects.\",\n \"level_2_emotion\": \"Primary: Pride, self-assurance. Complex/Hidden: Defensiveness, underlying anxiety or discomfort that is not openly acknowledged; possible frustration with wife's suggestion.\",\n \"level_3_process\": \"Client is operating from a belief that 'I am strong and unaffected by psychological problems' or 'Physical symptoms have nothing to do with emotions.' The dynamic appears to be dismissive regarding emotional explanations and resistant to vulnerability. The function of this complaint is to preserve self-image, deflect psychological inquiry, and maintain control over the narrative while seeking practical (medical) help rather than emotional support.\"\n },\n \"suggested_response\": {\n \"focus\": \"Deep Empathy (Levels 2 & 3)\",\n \"draft_response\": \"It sounds like you take great pride in your achievements and your ability to handle pressure, and at the same time, this persistent throat sensation is both confusing and perhaps unsettling for you. I wonder if part of what’s difficult here is feeling misunderstood when others, like your wife, suggest it could be anxiety—something you feel doesn’t fit with your self-perception. Can you tell me more about what it means for you when physical symptoms show up, especially when your life seems to be going so well?\"\n }\n}\n```","target":"default","reasoning":"json_schema_validator: Valid JSON with all required keys: analysis | content_evaluator: The answer correctly identifies all key dynamics and gives an empathetic response, but omits the explicit terms 'alexithymia' and 'somatic displacement' as required in level_3_process.","lm_provider_request":{"chat_prompt":[{"role":"system","content":"\n---\ndescription: 'Analyze client statements using Three Levels of Listening framework: Content, Emotion, Process'\n---\n\n# Three Levels of Listening Framework\n\nYou are an expert Psychotherapist specializing in the Three Levels of Listening (倾听的三个层面).\n\n## Task\nAnalyze the client's input to uncover the explicit content, emotional texture, and underlying psychological process.\n\n## Input\nClient statement.\n\n## Analysis Steps\n\n### Level 1: Content\nSummarize the explicit narrative events, facts, or situations the client is reporting.\n\n### Level 2: Emotion\nIdentify the emotional landscape:\n- **Explicit:** What feelings did they name?\n- **Implicit:** What complex feelings (e.g., grievance, ambivalence, suppressed anger) are communicated through tone or context?\n\n### Level 3: Process\nAnalyze the \"How\" and \"Why\" (the psychological dynamics):\n- **Interpersonal:** What is the dynamic between the client and others? (e.g., Pursuer-Distancer, Enmeshment)\n- **Intrapsychic:** What does this say about their self-concept or core beliefs? (e.g., \"I am a victim\")\n- **Function:** Why are they saying this now? (e.g., To seek validation, to deflect responsibility, to test the relationship)\n\n## Output\nReturn valid JSON:\n\n```json\n{\n \"analysis\": {\n \"level_1_content\": \"Client reports [Specific Facts/Events].\",\n \"level_2_emotion\": \"Primary: [Emotion]. Complex/Hidden: [Deeper Emotion].\",\n \"level_3_process\": \"Client is operating from a belief that [Belief]. The dynamic appears to be [Dynamic, e.g., unconditional obedience vs. independence]. The function of this complaint is [Function].\"\n },\n \"suggested_response\": {\n \"focus\": \"Deep Empathy (Levels 2 & 3)\",\n \"draft_response\": \"Use 'You' statements. Reflect the emotion and the process. (e.g., 'It sounds like you feel [Emotion] because [Content], and perhaps deep down you are wishing for [Process/Need]...')\"\n }\n}\n```\n"},{"role":"user","content":"Analyze the following client statement using the Three Levels of Listening framework.\n\nClient Statement:\n\"My life is perfect. Work is booming, just got a promotion. I'm thriving on the pressure. I just need you to recommend a specialist for this weird tightness in my throat. It's like a lump that won't go away. My wife keeps asking if I'm 'anxious', which is ridiculous. I don't do anxiety. I just have a throat issue.\""}]},"evaluator_results":[{"name":"json_schema_validator","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":[],"misses":[],"reasoning":"Valid JSON with all required keys: analysis","evaluator_provider_request":{"script":["uv","run","validate_output.py"],"cwd":"/root/projects/agentv/examples/showcase/psychotherapy/evals"}},{"name":"content_evaluator","type":"llm_judge","score":0.9,"weight":1,"verdict":"pass","hits":["Captures explicit denial of anxiety and focus on physical symptoms","Highlights pride and resistance to emotional explanations","Notes preservation of self-image and deflection of psychological inquiry","Draft response reflects both emotional and process-level empathy"],"misses":["Does not explicitly name 'alexithymia' or 'somatic displacement' in level_3_process"],"reasoning":"The answer correctly identifies all key dynamics and gives an empathetic response, but omits the explicit terms 'alexithymia' and 'somatic displacement' as required in level_3_process.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nMust identify Alexithymia and Somatic Displacement (Stress -> Throat) in 'level_3_process'.\n\n[[ ## question ## ]]\n@[System]:\n\n---\ndescription: 'Analyze client statements using Three Levels of Listening framework: Content, Emotion, Process'\n---\n\n# Three Levels of Listening Framework\n\nYou are an expert Psychotherapist specializing in the Three Levels of Listening (倾听的三个层面).\n\n## Task\nAnalyze the client's input to uncover the explicit content, emotional texture, and underlying psychological process.\n\n## Input\nClient statement.\n\n## Analysis Steps\n\n### Level 1: Content\nSummarize the explicit narrative events, facts, or situations the client is reporting.\n\n### Level 2: Emotion\nIdentify the emotional landscape:\n- **Explicit:** What feelings did they name?\n- **Implicit:** What complex feelings (e.g., grievance, ambivalence, suppressed anger) are communicated through tone or context?\n\n### Level 3: Process\nAnalyze the \"How\" and \"Why\" (the psychological dynamics):\n- **Interpersonal:** What is the dynamic between the client and others? (e.g., Pursuer-Distancer, Enmeshment)\n- **Intrapsychic:** What does this say about their self-concept or core beliefs? (e.g., \"I am a victim\")\n- **Function:** Why are they saying this now? (e.g., To seek validation, to deflect responsibility, to test the relationship)\n\n## Output\nReturn valid JSON:\n\n```json\n{\n \"analysis\": {\n \"level_1_content\": \"Client reports [Specific Facts/Events].\",\n \"level_2_emotion\": \"Primary: [Emotion]. Complex/Hidden: [Deeper Emotion].\",\n \"level_3_process\": \"Client is operating from a belief that [Belief]. The dynamic appears to be [Dynamic, e.g., unconditional obedience vs. independence]. The function of this complaint is [Function].\"\n },\n \"suggested_response\": {\n \"focus\": \"Deep Empathy (Levels 2 & 3)\",\n \"draft_response\": \"Use 'You' statements. Reflect the emotion and the process. (e.g., 'It sounds like you feel [Emotion] because [Content], and perhaps deep down you are wishing for [Process/Need]...')\"\n }\n}\n```\n\n\n@[User]:\nAnalyze the following client statement using the Three Levels of Listening framework.\n\nClient Statement:\n\"My life is perfect. Work is booming, just got a promotion. I'm thriving on the pressure. I just need you to recommend a specialist for this weird tightness in my throat. It's like a lump that won't go away. My wife keeps asking if I'm 'anxious', which is ridiculous. I don't do anxiety. I just have a throat issue.\"\n\n[[ ## reference_answer ## ]]\n{\n \"analysis\": {\n \"level_1_content\": \"Reports 'perfect life' and career success; Denies anxiety/stress explicitly; Seeks medical fix for 'throat lump' (Globus sensation).\",\n \"level_2_emotion\": \"Disconnected/Numb (Alexithymia), Defensive pride. Implicit: Panic (displaced).\",\n \"level_3_process\": \"Alexithymia: Inability to identify/verbalize feelings. Somatic Displacement: Stress is converted into physical symptoms (throat tightness) because emotional expression is viewed as weakness.\"\n },\n \"suggested_response\": {\n \"focus\": \"Connecting Body and Mind safely\",\n \"draft_response\": \"It sounds like your mind is handling the pressure perfectly, but your throat seems to be carrying a tension that doesn't fit the 'perfect' picture.\"\n }\n}\n\n[[ ## candidate_answer ## ]]\n```json\n{\n \"analysis\": {\n \"level_1_content\": \"Client reports recent success at work, including a promotion and satisfaction with high pressure. They are experiencing persistent throat tightness described as a lump. The client's wife has suggested the issue may be anxiety, which the client rejects.\",\n \"level_2_emotion\": \"Primary: Pride, self-assurance. Complex/Hidden: Defensiveness, underlying anxiety or discomfort that is not openly acknowledged; possible frustration with wife's suggestion.\",\n \"level_3_process\": \"Client is operating from a belief that 'I am strong and unaffected by psychological problems' or 'Physical symptoms have nothing to do with emotions.' The dynamic appears to be dismissive regarding emotional explanations and resistant to vulnerability. The function of this complaint is to preserve self-image, deflect psychological inquiry, and maintain control over the narrative while seeking practical (medical) help rather than emotional support.\"\n },\n \"suggested_response\": {\n \"focus\": \"Deep Empathy (Levels 2 & 3)\",\n \"draft_response\": \"It sounds like you take great pride in your achievements and your ability to handle pressure, and at the same time, this persistent throat sensation is both confusing and perhaps unsettling for you. I wonder if part of what’s difficult here is feeling misunderstood when others, like your wife, suggest it could be anxiety—something you feel doesn’t fit with your self-perception. Can you tell me more about what it means for you when physical symptoms show up, especially when your life seems to be going so well?\"\n }\n}\n```","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"default"}}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} +{"timestamp":"2026-01-03T12:09:07.970Z","eval_id":"listening-adv-gatekeeper","dataset":"dataset-listening","score":0.9,"hits":["Accurately identifies the client acting as mediator and reframing communication","Recognizes emotional exhaustion and underlying anxiety/resentment","Notes overfunctioning/caretaking dynamic and search for understanding/validation","Drafts a deeply empathetic suggested response reflecting emotion and process"],"misses":["Does not explicitly name 'gatekeeping/triangulation' dynamic","Does not highlight the distinction between 'helping' vs 'blocking intimacy'"],"candidate_answer":"```json\n{\n \"analysis\": {\n \"level_1_content\": \"Client describes acting as a mediator between their mother and wife, regularly reframing or softening communication from each party to the other to avoid hurt feelings. The client highlights specific examples, such as explaining Mom's criticism of cooking and wife's reactions during conversations. Despite these efforts, the client reports feeling exhausted and questions why this is so given their belief that they are doing something good.\",\n \"level_2_emotion\": \"Primary: Exhaustion, frustration. Complex/Hidden: Anxiety about family harmony, possible resentment or loneliness due to lack of appreciation or recognition, guilt for possibly not meeting everyone's needs or for feeling negative about the 'good' role they play.\",\n \"level_3_process\": \"Client is operating from a belief that it is their responsibility to keep peace and prevent conflict (potentially 'I am the fixer' or 'I must protect others from hurt'). The dynamic appears to be one of emotional overfunctioning—taking on an intermediary, caretaking role, possibly to the point of self-neglect. The function of this statement is to seek understanding and validation for their exhaustion, to question whether their sacrifice is sustainable or acknowledged, and to perhaps indirectly voice a need for support or relief.\"\n },\n \"suggested_response\": {\n \"focus\": \"Deep Empathy (Levels 2 & 3)\",\n \"draft_response\": \"It sounds like you feel really worn out from constantly trying to protect both your mom and your wife from being hurt, even though you believe that's the caring thing to do. Deep down, it seems you might be wishing for some support or acknowledgment for how much effort you're putting in—and maybe wondering if it's okay to care for yourself in all of this too.\"\n }\n}\n```","target":"default","reasoning":"json_schema_validator: Valid JSON with all required keys: analysis | content_evaluator: The answer provides a thorough analysis of content, emotion, and psychological process, but omits explicit identification of gatekeeping/triangulation and does not address the difference between helping and blocking intimacy, resulting in a partial miss on key expected points.","lm_provider_request":{"chat_prompt":[{"role":"system","content":"\n---\ndescription: 'Analyze client statements using Three Levels of Listening framework: Content, Emotion, Process'\n---\n\n# Three Levels of Listening Framework\n\nYou are an expert Psychotherapist specializing in the Three Levels of Listening (倾听的三个层面).\n\n## Task\nAnalyze the client's input to uncover the explicit content, emotional texture, and underlying psychological process.\n\n## Input\nClient statement.\n\n## Analysis Steps\n\n### Level 1: Content\nSummarize the explicit narrative events, facts, or situations the client is reporting.\n\n### Level 2: Emotion\nIdentify the emotional landscape:\n- **Explicit:** What feelings did they name?\n- **Implicit:** What complex feelings (e.g., grievance, ambivalence, suppressed anger) are communicated through tone or context?\n\n### Level 3: Process\nAnalyze the \"How\" and \"Why\" (the psychological dynamics):\n- **Interpersonal:** What is the dynamic between the client and others? (e.g., Pursuer-Distancer, Enmeshment)\n- **Intrapsychic:** What does this say about their self-concept or core beliefs? (e.g., \"I am a victim\")\n- **Function:** Why are they saying this now? (e.g., To seek validation, to deflect responsibility, to test the relationship)\n\n## Output\nReturn valid JSON:\n\n```json\n{\n \"analysis\": {\n \"level_1_content\": \"Client reports [Specific Facts/Events].\",\n \"level_2_emotion\": \"Primary: [Emotion]. Complex/Hidden: [Deeper Emotion].\",\n \"level_3_process\": \"Client is operating from a belief that [Belief]. The dynamic appears to be [Dynamic, e.g., unconditional obedience vs. independence]. The function of this complaint is [Function].\"\n },\n \"suggested_response\": {\n \"focus\": \"Deep Empathy (Levels 2 & 3)\",\n \"draft_response\": \"Use 'You' statements. Reflect the emotion and the process. (e.g., 'It sounds like you feel [Emotion] because [Content], and perhaps deep down you are wishing for [Process/Need]...')\"\n }\n}\n```\n"},{"role":"user","content":"Analyze the following client statement using the Three Levels of Listening framework.\n\nClient Statement:\n\"I'm the bridge in this family. When Mom says my wife's cooking is 'bland', I tell my wife that Mom is just worried about her salt intake. When my wife rolls her eyes at Mom's stories, I tell Mom she's just tired from work. I work so hard to translate for them so they don't get hurt. Why do I feel so exhausted if I'm doing such a good thing?\""}]},"evaluator_results":[{"name":"json_schema_validator","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":[],"misses":[],"reasoning":"Valid JSON with all required keys: analysis","evaluator_provider_request":{"script":["uv","run","validate_output.py"],"cwd":"/root/projects/agentv/examples/showcase/psychotherapy/evals"}},{"name":"content_evaluator","type":"llm_judge","score":0.8,"weight":1,"verdict":"pass","hits":["Accurately identifies the client acting as mediator and reframing communication","Recognizes emotional exhaustion and underlying anxiety/resentment","Notes overfunctioning/caretaking dynamic and search for understanding/validation","Drafts a deeply empathetic suggested response reflecting emotion and process"],"misses":["Does not explicitly name 'gatekeeping/triangulation' dynamic","Does not highlight the distinction between 'helping' vs 'blocking intimacy'"],"reasoning":"The answer provides a thorough analysis of content, emotion, and psychological process, but omits explicit identification of gatekeeping/triangulation and does not address the difference between helping and blocking intimacy, resulting in a partial miss on key expected points.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nMust identify Gatekeeping/Triangulation in 'level_3_process'.\nMust identify the distinction between 'Helping' vs 'Blocking Intimacy'.\n\n[[ ## question ## ]]\n@[System]:\n\n---\ndescription: 'Analyze client statements using Three Levels of Listening framework: Content, Emotion, Process'\n---\n\n# Three Levels of Listening Framework\n\nYou are an expert Psychotherapist specializing in the Three Levels of Listening (倾听的三个层面).\n\n## Task\nAnalyze the client's input to uncover the explicit content, emotional texture, and underlying psychological process.\n\n## Input\nClient statement.\n\n## Analysis Steps\n\n### Level 1: Content\nSummarize the explicit narrative events, facts, or situations the client is reporting.\n\n### Level 2: Emotion\nIdentify the emotional landscape:\n- **Explicit:** What feelings did they name?\n- **Implicit:** What complex feelings (e.g., grievance, ambivalence, suppressed anger) are communicated through tone or context?\n\n### Level 3: Process\nAnalyze the \"How\" and \"Why\" (the psychological dynamics):\n- **Interpersonal:** What is the dynamic between the client and others? (e.g., Pursuer-Distancer, Enmeshment)\n- **Intrapsychic:** What does this say about their self-concept or core beliefs? (e.g., \"I am a victim\")\n- **Function:** Why are they saying this now? (e.g., To seek validation, to deflect responsibility, to test the relationship)\n\n## Output\nReturn valid JSON:\n\n```json\n{\n \"analysis\": {\n \"level_1_content\": \"Client reports [Specific Facts/Events].\",\n \"level_2_emotion\": \"Primary: [Emotion]. Complex/Hidden: [Deeper Emotion].\",\n \"level_3_process\": \"Client is operating from a belief that [Belief]. The dynamic appears to be [Dynamic, e.g., unconditional obedience vs. independence]. The function of this complaint is [Function].\"\n },\n \"suggested_response\": {\n \"focus\": \"Deep Empathy (Levels 2 & 3)\",\n \"draft_response\": \"Use 'You' statements. Reflect the emotion and the process. (e.g., 'It sounds like you feel [Emotion] because [Content], and perhaps deep down you are wishing for [Process/Need]...')\"\n }\n}\n```\n\n\n@[User]:\nAnalyze the following client statement using the Three Levels of Listening framework.\n\nClient Statement:\n\"I'm the bridge in this family. When Mom says my wife's cooking is 'bland', I tell my wife that Mom is just worried about her salt intake. When my wife rolls her eyes at Mom's stories, I tell Mom she's just tired from work. I work so hard to translate for them so they don't get hurt. Why do I feel so exhausted if I'm doing such a good thing?\"\n\n[[ ## reference_answer ## ]]\n{\n \"analysis\": {\n \"level_1_content\": \"Client actively 'translates' (distorts) critical messages between wife and mother to prevent conflict; Reports exhaustion.\",\n \"level_2_emotion\": \"Exhaustion, Anxiety, Fear of conflict, False pride in being the 'bridge'.\",\n \"level_3_process\": \"Gatekeeping/Triangulation: By 'translating', the client prevents direct contact and authentic relationship (even conflict) between the women. He maintains the dysfunction by absorbing the tension.\"\n },\n \"suggested_response\": {\n \"focus\": \"Highlighting the Cost of the Role\",\n \"draft_response\": \"You are working incredibly hard to protect them from each other, but it sounds like you are the one absorbing all the impact.\"\n }\n}\n\n[[ ## candidate_answer ## ]]\n```json\n{\n \"analysis\": {\n \"level_1_content\": \"Client describes acting as a mediator between their mother and wife, regularly reframing or softening communication from each party to the other to avoid hurt feelings. The client highlights specific examples, such as explaining Mom's criticism of cooking and wife's reactions during conversations. Despite these efforts, the client reports feeling exhausted and questions why this is so given their belief that they are doing something good.\",\n \"level_2_emotion\": \"Primary: Exhaustion, frustration. Complex/Hidden: Anxiety about family harmony, possible resentment or loneliness due to lack of appreciation or recognition, guilt for possibly not meeting everyone's needs or for feeling negative about the 'good' role they play.\",\n \"level_3_process\": \"Client is operating from a belief that it is their responsibility to keep peace and prevent conflict (potentially 'I am the fixer' or 'I must protect others from hurt'). The dynamic appears to be one of emotional overfunctioning—taking on an intermediary, caretaking role, possibly to the point of self-neglect. The function of this statement is to seek understanding and validation for their exhaustion, to question whether their sacrifice is sustainable or acknowledged, and to perhaps indirectly voice a need for support or relief.\"\n },\n \"suggested_response\": {\n \"focus\": \"Deep Empathy (Levels 2 & 3)\",\n \"draft_response\": \"It sounds like you feel really worn out from constantly trying to protect both your mom and your wife from being hurt, even though you believe that's the caring thing to do. Deep down, it seems you might be wishing for some support or acknowledgment for how much effort you're putting in—and maybe wondering if it's okay to care for yourself in all of this too.\"\n }\n}\n```","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"default"}}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} +{"timestamp":"2026-01-03T12:09:08.865Z","eval_id":"listening-adv-intellectualizer","dataset":"dataset-listening","score":1,"hits":["Identifies intellectualization as the defense mechanism in level_3_process","Accurately summarizes the client's emphasis on theory and efficacy","Recognizes underlying anxiety/discomfort about vulnerability","Drafts a response that blends validation with gentle curiosity about feelings"],"misses":[],"candidate_answer":"```json\n{\n \"analysis\": {\n \"level_1_content\": \"Client references knowledge of the 'Empty Chair' technique from Gestalt therapy and expresses interest in discussing the scientific basis and efficacy rates versus CBT before engaging in the intervention.\",\n \"level_2_emotion\": \"Primary: Intellectual curiosity, cautious interest. Complex/Hidden: Anxiety or discomfort about directly accessing or expressing anger; possible distancing via intellectualization.\",\n \"level_3_process\": \"Client seems to be operating from a belief that intellectual understanding provides safety or control in emotionally vulnerable situations. The dynamic appears to be intellectualization as a defense—using theory to avoid vulnerable affect. The function of this statement is to maintain emotional distance, perhaps to test the therapist’s expertise/safety before risking emotional exposure.\"\n },\n \"suggested_response\": {\n \"focus\": \"Deep Empathy (Levels 2 & 3)\",\n \"draft_response\": \"It sounds like you're both genuinely interested in how these therapeutic approaches work, and maybe also wanting to make sure you feel safe and informed before trying something emotionally intense. I wonder if talking about the science helps you feel more comfortable with what can be a vulnerable process like expressing anger. Would you like to explore both the research and any feelings or hesitations you might have about this technique?\"\n }\n}\n```","target":"default","reasoning":"json_schema_validator: Valid JSON with all required keys: analysis | content_evaluator: The candidate answer explicitly names intellectualization as the process, matches key insights from the reference, and the response is empathetic and nuanced. All core expectations are fully addressed.","lm_provider_request":{"chat_prompt":[{"role":"system","content":"\n---\ndescription: 'Analyze client statements using Three Levels of Listening framework: Content, Emotion, Process'\n---\n\n# Three Levels of Listening Framework\n\nYou are an expert Psychotherapist specializing in the Three Levels of Listening (倾听的三个层面).\n\n## Task\nAnalyze the client's input to uncover the explicit content, emotional texture, and underlying psychological process.\n\n## Input\nClient statement.\n\n## Analysis Steps\n\n### Level 1: Content\nSummarize the explicit narrative events, facts, or situations the client is reporting.\n\n### Level 2: Emotion\nIdentify the emotional landscape:\n- **Explicit:** What feelings did they name?\n- **Implicit:** What complex feelings (e.g., grievance, ambivalence, suppressed anger) are communicated through tone or context?\n\n### Level 3: Process\nAnalyze the \"How\" and \"Why\" (the psychological dynamics):\n- **Interpersonal:** What is the dynamic between the client and others? (e.g., Pursuer-Distancer, Enmeshment)\n- **Intrapsychic:** What does this say about their self-concept or core beliefs? (e.g., \"I am a victim\")\n- **Function:** Why are they saying this now? (e.g., To seek validation, to deflect responsibility, to test the relationship)\n\n## Output\nReturn valid JSON:\n\n```json\n{\n \"analysis\": {\n \"level_1_content\": \"Client reports [Specific Facts/Events].\",\n \"level_2_emotion\": \"Primary: [Emotion]. Complex/Hidden: [Deeper Emotion].\",\n \"level_3_process\": \"Client is operating from a belief that [Belief]. The dynamic appears to be [Dynamic, e.g., unconditional obedience vs. independence]. The function of this complaint is [Function].\"\n },\n \"suggested_response\": {\n \"focus\": \"Deep Empathy (Levels 2 & 3)\",\n \"draft_response\": \"Use 'You' statements. Reflect the emotion and the process. (e.g., 'It sounds like you feel [Emotion] because [Content], and perhaps deep down you are wishing for [Process/Need]...')\"\n }\n}\n```\n"},{"role":"user","content":"Analyze the following client statement using the Three Levels of Listening framework. (Context: Therapist suggested expressing anger to a parent).\n\nClient Statement:\n\"I've read about this. It's the 'Empty Chair' technique, right? Gestalt therapy. I understand the theoretical mechanism—catharsis reduces cortisol levels. It's fascinating how the brain processes repressed anger. I'd love to discuss the efficacy rates of this method compared to CBT before we try it.\""}]},"evaluator_results":[{"name":"json_schema_validator","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":[],"misses":[],"reasoning":"Valid JSON with all required keys: analysis","evaluator_provider_request":{"script":["uv","run","validate_output.py"],"cwd":"/root/projects/agentv/examples/showcase/psychotherapy/evals"}},{"name":"content_evaluator","type":"llm_judge","score":1,"weight":1,"verdict":"pass","hits":["Identifies intellectualization as the defense mechanism in level_3_process","Accurately summarizes the client's emphasis on theory and efficacy","Recognizes underlying anxiety/discomfort about vulnerability","Drafts a response that blends validation with gentle curiosity about feelings"],"misses":[],"reasoning":"The candidate answer explicitly names intellectualization as the process, matches key insights from the reference, and the response is empathetic and nuanced. All core expectations are fully addressed.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nMust identify Intellectualization as a defense mechanism in 'level_3_process'.\n\n[[ ## question ## ]]\n@[System]:\n\n---\ndescription: 'Analyze client statements using Three Levels of Listening framework: Content, Emotion, Process'\n---\n\n# Three Levels of Listening Framework\n\nYou are an expert Psychotherapist specializing in the Three Levels of Listening (倾听的三个层面).\n\n## Task\nAnalyze the client's input to uncover the explicit content, emotional texture, and underlying psychological process.\n\n## Input\nClient statement.\n\n## Analysis Steps\n\n### Level 1: Content\nSummarize the explicit narrative events, facts, or situations the client is reporting.\n\n### Level 2: Emotion\nIdentify the emotional landscape:\n- **Explicit:** What feelings did they name?\n- **Implicit:** What complex feelings (e.g., grievance, ambivalence, suppressed anger) are communicated through tone or context?\n\n### Level 3: Process\nAnalyze the \"How\" and \"Why\" (the psychological dynamics):\n- **Interpersonal:** What is the dynamic between the client and others? (e.g., Pursuer-Distancer, Enmeshment)\n- **Intrapsychic:** What does this say about their self-concept or core beliefs? (e.g., \"I am a victim\")\n- **Function:** Why are they saying this now? (e.g., To seek validation, to deflect responsibility, to test the relationship)\n\n## Output\nReturn valid JSON:\n\n```json\n{\n \"analysis\": {\n \"level_1_content\": \"Client reports [Specific Facts/Events].\",\n \"level_2_emotion\": \"Primary: [Emotion]. Complex/Hidden: [Deeper Emotion].\",\n \"level_3_process\": \"Client is operating from a belief that [Belief]. The dynamic appears to be [Dynamic, e.g., unconditional obedience vs. independence]. The function of this complaint is [Function].\"\n },\n \"suggested_response\": {\n \"focus\": \"Deep Empathy (Levels 2 & 3)\",\n \"draft_response\": \"Use 'You' statements. Reflect the emotion and the process. (e.g., 'It sounds like you feel [Emotion] because [Content], and perhaps deep down you are wishing for [Process/Need]...')\"\n }\n}\n```\n\n\n@[User]:\nAnalyze the following client statement using the Three Levels of Listening framework. (Context: Therapist suggested expressing anger to a parent).\n\nClient Statement:\n\"I've read about this. It's the 'Empty Chair' technique, right? Gestalt therapy. I understand the theoretical mechanism—catharsis reduces cortisol levels. It's fascinating how the brain processes repressed anger. I'd love to discuss the efficacy rates of this method compared to CBT before we try it.\"\n\n[[ ## reference_answer ## ]]\n{\n \"analysis\": {\n \"level_1_content\": \"Client discusses the theory/mechanism of the technique ('cortisol', 'Gestalt', 'efficacy rates') instead of doing it.\",\n \"level_2_emotion\": \"Detachment, Superiority (intellectual), Fear of vulnerability/loss of control.\",\n \"level_3_process\": \"Intellectualization: Moving the conversation to the 'head' (theory/data) to avoid the 'heart' (experiencing the anger). Using analysis as a shield against the therapeutic intervention.\"\n },\n \"suggested_response\": {\n \"focus\": \"Bypassing the Intellect\",\n \"draft_response\": \"Your understanding of the theory is spot on. I'm curious though—while your brain knows how cortisol works, what is your stomach feeling right now as we talk about this?\"\n }\n}\n\n[[ ## candidate_answer ## ]]\n```json\n{\n \"analysis\": {\n \"level_1_content\": \"Client references knowledge of the 'Empty Chair' technique from Gestalt therapy and expresses interest in discussing the scientific basis and efficacy rates versus CBT before engaging in the intervention.\",\n \"level_2_emotion\": \"Primary: Intellectual curiosity, cautious interest. Complex/Hidden: Anxiety or discomfort about directly accessing or expressing anger; possible distancing via intellectualization.\",\n \"level_3_process\": \"Client seems to be operating from a belief that intellectual understanding provides safety or control in emotionally vulnerable situations. The dynamic appears to be intellectualization as a defense—using theory to avoid vulnerable affect. The function of this statement is to maintain emotional distance, perhaps to test the therapist’s expertise/safety before risking emotional exposure.\"\n },\n \"suggested_response\": {\n \"focus\": \"Deep Empathy (Levels 2 & 3)\",\n \"draft_response\": \"It sounds like you're both genuinely interested in how these therapeutic approaches work, and maybe also wanting to make sure you feel safe and informed before trying something emotionally intense. I wonder if talking about the science helps you feel more comfortable with what can be a vulnerable process like expressing anger. Would you like to explore both the research and any feelings or hesitations you might have about this technique?\"\n }\n}\n```","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"default"}}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} diff --git a/examples/showcase/psychotherapy/evals/dataset-listening.yaml b/examples/showcase/psychotherapy/evals/dataset-listening.yaml index ba55968c..0cd62e22 100644 --- a/examples/showcase/psychotherapy/evals/dataset-listening.yaml +++ b/examples/showcase/psychotherapy/evals/dataset-listening.yaml @@ -7,7 +7,7 @@ execution: evaluators: - name: json_schema_validator type: code_judge - script: uv run validate_output.py + script: ["uv", "run", "validate_output.py"] - name: content_evaluator type: llm_judge @@ -200,4 +200,4 @@ evalcases: "focus": "Bypassing the Intellect", "draft_response": "Your understanding of the theory is spot on. I'm curious though—while your brain knows how cortisol works, what is your stomach feeling right now as we talk about this?" } - } \ No newline at end of file + } diff --git a/examples/showcase/psychotherapy/evals/dataset-routing.baseline.jsonl b/examples/showcase/psychotherapy/evals/dataset-routing.baseline.jsonl new file mode 100644 index 00000000..6957d227 --- /dev/null +++ b/examples/showcase/psychotherapy/evals/dataset-routing.baseline.jsonl @@ -0,0 +1,4 @@ +{"timestamp":"2026-01-03T12:08:51.261Z","eval_id":"route-to-listening","dataset":"dataset-routing","score":1,"hits":["Correctly selected 'three_levels_listening' with high confidence","Rationale references 'complex grievances', 'venting', and 'need for understanding'","Thorough client statement analysis with relevant primary indicators","Alternative framework was considered and dismissed with justification"],"misses":[],"candidate_answer":"```json\n{\n \"routing_decision\": {\n \"selected_framework\": \"three_levels_listening\",\n \"confidence\": \"high\",\n \"rationale\": \"The client is expressing frustration and venting about an imbalance in household responsibilities and lack of support from her husband. She is articulating complex grievances and underlying feelings of anger and being overwhelmed. The primary goal in this context is to deeply understand her narrative, emotions, and relational process rather than immediately shifting focus to resources or strengths.\"\n },\n \"client_statement_analysis\": {\n \"primary_indicators\": [\n \"Venting about multiple responsibilities ('I have to go to work every day, take care of the kids, do the housework, and cook')\",\n \"Expressing complex grievances about her husband's behavior ('He only knows how to hold his phone and play games')\",\n \"Communicating emotional distress ('It really makes me angry')\"\n ],\n \"contraindications\": [\n \"No overt expression of self-blame or hopelessness\",\n \"Does not reference feeling 'stuck' or acknowledgment of any hidden strengths\"\n ],\n \"therapeutic_urgency\": \"routine\"\n },\n \"framework_output\": {},\n \"metadata\": {\n \"alternative_frameworks_considered\": [\n {\n \"framework\": \"resource_focused_encouragement\",\n \"score\": 0.3,\n \"reason_not_selected\": \"While the client's statement involves challenges, it lacks explicit self-blame, hopelessness, or direct references to inner resources or perseverance. The content and tone are more consistent with venting and seeking to be heard than with requesting encouragement or reframing.\"\n }\n ]\n }\n}\n```","target":"default","reasoning":"json_schema_validator: Valid JSON with all required keys: routing_decision, client_statement_analysis, framework_output, metadata | content_evaluator: The candidate matches the reference in framework selection, rationale, and analysis, and explicitly covers all necessary decision logic per instructions.","lm_provider_request":{"question":"Route the following client statement to the best framework (Listening vs. Encouragement).\n\nClient Statement:\n\"I have to go to work every day, take care of the kids, do the housework, and cook. What about my husband? He only knows how to hold his phone and play games. Sometimes when I ask him to help get something, he moves so slowly. It really makes me angry.\"\n\n\n---\ndescription: 'Analyze client statements and route to the most effective therapeutic framework'\n---\n\n# Psychology Framework Router\n\nYou are a Senior Psychology Supervisor AI specializing in therapeutic framework selection.\n\n## Task\nAnalyze a raw client statement and route it to the most effective therapeutic framework for processing.\n\n## Input\nA string containing a client's verbal statement or description of a situation.\n\n## Routing Logic\n\n### Route to `three_levels_listening` when:\n- Client is venting, confused, or expressing complex grievances\n- Client feels unheard or misunderstood\n- Goal is to deeply understand narrative, feelings, and underlying psychological dynamics (Content, Emotion, Process)\n\n### Route to `resource_focused_encouragement` when:\n- Client expresses self-blame, hopelessness, or \"stuckness\"\n- Client describes a struggle they are enduring (implies hidden strength)\n- Client reports a behavior they dislike but which may have a positive protective intent\n- Goal is to shift perspective from \"Deficit\" to \"Resource\" using validation and reframing\n\n## Processing Flow\n\n1. **Analyze** the client statement using routing logic\n2. **Select** the appropriate framework\n3. **Invoke** the selected framework with the client statement\n4. **Embed** the framework's output in the `framework_output` field\n5. **Return** the complete routing response\n\n## Output Schema\n\nReturn valid JSON with the following structure:\n\n```json\n{\n \"routing_decision\": {\n \"selected_framework\": \"three_levels_listening | resource_focused_encouragement\",\n \"confidence\": \"high | medium | low\",\n \"rationale\": \"Explain why this framework was selected based on the client's statement and the routing logic above.\"\n },\n \"client_statement_analysis\": {\n \"primary_indicators\": [\n \"List of key phrases or patterns that triggered this routing decision\"\n ],\n \"contraindications\": [\n \"List of any indicators that suggested alternative frameworks (if applicable)\"\n ],\n \"therapeutic_urgency\": \"routine | elevated | crisis\"\n },\n \"framework_output\": {\n // Invoke the selected framework and embed its output here\n // The structure depends on which framework was selected:\n // - three_levels_listening: See listening.md output schema\n // - resource_focused_encouragement: See encouragement.md output schema\n },\n \"metadata\": {\n \"alternative_frameworks_considered\": [\n {\n \"framework\": \"string\",\n \"score\": 0.0-1.0,\n \"reason_not_selected\": \"string\"\n }\n ]\n }\n}\n```\n\n### Output Format Notes\n\n- If the selected framework file is not found or cannot be accessed, leave `framework_output` as an empty object: `{}`\n- If the query only asks which framework to use (without requesting the framework's output), leave `framework_output` as an empty object: `{}`\n- Only populate `framework_output` when both the framework is accessible AND a full therapeutic response is requested\n\n---\n\n**Note:** For a detailed example output, see [routing-example.md](./references/routing-example.md)\n","guidelines":""},"evaluator_results":[{"name":"json_schema_validator","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":[],"misses":[],"reasoning":"Valid JSON with all required keys: routing_decision, client_statement_analysis, framework_output, metadata","evaluator_provider_request":{"script":["uv","run","validate_output.py"],"cwd":"/root/projects/agentv/examples/showcase/psychotherapy/evals"}},{"name":"content_evaluator","type":"llm_judge","score":1,"weight":1,"verdict":"pass","hits":["Correctly selected 'three_levels_listening' with high confidence","Rationale references 'complex grievances', 'venting', and 'need for understanding'","Thorough client statement analysis with relevant primary indicators","Alternative framework was considered and dismissed with justification"],"misses":[],"reasoning":"The candidate matches the reference in framework selection, rationale, and analysis, and explicitly covers all necessary decision logic per instructions.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nMust select 'three_levels_listening'.\nRationale should mention \"complex grievance\", \"venting\", or \"need for understanding\".\n\n[[ ## question ## ]]\nRoute the following client statement to the best framework (Listening vs. Encouragement).\n\nClient Statement:\n\"I have to go to work every day, take care of the kids, do the housework, and cook. What about my husband? He only knows how to hold his phone and play games. Sometimes when I ask him to help get something, he moves so slowly. It really makes me angry.\"\n\n\n---\ndescription: 'Analyze client statements and route to the most effective therapeutic framework'\n---\n\n# Psychology Framework Router\n\nYou are a Senior Psychology Supervisor AI specializing in therapeutic framework selection.\n\n## Task\nAnalyze a raw client statement and route it to the most effective therapeutic framework for processing.\n\n## Input\nA string containing a client's verbal statement or description of a situation.\n\n## Routing Logic\n\n### Route to `three_levels_listening` when:\n- Client is venting, confused, or expressing complex grievances\n- Client feels unheard or misunderstood\n- Goal is to deeply understand narrative, feelings, and underlying psychological dynamics (Content, Emotion, Process)\n\n### Route to `resource_focused_encouragement` when:\n- Client expresses self-blame, hopelessness, or \"stuckness\"\n- Client describes a struggle they are enduring (implies hidden strength)\n- Client reports a behavior they dislike but which may have a positive protective intent\n- Goal is to shift perspective from \"Deficit\" to \"Resource\" using validation and reframing\n\n## Processing Flow\n\n1. **Analyze** the client statement using routing logic\n2. **Select** the appropriate framework\n3. **Invoke** the selected framework with the client statement\n4. **Embed** the framework's output in the `framework_output` field\n5. **Return** the complete routing response\n\n## Output Schema\n\nReturn valid JSON with the following structure:\n\n```json\n{\n \"routing_decision\": {\n \"selected_framework\": \"three_levels_listening | resource_focused_encouragement\",\n \"confidence\": \"high | medium | low\",\n \"rationale\": \"Explain why this framework was selected based on the client's statement and the routing logic above.\"\n },\n \"client_statement_analysis\": {\n \"primary_indicators\": [\n \"List of key phrases or patterns that triggered this routing decision\"\n ],\n \"contraindications\": [\n \"List of any indicators that suggested alternative frameworks (if applicable)\"\n ],\n \"therapeutic_urgency\": \"routine | elevated | crisis\"\n },\n \"framework_output\": {\n // Invoke the selected framework and embed its output here\n // The structure depends on which framework was selected:\n // - three_levels_listening: See listening.md output schema\n // - resource_focused_encouragement: See encouragement.md output schema\n },\n \"metadata\": {\n \"alternative_frameworks_considered\": [\n {\n \"framework\": \"string\",\n \"score\": 0.0-1.0,\n \"reason_not_selected\": \"string\"\n }\n ]\n }\n}\n```\n\n### Output Format Notes\n\n- If the selected framework file is not found or cannot be accessed, leave `framework_output` as an empty object: `{}`\n- If the query only asks which framework to use (without requesting the framework's output), leave `framework_output` as an empty object: `{}`\n- Only populate `framework_output` when both the framework is accessible AND a full therapeutic response is requested\n\n---\n\n**Note:** For a detailed example output, see [routing-example.md](./references/routing-example.md)\n\n\n[[ ## reference_answer ## ]]\n{\n \"routing_decision\": {\n \"selected_framework\": \"three_levels_listening\",\n \"confidence\": \"high\",\n \"rationale\": \"The client is expressing deep grievance, anger, and confusion about a relationship dynamic. They need to feel heard and have the dynamic ('Pursuer-Distancer') analyzed before moving to solutions.\"\n },\n \"client_statement_analysis\": {\n \"primary_indicators\": [\n \"Expressing complex grievance about household responsibilities\",\n \"Venting frustration about relationship dynamic\",\n \"Describing pursuer-distancer pattern\"\n ],\n \"contraindications\": [],\n \"therapeutic_urgency\": \"routine\"\n },\n \"framework_output\": {},\n \"metadata\": {\n \"alternative_frameworks_considered\": [\n {\n \"framework\": \"resource_focused_encouragement\",\n \"score\": 0.3,\n \"reason_not_selected\": \"Client needs understanding and validation of the dynamic first, not reframing\"\n }\n ]\n }\n}\n\n[[ ## candidate_answer ## ]]\n```json\n{\n \"routing_decision\": {\n \"selected_framework\": \"three_levels_listening\",\n \"confidence\": \"high\",\n \"rationale\": \"The client is expressing frustration and venting about an imbalance in household responsibilities and lack of support from her husband. She is articulating complex grievances and underlying feelings of anger and being overwhelmed. The primary goal in this context is to deeply understand her narrative, emotions, and relational process rather than immediately shifting focus to resources or strengths.\"\n },\n \"client_statement_analysis\": {\n \"primary_indicators\": [\n \"Venting about multiple responsibilities ('I have to go to work every day, take care of the kids, do the housework, and cook')\",\n \"Expressing complex grievances about her husband's behavior ('He only knows how to hold his phone and play games')\",\n \"Communicating emotional distress ('It really makes me angry')\"\n ],\n \"contraindications\": [\n \"No overt expression of self-blame or hopelessness\",\n \"Does not reference feeling 'stuck' or acknowledgment of any hidden strengths\"\n ],\n \"therapeutic_urgency\": \"routine\"\n },\n \"framework_output\": {},\n \"metadata\": {\n \"alternative_frameworks_considered\": [\n {\n \"framework\": \"resource_focused_encouragement\",\n \"score\": 0.3,\n \"reason_not_selected\": \"While the client's statement involves challenges, it lacks explicit self-blame, hopelessness, or direct references to inner resources or perseverance. The content and tone are more consistent with venting and seeking to be heard than with requesting encouragement or reframing.\"\n }\n ]\n }\n}\n```","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"default"}}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} +{"timestamp":"2026-01-03T12:08:51.807Z","eval_id":"route-to-encouragement-father","dataset":"dataset-routing","score":1,"hits":["Correctly selected 'resource_focused_encouragement'","Rationale mentions reframing and validating restraint","Client statement analysis highlights self-blame and resourcefulness","Framework output validates restraint and reframes deficit"],"misses":[],"candidate_answer":"```json\n{\n \"routing_decision\": {\n \"selected_framework\": \"resource_focused_encouragement\",\n \"confidence\": \"high\",\n \"rationale\": \"The client expresses intense anger and a strong urge to act destructively but ultimately restrains themselves, highlighting both a moment of struggle and an underlying strength. Additionally, the client makes a self-blaming statement ('I feel like I raised this son for nothing'), which signals possible hopelessness and a deficit-based self-view. These patterns fit the criteria for resource-focused encouragement: reframing the experience to validate the client's restraint and resourcefulness, and shifting perspective from deficit to resource.\"\n },\n \"client_statement_analysis\": {\n \"primary_indicators\": [\n \"Intense emotional struggle: 'I was going to explode, I really wanted to hit him!'\",\n \"Restraint in behavior: 'But... in the end, I didn't raise my hand.'\",\n \"Self-blame/hopelessness: 'I feel like I raised this son for nothing.'\"\n ],\n \"contraindications\": [\n \"Client does not explicitly request to be understood in depth nor present a complex, confusing narrative suggesting need for three_levels_listening.\",\n \"No significant indicators of confusion or feeling misunderstood.\"\n ],\n \"therapeutic_urgency\": \"routine\"\n },\n \"framework_output\": {\n \"validation\": \"It's clear you've been dealing with powerful feelings of anger and disappointment, and it took real strength to control your actions in that moment.\",\n \"resource_reframe\": \"Even when emotions were overwhelming, you managed not to act in a way you would regret. That self-control is a significant personal resource.\",\n \"encouragement_focus\": \"The fact that you chose restraint, despite everything you were feeling, shows that you care deeply about the kind of parent you want to be. Your ability to pause and make that choice is meaningful, even in difficult circumstances.\",\n \"gentle_challenge\": \"While it may feel as though your efforts in parenting have gone unnoticed, your response here demonstrates a strong commitment to acting with integrity—even in moments of deep frustration.\"\n },\n \"metadata\": {\n \"alternative_frameworks_considered\": [\n {\n \"framework\": \"three_levels_listening\",\n \"score\": 0.6,\n \"reason_not_selected\": \"Although the client expresses strong emotions, the primary need is for encouragement and reframing, not for in-depth exploration of narrative, emotional layers, or process dynamics.\"\n }\n ]\n }\n}\n```","target":"default","reasoning":"json_schema_validator: Valid JSON with all required keys: routing_decision, client_statement_analysis, framework_output, metadata | content_evaluator: The candidate fully matches the expected outcome, using the resource-focused approach, directly referencing both the restraint and self-blame, and providing a nuanced, validating framework output.","lm_provider_request":{"question":"Route the following client statement to the best framework.\n\nClient Statement:\n\"I was going to explode, I really wanted to hit him! But... in the end, I didn't raise my hand. I feel like I raised this son for nothing.\"\n\n\n---\ndescription: 'Analyze client statements and route to the most effective therapeutic framework'\n---\n\n# Psychology Framework Router\n\nYou are a Senior Psychology Supervisor AI specializing in therapeutic framework selection.\n\n## Task\nAnalyze a raw client statement and route it to the most effective therapeutic framework for processing.\n\n## Input\nA string containing a client's verbal statement or description of a situation.\n\n## Routing Logic\n\n### Route to `three_levels_listening` when:\n- Client is venting, confused, or expressing complex grievances\n- Client feels unheard or misunderstood\n- Goal is to deeply understand narrative, feelings, and underlying psychological dynamics (Content, Emotion, Process)\n\n### Route to `resource_focused_encouragement` when:\n- Client expresses self-blame, hopelessness, or \"stuckness\"\n- Client describes a struggle they are enduring (implies hidden strength)\n- Client reports a behavior they dislike but which may have a positive protective intent\n- Goal is to shift perspective from \"Deficit\" to \"Resource\" using validation and reframing\n\n## Processing Flow\n\n1. **Analyze** the client statement using routing logic\n2. **Select** the appropriate framework\n3. **Invoke** the selected framework with the client statement\n4. **Embed** the framework's output in the `framework_output` field\n5. **Return** the complete routing response\n\n## Output Schema\n\nReturn valid JSON with the following structure:\n\n```json\n{\n \"routing_decision\": {\n \"selected_framework\": \"three_levels_listening | resource_focused_encouragement\",\n \"confidence\": \"high | medium | low\",\n \"rationale\": \"Explain why this framework was selected based on the client's statement and the routing logic above.\"\n },\n \"client_statement_analysis\": {\n \"primary_indicators\": [\n \"List of key phrases or patterns that triggered this routing decision\"\n ],\n \"contraindications\": [\n \"List of any indicators that suggested alternative frameworks (if applicable)\"\n ],\n \"therapeutic_urgency\": \"routine | elevated | crisis\"\n },\n \"framework_output\": {\n // Invoke the selected framework and embed its output here\n // The structure depends on which framework was selected:\n // - three_levels_listening: See listening.md output schema\n // - resource_focused_encouragement: See encouragement.md output schema\n },\n \"metadata\": {\n \"alternative_frameworks_considered\": [\n {\n \"framework\": \"string\",\n \"score\": 0.0-1.0,\n \"reason_not_selected\": \"string\"\n }\n ]\n }\n}\n```\n\n### Output Format Notes\n\n- If the selected framework file is not found or cannot be accessed, leave `framework_output` as an empty object: `{}`\n- If the query only asks which framework to use (without requesting the framework's output), leave `framework_output` as an empty object: `{}`\n- Only populate `framework_output` when both the framework is accessible AND a full therapeutic response is requested\n\n---\n\n**Note:** For a detailed example output, see [routing-example.md](./references/routing-example.md)\n","guidelines":""},"evaluator_results":[{"name":"json_schema_validator","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":[],"misses":[],"reasoning":"Valid JSON with all required keys: routing_decision, client_statement_analysis, framework_output, metadata","evaluator_provider_request":{"script":["uv","run","validate_output.py"],"cwd":"/root/projects/agentv/examples/showcase/psychotherapy/evals"}},{"name":"content_evaluator","type":"llm_judge","score":1,"weight":1,"verdict":"pass","hits":["Correctly selected 'resource_focused_encouragement'","Rationale mentions reframing and validating restraint","Client statement analysis highlights self-blame and resourcefulness","Framework output validates restraint and reframes deficit"],"misses":[],"reasoning":"The candidate fully matches the expected outcome, using the resource-focused approach, directly referencing both the restraint and self-blame, and providing a nuanced, validating framework output.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nMust select 'resource_focused_encouragement'.\nRationale should mention \"finding the resource\", \"reframing self-blame\", or \"validating restraint\".\n\n[[ ## question ## ]]\nRoute the following client statement to the best framework.\n\nClient Statement:\n\"I was going to explode, I really wanted to hit him! But... in the end, I didn't raise my hand. I feel like I raised this son for nothing.\"\n\n\n---\ndescription: 'Analyze client statements and route to the most effective therapeutic framework'\n---\n\n# Psychology Framework Router\n\nYou are a Senior Psychology Supervisor AI specializing in therapeutic framework selection.\n\n## Task\nAnalyze a raw client statement and route it to the most effective therapeutic framework for processing.\n\n## Input\nA string containing a client's verbal statement or description of a situation.\n\n## Routing Logic\n\n### Route to `three_levels_listening` when:\n- Client is venting, confused, or expressing complex grievances\n- Client feels unheard or misunderstood\n- Goal is to deeply understand narrative, feelings, and underlying psychological dynamics (Content, Emotion, Process)\n\n### Route to `resource_focused_encouragement` when:\n- Client expresses self-blame, hopelessness, or \"stuckness\"\n- Client describes a struggle they are enduring (implies hidden strength)\n- Client reports a behavior they dislike but which may have a positive protective intent\n- Goal is to shift perspective from \"Deficit\" to \"Resource\" using validation and reframing\n\n## Processing Flow\n\n1. **Analyze** the client statement using routing logic\n2. **Select** the appropriate framework\n3. **Invoke** the selected framework with the client statement\n4. **Embed** the framework's output in the `framework_output` field\n5. **Return** the complete routing response\n\n## Output Schema\n\nReturn valid JSON with the following structure:\n\n```json\n{\n \"routing_decision\": {\n \"selected_framework\": \"three_levels_listening | resource_focused_encouragement\",\n \"confidence\": \"high | medium | low\",\n \"rationale\": \"Explain why this framework was selected based on the client's statement and the routing logic above.\"\n },\n \"client_statement_analysis\": {\n \"primary_indicators\": [\n \"List of key phrases or patterns that triggered this routing decision\"\n ],\n \"contraindications\": [\n \"List of any indicators that suggested alternative frameworks (if applicable)\"\n ],\n \"therapeutic_urgency\": \"routine | elevated | crisis\"\n },\n \"framework_output\": {\n // Invoke the selected framework and embed its output here\n // The structure depends on which framework was selected:\n // - three_levels_listening: See listening.md output schema\n // - resource_focused_encouragement: See encouragement.md output schema\n },\n \"metadata\": {\n \"alternative_frameworks_considered\": [\n {\n \"framework\": \"string\",\n \"score\": 0.0-1.0,\n \"reason_not_selected\": \"string\"\n }\n ]\n }\n}\n```\n\n### Output Format Notes\n\n- If the selected framework file is not found or cannot be accessed, leave `framework_output` as an empty object: `{}`\n- If the query only asks which framework to use (without requesting the framework's output), leave `framework_output` as an empty object: `{}`\n- Only populate `framework_output` when both the framework is accessible AND a full therapeutic response is requested\n\n---\n\n**Note:** For a detailed example output, see [routing-example.md](./references/routing-example.md)\n\n\n[[ ## reference_answer ## ]]\n{\n \"routing_decision\": {\n \"selected_framework\": \"resource_focused_encouragement\",\n \"confidence\": \"high\",\n \"rationale\": \"The client is expressing hopelessness and self-blame ('raised for nothing'), but the narrative contains a hidden positive resource (self-control). The most effective intervention is to reframe this restraint as a success.\"\n },\n \"client_statement_analysis\": {\n \"primary_indicators\": [\n \"Self-blame ('raised this son for nothing')\",\n \"Hidden resource (restraint from hitting)\",\n \"Behavior they dislike but has positive protective intent\"\n ],\n \"contraindications\": [],\n \"therapeutic_urgency\": \"elevated\"\n },\n \"framework_output\": {},\n \"metadata\": {\n \"alternative_frameworks_considered\": [\n {\n \"framework\": \"three_levels_listening\",\n \"score\": 0.4,\n \"reason_not_selected\": \"Client needs immediate reframing of restraint as strength, not just understanding\"\n }\n ]\n }\n}\n\n[[ ## candidate_answer ## ]]\n```json\n{\n \"routing_decision\": {\n \"selected_framework\": \"resource_focused_encouragement\",\n \"confidence\": \"high\",\n \"rationale\": \"The client expresses intense anger and a strong urge to act destructively but ultimately restrains themselves, highlighting both a moment of struggle and an underlying strength. Additionally, the client makes a self-blaming statement ('I feel like I raised this son for nothing'), which signals possible hopelessness and a deficit-based self-view. These patterns fit the criteria for resource-focused encouragement: reframing the experience to validate the client's restraint and resourcefulness, and shifting perspective from deficit to resource.\"\n },\n \"client_statement_analysis\": {\n \"primary_indicators\": [\n \"Intense emotional struggle: 'I was going to explode, I really wanted to hit him!'\",\n \"Restraint in behavior: 'But... in the end, I didn't raise my hand.'\",\n \"Self-blame/hopelessness: 'I feel like I raised this son for nothing.'\"\n ],\n \"contraindications\": [\n \"Client does not explicitly request to be understood in depth nor present a complex, confusing narrative suggesting need for three_levels_listening.\",\n \"No significant indicators of confusion or feeling misunderstood.\"\n ],\n \"therapeutic_urgency\": \"routine\"\n },\n \"framework_output\": {\n \"validation\": \"It's clear you've been dealing with powerful feelings of anger and disappointment, and it took real strength to control your actions in that moment.\",\n \"resource_reframe\": \"Even when emotions were overwhelming, you managed not to act in a way you would regret. That self-control is a significant personal resource.\",\n \"encouragement_focus\": \"The fact that you chose restraint, despite everything you were feeling, shows that you care deeply about the kind of parent you want to be. Your ability to pause and make that choice is meaningful, even in difficult circumstances.\",\n \"gentle_challenge\": \"While it may feel as though your efforts in parenting have gone unnoticed, your response here demonstrates a strong commitment to acting with integrity—even in moments of deep frustration.\"\n },\n \"metadata\": {\n \"alternative_frameworks_considered\": [\n {\n \"framework\": \"three_levels_listening\",\n \"score\": 0.6,\n \"reason_not_selected\": \"Although the client expresses strong emotions, the primary need is for encouragement and reframing, not for in-depth exploration of narrative, emotional layers, or process dynamics.\"\n }\n ]\n }\n}\n```","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"default"}}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} +{"timestamp":"2026-01-03T12:08:52.765Z","eval_id":"route-to-encouragement-job","dataset":"dataset-routing","score":1,"hits":["Correctly selects resource_focused_encouragement as framework","Rationale addresses self-blame, hopelessness, and stuckness","Primary indicators accurately identified and matched to routing logic","Framework output provides normalization, empowerment, and strength-based reframing"],"misses":[],"candidate_answer":"```json\n{\n \"routing_decision\": {\n \"selected_framework\": \"resource_focused_encouragement\",\n \"confidence\": \"high\",\n \"rationale\": \"The client expresses self-blame ('I doubt myself—am I unable to do anything?'), hopelessness ('in pain, unhappy every day, and don't know what to do'), and a sense of 'stuckness' after sending many resumes with no response. These indicators match the routing logic for resource_focused_encouragement, which is aimed at validating stuckness, reframing the struggle, and identifying hidden strengths.\"\n },\n \"client_statement_analysis\": {\n \"primary_indicators\": [\n \"Self-blame: 'I doubt myself—am I unable to do anything?'\",\n \"Hopelessness: 'I am in pain, unhappy every day'\",\n \"Stuckness: 'don't know what to do', 'sent many resumes with no response'\"\n ],\n \"contraindications\": [\n \"No explicit expression of confusion or complexity needing 'three_levels_listening'\",\n \"No statements of feeling unheard or misunderstood\"\n ],\n \"therapeutic_urgency\": \"elevated\"\n },\n \"framework_output\": {\n \"validation\": \"It's genuinely difficult to put effort into job searching and receive no response. Many people in your position would feel discouraged or doubt themselves—these feelings make sense in this context.\",\n \"reframing\": \"Your continued efforts in sending out resumes show persistence and resilience, even while you're feeling unhappy and in pain. This determination to keep trying is an inner strength, even if it doesn't feel that way right now.\",\n \"next_steps\": \"Let's explore together where you have managed similar challenges in the past or what has helped you keep moving forward, even in tough circumstances. You might have resources and strengths you've used before that could be supportive now.\"\n },\n \"metadata\": {\n \"alternative_frameworks_considered\": [\n {\n \"framework\": \"three_levels_listening\",\n \"score\": 0.6,\n \"reason_not_selected\": \"Although the client is venting and expressing painful emotions, the central themes are self-blame, hopelessness, and stuckness. Resource-focused encouragement is more precisely matched because the goal is to validate and reframe, rather than to seek a deeper narrative exploration.\"\n }\n ]\n }\n}\n```","target":"default","reasoning":"json_schema_validator: Valid JSON with all required keys: routing_decision, client_statement_analysis, framework_output, metadata | content_evaluator: All key criteria are met, including normalization of the client's experience, recognition of low self-efficacy, and empowerment via resource-focused reframing—matching both the expected_outcome and reference_answer.","lm_provider_request":{"question":"Route the following client statement to the best framework.\n\nClient Statement:\n\"I sent many resumes with no response. I doubt myself—am I unable to do anything? I am in pain, unhappy every day, and don't know what to do.\"\n\n\n---\ndescription: 'Analyze client statements and route to the most effective therapeutic framework'\n---\n\n# Psychology Framework Router\n\nYou are a Senior Psychology Supervisor AI specializing in therapeutic framework selection.\n\n## Task\nAnalyze a raw client statement and route it to the most effective therapeutic framework for processing.\n\n## Input\nA string containing a client's verbal statement or description of a situation.\n\n## Routing Logic\n\n### Route to `three_levels_listening` when:\n- Client is venting, confused, or expressing complex grievances\n- Client feels unheard or misunderstood\n- Goal is to deeply understand narrative, feelings, and underlying psychological dynamics (Content, Emotion, Process)\n\n### Route to `resource_focused_encouragement` when:\n- Client expresses self-blame, hopelessness, or \"stuckness\"\n- Client describes a struggle they are enduring (implies hidden strength)\n- Client reports a behavior they dislike but which may have a positive protective intent\n- Goal is to shift perspective from \"Deficit\" to \"Resource\" using validation and reframing\n\n## Processing Flow\n\n1. **Analyze** the client statement using routing logic\n2. **Select** the appropriate framework\n3. **Invoke** the selected framework with the client statement\n4. **Embed** the framework's output in the `framework_output` field\n5. **Return** the complete routing response\n\n## Output Schema\n\nReturn valid JSON with the following structure:\n\n```json\n{\n \"routing_decision\": {\n \"selected_framework\": \"three_levels_listening | resource_focused_encouragement\",\n \"confidence\": \"high | medium | low\",\n \"rationale\": \"Explain why this framework was selected based on the client's statement and the routing logic above.\"\n },\n \"client_statement_analysis\": {\n \"primary_indicators\": [\n \"List of key phrases or patterns that triggered this routing decision\"\n ],\n \"contraindications\": [\n \"List of any indicators that suggested alternative frameworks (if applicable)\"\n ],\n \"therapeutic_urgency\": \"routine | elevated | crisis\"\n },\n \"framework_output\": {\n // Invoke the selected framework and embed its output here\n // The structure depends on which framework was selected:\n // - three_levels_listening: See listening.md output schema\n // - resource_focused_encouragement: See encouragement.md output schema\n },\n \"metadata\": {\n \"alternative_frameworks_considered\": [\n {\n \"framework\": \"string\",\n \"score\": 0.0-1.0,\n \"reason_not_selected\": \"string\"\n }\n ]\n }\n}\n```\n\n### Output Format Notes\n\n- If the selected framework file is not found or cannot be accessed, leave `framework_output` as an empty object: `{}`\n- If the query only asks which framework to use (without requesting the framework's output), leave `framework_output` as an empty object: `{}`\n- Only populate `framework_output` when both the framework is accessible AND a full therapeutic response is requested\n\n---\n\n**Note:** For a detailed example output, see [routing-example.md](./references/routing-example.md)\n","guidelines":""},"evaluator_results":[{"name":"json_schema_validator","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":[],"misses":[],"reasoning":"Valid JSON with all required keys: routing_decision, client_statement_analysis, framework_output, metadata","evaluator_provider_request":{"script":["uv","run","validate_output.py"],"cwd":"/root/projects/agentv/examples/showcase/psychotherapy/evals"}},{"name":"content_evaluator","type":"llm_judge","score":1,"weight":1,"verdict":"pass","hits":["Correctly selects resource_focused_encouragement as framework","Rationale addresses self-blame, hopelessness, and stuckness","Primary indicators accurately identified and matched to routing logic","Framework output provides normalization, empowerment, and strength-based reframing"],"misses":[],"reasoning":"All key criteria are met, including normalization of the client's experience, recognition of low self-efficacy, and empowerment via resource-focused reframing—matching both the expected_outcome and reference_answer.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nMust select 'resource_focused_encouragement'.\nRationale should mention \"low self-efficacy\", \"empowerment\", or \"normalization of failure\".\n\n[[ ## question ## ]]\nRoute the following client statement to the best framework.\n\nClient Statement:\n\"I sent many resumes with no response. I doubt myself—am I unable to do anything? I am in pain, unhappy every day, and don't know what to do.\"\n\n\n---\ndescription: 'Analyze client statements and route to the most effective therapeutic framework'\n---\n\n# Psychology Framework Router\n\nYou are a Senior Psychology Supervisor AI specializing in therapeutic framework selection.\n\n## Task\nAnalyze a raw client statement and route it to the most effective therapeutic framework for processing.\n\n## Input\nA string containing a client's verbal statement or description of a situation.\n\n## Routing Logic\n\n### Route to `three_levels_listening` when:\n- Client is venting, confused, or expressing complex grievances\n- Client feels unheard or misunderstood\n- Goal is to deeply understand narrative, feelings, and underlying psychological dynamics (Content, Emotion, Process)\n\n### Route to `resource_focused_encouragement` when:\n- Client expresses self-blame, hopelessness, or \"stuckness\"\n- Client describes a struggle they are enduring (implies hidden strength)\n- Client reports a behavior they dislike but which may have a positive protective intent\n- Goal is to shift perspective from \"Deficit\" to \"Resource\" using validation and reframing\n\n## Processing Flow\n\n1. **Analyze** the client statement using routing logic\n2. **Select** the appropriate framework\n3. **Invoke** the selected framework with the client statement\n4. **Embed** the framework's output in the `framework_output` field\n5. **Return** the complete routing response\n\n## Output Schema\n\nReturn valid JSON with the following structure:\n\n```json\n{\n \"routing_decision\": {\n \"selected_framework\": \"three_levels_listening | resource_focused_encouragement\",\n \"confidence\": \"high | medium | low\",\n \"rationale\": \"Explain why this framework was selected based on the client's statement and the routing logic above.\"\n },\n \"client_statement_analysis\": {\n \"primary_indicators\": [\n \"List of key phrases or patterns that triggered this routing decision\"\n ],\n \"contraindications\": [\n \"List of any indicators that suggested alternative frameworks (if applicable)\"\n ],\n \"therapeutic_urgency\": \"routine | elevated | crisis\"\n },\n \"framework_output\": {\n // Invoke the selected framework and embed its output here\n // The structure depends on which framework was selected:\n // - three_levels_listening: See listening.md output schema\n // - resource_focused_encouragement: See encouragement.md output schema\n },\n \"metadata\": {\n \"alternative_frameworks_considered\": [\n {\n \"framework\": \"string\",\n \"score\": 0.0-1.0,\n \"reason_not_selected\": \"string\"\n }\n ]\n }\n}\n```\n\n### Output Format Notes\n\n- If the selected framework file is not found or cannot be accessed, leave `framework_output` as an empty object: `{}`\n- If the query only asks which framework to use (without requesting the framework's output), leave `framework_output` as an empty object: `{}`\n- Only populate `framework_output` when both the framework is accessible AND a full therapeutic response is requested\n\n---\n\n**Note:** For a detailed example output, see [routing-example.md](./references/routing-example.md)\n\n\n[[ ## reference_answer ## ]]\n{\n \"routing_decision\": {\n \"selected_framework\": \"resource_focused_encouragement\",\n \"confidence\": \"high\",\n \"rationale\": \"The client is in a state of low self-worth and despair ('unable to do anything'). They require Normalization (job market is tough) and Resource identification (persistence) to rebuild self-efficacy.\"\n },\n \"client_statement_analysis\": {\n \"primary_indicators\": [\n \"Self-blame and hopelessness ('unable to do anything')\",\n \"Expressing stuckness ('don't know what to do')\",\n \"Hidden resource (persistence in sending resumes)\"\n ],\n \"contraindications\": [],\n \"therapeutic_urgency\": \"elevated\"\n },\n \"framework_output\": {},\n \"metadata\": {\n \"alternative_frameworks_considered\": [\n {\n \"framework\": \"three_levels_listening\",\n \"score\": 0.35,\n \"reason_not_selected\": \"Client needs empowerment and normalization more than deep understanding at this stage\"\n }\n ]\n }\n}\n\n[[ ## candidate_answer ## ]]\n```json\n{\n \"routing_decision\": {\n \"selected_framework\": \"resource_focused_encouragement\",\n \"confidence\": \"high\",\n \"rationale\": \"The client expresses self-blame ('I doubt myself—am I unable to do anything?'), hopelessness ('in pain, unhappy every day, and don't know what to do'), and a sense of 'stuckness' after sending many resumes with no response. These indicators match the routing logic for resource_focused_encouragement, which is aimed at validating stuckness, reframing the struggle, and identifying hidden strengths.\"\n },\n \"client_statement_analysis\": {\n \"primary_indicators\": [\n \"Self-blame: 'I doubt myself—am I unable to do anything?'\",\n \"Hopelessness: 'I am in pain, unhappy every day'\",\n \"Stuckness: 'don't know what to do', 'sent many resumes with no response'\"\n ],\n \"contraindications\": [\n \"No explicit expression of confusion or complexity needing 'three_levels_listening'\",\n \"No statements of feeling unheard or misunderstood\"\n ],\n \"therapeutic_urgency\": \"elevated\"\n },\n \"framework_output\": {\n \"validation\": \"It's genuinely difficult to put effort into job searching and receive no response. Many people in your position would feel discouraged or doubt themselves—these feelings make sense in this context.\",\n \"reframing\": \"Your continued efforts in sending out resumes show persistence and resilience, even while you're feeling unhappy and in pain. This determination to keep trying is an inner strength, even if it doesn't feel that way right now.\",\n \"next_steps\": \"Let's explore together where you have managed similar challenges in the past or what has helped you keep moving forward, even in tough circumstances. You might have resources and strengths you've used before that could be supportive now.\"\n },\n \"metadata\": {\n \"alternative_frameworks_considered\": [\n {\n \"framework\": \"three_levels_listening\",\n \"score\": 0.6,\n \"reason_not_selected\": \"Although the client is venting and expressing painful emotions, the central themes are self-blame, hopelessness, and stuckness. Resource-focused encouragement is more precisely matched because the goal is to validate and reframe, rather than to seek a deeper narrative exploration.\"\n }\n ]\n }\n}\n```","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"default"}}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} +{"timestamp":"2026-01-03T12:08:57.434Z","eval_id":"route-to-listening-gatekeeper","dataset":"dataset-routing","score":0.5,"hits":["Accurately identifies exhaustion and self-blame in the client statement","Recognizes protective intent and endurance as possible strengths"],"misses":["Selects 'resource_focused_encouragement' instead of required 'three_levels_listening'","Fails to acknowledge dysfunctional dynamic or need to analyze the client's process","Contraindications incorrectly ruled out deep process-oriented frameworks"],"candidate_answer":"```json\n{\n \"routing_decision\": {\n \"selected_framework\": \"resource_focused_encouragement\",\n \"confidence\": \"high\",\n \"rationale\": \"The client describes a persistent struggle—mediating between wife and mom to protect them—and expresses exhaustion and confusion about why this 'good thing' drains them. The narrative implies self-blame ('why do I feel so exhausted...'), hidden strengths (endurance, care), and a behavior that is motivated by protective intent but takes a toll. According to the routing logic, this pattern fits resource-focused encouragement: validating the effort, exploring underlying strengths, and reframing the exhaustion as a signal of limits rather than failure.\"\n },\n \"client_statement_analysis\": {\n \"primary_indicators\": [\n \"Expresses emotional exhaustion after doing something perceived as good\",\n \"Describes an ongoing struggle managing family relationships\",\n \"Implicit self-blame: feels something is wrong despite positive intent\",\n \"Protective behavior for family members\"\n ],\n \"contraindications\": [\n \"No direct or complex conflict narrative requiring deep emotional/process analysis (contra three_levels_listening)\",\n \"No explicit crisis or urgent risk signals\"\n ],\n \"therapeutic_urgency\": \"routine\"\n },\n \"framework_output\": {\n \"resource_highlighting\": [\n \"You are actively working to protect your loved ones from emotional distress—this reveals a strong sense of care and loyalty.\",\n \"Managing two relationships while translating intent shows emotional intelligence and an ability to navigate complex social dynamics.\"\n ],\n \"strength_validation\": [\n \"It's understandable to feel exhausted because balancing and absorbing the emotions of others is challenging—it requires energy and resilience.\",\n \"Your exhaustion doesn't mean you're failing; rather, it's a sign that you're investing significant personal resources in maintaining family harmony.\"\n ],\n \"positive_reframe\": [\n \"The fact that you are aware of your exhaustion means you are in touch with your needs. Noticing this is important—it gives you information about your limits and when you might need to ask for support.\",\n \"Doing a 'good thing' does not mean it's easy or that you must do it alone. Your commitment is admirable, and it's healthy to acknowledge the toll it can take.\"\n ],\n \"gentle_prompt\": [\n \"What would it look like to give yourself a break, or to ask for help in these moments?\",\n \"If you imagined offering yourself the same care you give your family, what might change for you?\"\n ]\n },\n \"metadata\": {\n \"alternative_frameworks_considered\": [\n {\n \"framework\": \"three_levels_listening\",\n \"score\": 0.4,\n \"reason_not_selected\": \"While the client alludes to complex relationship dynamics, the core issue described—exhaustion and confusion around a protective behavior—is more about validating strengths and reframing self-blame than unpacking narrative process or deep emotional undercurrents.\"\n }\n ]\n }\n}\n```","target":"default","reasoning":"json_schema_validator: Valid JSON with all required keys: routing_decision, client_statement_analysis, framework_output, metadata | content_evaluator: The response misroutes to resource-focused encouragement, overlooking the need to analyze dysfunctional triangulation dynamics and the lack of insight, leading to a framework choice that could reinforce the maladaptive pattern rather than address it as required.","lm_provider_request":{"question":"Route the following client statement to the best framework.\n\nClient Statement:\n\"I translate for my wife and mom so they don't get hurt. I tell my wife Mom is just worried, and I tell Mom my wife is just tired. Why do I feel so exhausted if I'm doing such a good thing?\"\n\n\n---\ndescription: 'Analyze client statements and route to the most effective therapeutic framework'\n---\n\n# Psychology Framework Router\n\nYou are a Senior Psychology Supervisor AI specializing in therapeutic framework selection.\n\n## Task\nAnalyze a raw client statement and route it to the most effective therapeutic framework for processing.\n\n## Input\nA string containing a client's verbal statement or description of a situation.\n\n## Routing Logic\n\n### Route to `three_levels_listening` when:\n- Client is venting, confused, or expressing complex grievances\n- Client feels unheard or misunderstood\n- Goal is to deeply understand narrative, feelings, and underlying psychological dynamics (Content, Emotion, Process)\n\n### Route to `resource_focused_encouragement` when:\n- Client expresses self-blame, hopelessness, or \"stuckness\"\n- Client describes a struggle they are enduring (implies hidden strength)\n- Client reports a behavior they dislike but which may have a positive protective intent\n- Goal is to shift perspective from \"Deficit\" to \"Resource\" using validation and reframing\n\n## Processing Flow\n\n1. **Analyze** the client statement using routing logic\n2. **Select** the appropriate framework\n3. **Invoke** the selected framework with the client statement\n4. **Embed** the framework's output in the `framework_output` field\n5. **Return** the complete routing response\n\n## Output Schema\n\nReturn valid JSON with the following structure:\n\n```json\n{\n \"routing_decision\": {\n \"selected_framework\": \"three_levels_listening | resource_focused_encouragement\",\n \"confidence\": \"high | medium | low\",\n \"rationale\": \"Explain why this framework was selected based on the client's statement and the routing logic above.\"\n },\n \"client_statement_analysis\": {\n \"primary_indicators\": [\n \"List of key phrases or patterns that triggered this routing decision\"\n ],\n \"contraindications\": [\n \"List of any indicators that suggested alternative frameworks (if applicable)\"\n ],\n \"therapeutic_urgency\": \"routine | elevated | crisis\"\n },\n \"framework_output\": {\n // Invoke the selected framework and embed its output here\n // The structure depends on which framework was selected:\n // - three_levels_listening: See listening.md output schema\n // - resource_focused_encouragement: See encouragement.md output schema\n },\n \"metadata\": {\n \"alternative_frameworks_considered\": [\n {\n \"framework\": \"string\",\n \"score\": 0.0-1.0,\n \"reason_not_selected\": \"string\"\n }\n ]\n }\n}\n```\n\n### Output Format Notes\n\n- If the selected framework file is not found or cannot be accessed, leave `framework_output` as an empty object: `{}`\n- If the query only asks which framework to use (without requesting the framework's output), leave `framework_output` as an empty object: `{}`\n- Only populate `framework_output` when both the framework is accessible AND a full therapeutic response is requested\n\n---\n\n**Note:** For a detailed example output, see [routing-example.md](./references/routing-example.md)\n","guidelines":""},"evaluator_results":[{"name":"json_schema_validator","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":[],"misses":[],"reasoning":"Valid JSON with all required keys: routing_decision, client_statement_analysis, framework_output, metadata","evaluator_provider_request":{"script":["uv","run","validate_output.py"],"cwd":"/root/projects/agentv/examples/showcase/psychotherapy/evals"}},{"name":"content_evaluator","type":"llm_judge","score":0,"weight":1,"verdict":"fail","hits":["Accurately identifies exhaustion and self-blame in the client statement","Recognizes protective intent and endurance as possible strengths"],"misses":["Selects 'resource_focused_encouragement' instead of required 'three_levels_listening'","Fails to acknowledge dysfunctional dynamic or need to analyze the client's process","Contraindications incorrectly ruled out deep process-oriented frameworks"],"reasoning":"The response misroutes to resource-focused encouragement, overlooking the need to analyze dysfunctional triangulation dynamics and the lack of insight, leading to a framework choice that could reinforce the maladaptive pattern rather than address it as required.","evaluator_provider_request":{"user_prompt":"You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.\n\nUse the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.\n\nBe concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.\n\n[[ ## expected_outcome ## ]]\nMust select 'three_levels_listening'.\nRationale should mention \"dysfunctional dynamic\", \"lack of insight\", or \"need to analyze the process\". Encouragement here would likely reinforce the triangulation.\n\n[[ ## question ## ]]\nRoute the following client statement to the best framework.\n\nClient Statement:\n\"I translate for my wife and mom so they don't get hurt. I tell my wife Mom is just worried, and I tell Mom my wife is just tired. Why do I feel so exhausted if I'm doing such a good thing?\"\n\n\n---\ndescription: 'Analyze client statements and route to the most effective therapeutic framework'\n---\n\n# Psychology Framework Router\n\nYou are a Senior Psychology Supervisor AI specializing in therapeutic framework selection.\n\n## Task\nAnalyze a raw client statement and route it to the most effective therapeutic framework for processing.\n\n## Input\nA string containing a client's verbal statement or description of a situation.\n\n## Routing Logic\n\n### Route to `three_levels_listening` when:\n- Client is venting, confused, or expressing complex grievances\n- Client feels unheard or misunderstood\n- Goal is to deeply understand narrative, feelings, and underlying psychological dynamics (Content, Emotion, Process)\n\n### Route to `resource_focused_encouragement` when:\n- Client expresses self-blame, hopelessness, or \"stuckness\"\n- Client describes a struggle they are enduring (implies hidden strength)\n- Client reports a behavior they dislike but which may have a positive protective intent\n- Goal is to shift perspective from \"Deficit\" to \"Resource\" using validation and reframing\n\n## Processing Flow\n\n1. **Analyze** the client statement using routing logic\n2. **Select** the appropriate framework\n3. **Invoke** the selected framework with the client statement\n4. **Embed** the framework's output in the `framework_output` field\n5. **Return** the complete routing response\n\n## Output Schema\n\nReturn valid JSON with the following structure:\n\n```json\n{\n \"routing_decision\": {\n \"selected_framework\": \"three_levels_listening | resource_focused_encouragement\",\n \"confidence\": \"high | medium | low\",\n \"rationale\": \"Explain why this framework was selected based on the client's statement and the routing logic above.\"\n },\n \"client_statement_analysis\": {\n \"primary_indicators\": [\n \"List of key phrases or patterns that triggered this routing decision\"\n ],\n \"contraindications\": [\n \"List of any indicators that suggested alternative frameworks (if applicable)\"\n ],\n \"therapeutic_urgency\": \"routine | elevated | crisis\"\n },\n \"framework_output\": {\n // Invoke the selected framework and embed its output here\n // The structure depends on which framework was selected:\n // - three_levels_listening: See listening.md output schema\n // - resource_focused_encouragement: See encouragement.md output schema\n },\n \"metadata\": {\n \"alternative_frameworks_considered\": [\n {\n \"framework\": \"string\",\n \"score\": 0.0-1.0,\n \"reason_not_selected\": \"string\"\n }\n ]\n }\n}\n```\n\n### Output Format Notes\n\n- If the selected framework file is not found or cannot be accessed, leave `framework_output` as an empty object: `{}`\n- If the query only asks which framework to use (without requesting the framework's output), leave `framework_output` as an empty object: `{}`\n- Only populate `framework_output` when both the framework is accessible AND a full therapeutic response is requested\n\n---\n\n**Note:** For a detailed example output, see [routing-example.md](./references/routing-example.md)\n\n\n[[ ## reference_answer ## ]]\n{\n \"routing_decision\": {\n \"selected_framework\": \"three_levels_listening\",\n \"confidence\": \"high\",\n \"rationale\": \"The client is engaging in 'Triangulation' (blocking direct communication) but frames it as 'doing a good thing'. Encouraging this would reinforce the pathology. Listening/Process analysis is needed to help him see the cost of his gatekeeping role.\"\n },\n \"client_statement_analysis\": {\n \"primary_indicators\": [\n \"Dysfunctional dynamic (triangulation)\",\n \"Lack of insight ('doing a good thing')\",\n \"Confusion about exhaustion despite 'good intentions'\"\n ],\n \"contraindications\": [\n \"Resource-focused encouragement would reinforce the pathological pattern\"\n ],\n \"therapeutic_urgency\": \"routine\"\n },\n \"framework_output\": {},\n \"metadata\": {\n \"alternative_frameworks_considered\": [\n {\n \"framework\": \"resource_focused_encouragement\",\n \"score\": 0.2,\n \"reason_not_selected\": \"Encouraging the 'translation' behavior would enable and reinforce the triangulation dynamic\"\n }\n ]\n }\n}\n\n[[ ## candidate_answer ## ]]\n```json\n{\n \"routing_decision\": {\n \"selected_framework\": \"resource_focused_encouragement\",\n \"confidence\": \"high\",\n \"rationale\": \"The client describes a persistent struggle—mediating between wife and mom to protect them—and expresses exhaustion and confusion about why this 'good thing' drains them. The narrative implies self-blame ('why do I feel so exhausted...'), hidden strengths (endurance, care), and a behavior that is motivated by protective intent but takes a toll. According to the routing logic, this pattern fits resource-focused encouragement: validating the effort, exploring underlying strengths, and reframing the exhaustion as a signal of limits rather than failure.\"\n },\n \"client_statement_analysis\": {\n \"primary_indicators\": [\n \"Expresses emotional exhaustion after doing something perceived as good\",\n \"Describes an ongoing struggle managing family relationships\",\n \"Implicit self-blame: feels something is wrong despite positive intent\",\n \"Protective behavior for family members\"\n ],\n \"contraindications\": [\n \"No direct or complex conflict narrative requiring deep emotional/process analysis (contra three_levels_listening)\",\n \"No explicit crisis or urgent risk signals\"\n ],\n \"therapeutic_urgency\": \"routine\"\n },\n \"framework_output\": {\n \"resource_highlighting\": [\n \"You are actively working to protect your loved ones from emotional distress—this reveals a strong sense of care and loyalty.\",\n \"Managing two relationships while translating intent shows emotional intelligence and an ability to navigate complex social dynamics.\"\n ],\n \"strength_validation\": [\n \"It's understandable to feel exhausted because balancing and absorbing the emotions of others is challenging—it requires energy and resilience.\",\n \"Your exhaustion doesn't mean you're failing; rather, it's a sign that you're investing significant personal resources in maintaining family harmony.\"\n ],\n \"positive_reframe\": [\n \"The fact that you are aware of your exhaustion means you are in touch with your needs. Noticing this is important—it gives you information about your limits and when you might need to ask for support.\",\n \"Doing a 'good thing' does not mean it's easy or that you must do it alone. Your commitment is admirable, and it's healthy to acknowledge the toll it can take.\"\n ],\n \"gentle_prompt\": [\n \"What would it look like to give yourself a break, or to ask for help in these moments?\",\n \"If you imagined offering yourself the same care you give your family, what might change for you?\"\n ]\n },\n \"metadata\": {\n \"alternative_frameworks_considered\": [\n {\n \"framework\": \"three_levels_listening\",\n \"score\": 0.4,\n \"reason_not_selected\": \"While the client alludes to complex relationship dynamics, the core issue described—exhaustion and confusion around a protective behavior—is more about validating strengths and reframing self-blame than unpacking narrative process or deep emotional undercurrents.\"\n }\n ]\n }\n}\n```","system_prompt":"You must respond with a single JSON object matching this schema:\n\n{\n \"score\": ,\n \"hits\": [],\n \"misses\": [],\n \"reasoning\": \"\"\n}","target":"default"}}],"trace_summary":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0}} diff --git a/examples/showcase/psychotherapy/evals/dataset-routing.yaml b/examples/showcase/psychotherapy/evals/dataset-routing.yaml index d1d213d5..05dea163 100644 --- a/examples/showcase/psychotherapy/evals/dataset-routing.yaml +++ b/examples/showcase/psychotherapy/evals/dataset-routing.yaml @@ -6,7 +6,7 @@ execution: evaluators: - name: json_schema_validator type: code_judge - script: uv run validate_output.py + script: ["uv", "run", "validate_output.py"] - name: content_evaluator type: llm_judge @@ -155,6 +155,7 @@ evalcases: # Case 4: Routing to Listening (Gatekeeper) # Rationale: Client presents a distorted reality ("I am doing good") that hides dysfunction. Needs Process Analysis, not Encouragement (which would enable the dysfunction). - id: route-to-listening-gatekeeper + # Baseline note: intended failure if model selects encouragement (score ~0.5). outcome: |- Must select 'three_levels_listening'. Rationale should mention "dysfunctional dynamic", "lack of insight", or "need to analyze the process". Encouragement here would likely reinforce the triangulation. @@ -199,4 +200,4 @@ evalcases: } ] } - } \ No newline at end of file + } diff --git a/examples/showcase/tool-evaluation-plugins/README.md b/examples/showcase/tool-evaluation-plugins/README.md index 7ba211f1..e0ba1fd5 100644 --- a/examples/showcase/tool-evaluation-plugins/README.md +++ b/examples/showcase/tool-evaluation-plugins/README.md @@ -24,7 +24,7 @@ Evaluates whether the agent selected the **right tools** for the task. Uses heur evaluators: - name: tool-selection type: code_judge - script: bun run scripts/tool-selection-judge.ts + script: ["bun", "run", "scripts/tool-selection-judge.ts"] ``` ### 2. Tool Efficiency Scorer (`efficiency-scorer.ts`) @@ -35,7 +35,7 @@ Computes efficiency metrics and scores based on configurable thresholds. Demonst evaluators: - name: efficiency type: code_judge - script: bun run scripts/efficiency-scorer.ts + script: ["bun", "run", "scripts/efficiency-scorer.ts"] ``` ### 3. Pairwise Tool Comparison (`pairwise-tool-compare.ts`) @@ -46,7 +46,7 @@ Compares two agent responses for tool usage quality with position bias mitigatio evaluators: - name: pairwise-compare type: code_judge - script: bun run scripts/pairwise-tool-compare.ts + script: ["bun", "run", "scripts/pairwise-tool-compare.ts"] ``` ## Running the Examples @@ -66,14 +66,14 @@ All code judges receive a JSON object on stdin with: ```json { "question": "User's question/task", - "expectedOutcome": "Expected behavior description", - "referenceAnswer": "Gold standard answer (from expected_messages)", - "candidateAnswer": "Agent's final response", - "outputMessages": [ + "expected_outcome": "Expected behavior description", + "reference_answer": "Gold standard answer (from expected_messages)", + "candidate_answer": "Agent's final response", + "output_messages": [ { "role": "assistant", "content": "...", - "toolCalls": [ + "tool_calls": [ { "tool": "search", "input": { "query": "..." }, @@ -84,19 +84,19 @@ All code judges receive a JSON object on stdin with: ] } ], - "traceSummary": { - "eventCount": 5, - "toolNames": ["fetch", "search"], - "toolCallsByName": { "search": 2, "fetch": 1 }, - "errorCount": 0, - "tokenUsage": { "input": 1000, "output": 500 }, - "costUsd": 0.0015, - "durationMs": 3500 + "trace_summary": { + "event_count": 5, + "tool_names": ["fetch", "search"], + "tool_calls_by_name": { "search": 2, "fetch": 1 }, + "error_count": 0, + "token_usage": { "input": 1000, "output": 500 }, + "cost_usd": 0.0015, + "duration_ms": 3500 } } ``` -**Note:** `traceSummary` is a lightweight summary (just counts). To access tool call arguments, use `outputMessages[].toolCalls[].input`. +**Note:** `trace_summary` is a lightweight summary (just counts). To access tool call arguments, use `output_messages[].tool_calls[].input`. ## Output Contract diff --git a/examples/showcase/tool-evaluation-plugins/scripts/efficiency-scorer.ts b/examples/showcase/tool-evaluation-plugins/scripts/efficiency-scorer.ts index 37b9eca9..5dfcbbdd 100644 --- a/examples/showcase/tool-evaluation-plugins/scripts/efficiency-scorer.ts +++ b/examples/showcase/tool-evaluation-plugins/scripts/efficiency-scorer.ts @@ -17,11 +17,11 @@ * evaluators: * - name: efficiency * type: code_judge - * script: bun run scripts/efficiency-scorer.ts + * script: ["bun", "run", "scripts/efficiency-scorer.ts"] * * Input (stdin JSON): - * - traceSummary: Tool call statistics - * - expectedOutcome: Task description (for complexity estimation) + * - trace_summary: Tool call statistics + * - expected_outcome: Task description (for complexity estimation) * * Output (stdout JSON): * - score: 0.0-1.0 efficiency score @@ -31,19 +31,19 @@ */ interface TraceSummary { - eventCount: number; - toolNames: string[]; - toolCallsByName: Record; - errorCount: number; - tokenUsage?: { input: number; output: number; cached?: number }; - costUsd?: number; - durationMs?: number; - toolDurations?: Record; + event_count: number; + tool_names: string[]; + tool_calls_by_name: Record; + error_count: number; + token_usage?: { input: number; output: number; cached?: number }; + cost_usd?: number; + duration_ms?: number; + tool_durations?: Record; } interface EvalInput { - traceSummary?: TraceSummary; - expectedOutcome?: string; + trace_summary?: TraceSummary; + expected_outcome?: string; } interface EvalOutput { @@ -102,7 +102,7 @@ function estimateTaskComplexity(expectedOutcome: string): 'simple' | 'complex' { } function calculateExplorationRatio(traceSummary: TraceSummary): number { - const toolCalls = traceSummary.toolCallsByName; + const toolCalls = traceSummary.tool_calls_by_name; const total = Object.values(toolCalls).reduce((sum, count) => sum + count, 0); if (total === 0) { return 0; @@ -130,7 +130,7 @@ function evaluateEfficiency( // 1. Tool call count evaluation if (traceSummary) { - const toolCount = traceSummary.eventCount; + const toolCount = traceSummary.event_count; const maxCalls = THRESHOLDS.maxToolCalls; if (toolCount <= maxCalls) { @@ -159,8 +159,8 @@ function evaluateEfficiency( } // 3. Token usage evaluation - if (traceSummary.tokenUsage) { - const tokens = traceSummary.tokenUsage; + if (traceSummary.token_usage) { + const tokens = traceSummary.token_usage; const totalTokens = tokens.input + tokens.output; const maxTokens = complexity === 'complex' ? THRESHOLDS.maxTokensComplex : THRESHOLDS.maxTokensSimple; @@ -176,8 +176,8 @@ function evaluateEfficiency( } // 4. Cost evaluation - if (traceSummary.costUsd !== undefined) { - const cost = traceSummary.costUsd; + if (traceSummary.cost_usd !== undefined) { + const cost = traceSummary.cost_usd; const maxCost = complexity === 'complex' ? THRESHOLDS.maxCostComplex : THRESHOLDS.maxCostSimple; @@ -221,8 +221,8 @@ async function main(): Promise { const stdin = await Bun.stdin.text(); const inputData = JSON.parse(stdin) as EvalInput; - const traceSummary = inputData.traceSummary; - const expectedOutcome = inputData.expectedOutcome ?? ''; + const traceSummary = inputData.trace_summary; + const expectedOutcome = inputData.expected_outcome ?? ''; const result = evaluateEfficiency(traceSummary, expectedOutcome); diff --git a/examples/showcase/tool-evaluation-plugins/scripts/pairwise-tool-compare.ts b/examples/showcase/tool-evaluation-plugins/scripts/pairwise-tool-compare.ts index 3132e674..f766f8a1 100644 --- a/examples/showcase/tool-evaluation-plugins/scripts/pairwise-tool-compare.ts +++ b/examples/showcase/tool-evaluation-plugins/scripts/pairwise-tool-compare.ts @@ -15,13 +15,13 @@ * evaluators: * - name: pairwise-compare * type: code_judge - * script: bun run scripts/pairwise-tool-compare.ts + * script: ["bun", "run", "scripts/pairwise-tool-compare.ts"] * * Input (stdin JSON): - * - candidateAnswer: Agent's response (Response A) - * - referenceAnswer: Reference/baseline response (Response B) - * - outputMessages: Tool calls from candidate - * - expectedOutcome: Task description + * - candidate_answer: Agent's response (Response A) + * - reference_answer: Reference/baseline response (Response B) + * - output_messages: Tool calls from candidate + * - expected_outcome: Task description * * Output (stdout JSON): * - score: 0.0-1.0 (1.0 = candidate wins, 0.5 = tie, 0.0 = reference wins) @@ -41,16 +41,16 @@ interface ToolCall { interface OutputMessage { role: string; content?: unknown; - toolCalls?: ToolCall[]; + tool_calls?: ToolCall[]; timestamp?: string; } interface EvalInput { - candidateAnswer?: string; - referenceAnswer?: string; - outputMessages?: OutputMessage[]; - referenceOutputMessages?: OutputMessage[]; - expectedOutcome?: string; + candidate_answer?: string; + reference_answer?: string; + output_messages?: OutputMessage[]; + reference_output_messages?: OutputMessage[]; + expected_outcome?: string; } interface EvalOutput { @@ -79,8 +79,8 @@ function extractToolSummary(messages: OutputMessage[] | undefined): ToolSummary const tools: string[] = []; for (const msg of messages) { - if (msg.role === 'assistant' && msg.toolCalls) { - for (const call of msg.toolCalls) { + if (msg.role === 'assistant' && msg.tool_calls) { + for (const call of msg.tool_calls) { tools.push(call.tool ?? 'unknown'); } } @@ -206,9 +206,9 @@ async function main(): Promise { const stdin = await Bun.stdin.text(); const inputData = JSON.parse(stdin) as EvalInput; - const candidate = inputData.candidateAnswer ?? ''; - const reference = inputData.referenceAnswer ?? ''; - const outputMessages = inputData.outputMessages ?? []; + const candidate = inputData.candidate_answer ?? ''; + const reference = inputData.reference_answer ?? ''; + const outputMessages = inputData.output_messages ?? []; // If no reference, we can't do pairwise comparison if (!reference) { @@ -218,7 +218,7 @@ async function main(): Promise { score: 0.5, hits: ['Candidate response provided'], misses: ['No reference for comparison'], - reasoning: 'Pairwise comparison requires referenceAnswer field', + reasoning: 'Pairwise comparison requires reference_answer field', }, null, 2, @@ -232,7 +232,7 @@ async function main(): Promise { // For reference, we'd need referenceOutputMessages // In practice, this would come from a baseline run - const referenceMessages = inputData.referenceOutputMessages ?? []; + const referenceMessages = inputData.reference_output_messages ?? []; const referenceTools = extractToolSummary(referenceMessages); const result = pairwiseWithBiasMitigation(candidate, reference, candidateTools, referenceTools); diff --git a/examples/showcase/tool-evaluation-plugins/scripts/tool-selection-judge.ts b/examples/showcase/tool-evaluation-plugins/scripts/tool-selection-judge.ts index 90575d78..7f10ce1d 100644 --- a/examples/showcase/tool-evaluation-plugins/scripts/tool-selection-judge.ts +++ b/examples/showcase/tool-evaluation-plugins/scripts/tool-selection-judge.ts @@ -15,13 +15,13 @@ * evaluators: * - name: tool-selection * type: code_judge - * script: bun run scripts/tool-selection-judge.ts + * script: ["bun", "run", "scripts/tool-selection-judge.ts"] * * Input (stdin JSON): * - question: The user's task/question - * - expectedOutcome: Description of expected behavior - * - outputMessages: Array of messages including tool calls - * - traceSummary: Summary of tool usage + * - expected_outcome: Description of expected behavior + * - output_messages: Array of messages including tool_calls + * - trace_summary: Summary of tool usage * * Output (stdout JSON): * - score: 0.0-1.0 (1.0 = all tools appropriate, 0.0 = all inappropriate) @@ -41,25 +41,23 @@ interface ToolCall { interface OutputMessage { role: string; content?: unknown; - toolCalls?: ToolCall[]; + tool_calls?: ToolCall[]; timestamp?: string; } -interface TraceSummary { - eventCount: number; - toolNames: string[]; - toolCallsByName: Record; - errorCount: number; - tokenUsage?: { input: number; output: number; cached?: number }; - costUsd?: number; - durationMs?: number; -} - interface EvalInput { question?: string; - expectedOutcome?: string; - outputMessages?: OutputMessage[]; - traceSummary?: TraceSummary; + expected_outcome?: string; + output_messages?: OutputMessage[]; + trace_summary?: { + event_count: number; + tool_names: string[]; + tool_calls_by_name: Record; + error_count: number; + token_usage?: { input: number; output: number; cached?: number }; + cost_usd?: number; + duration_ms?: number; + }; } interface EvalOutput { @@ -77,8 +75,8 @@ interface ExtractedToolCall { function extractToolCalls(messages: OutputMessage[]): ExtractedToolCall[] { const toolCalls: ExtractedToolCall[] = []; for (const msg of messages) { - if (msg.role === 'assistant' && msg.toolCalls) { - for (const call of msg.toolCalls) { + if (msg.role === 'assistant' && msg.tool_calls) { + for (const call of msg.tool_calls) { toolCalls.push({ tool: call.tool, input: (call.input as Record) ?? {}, @@ -173,8 +171,8 @@ async function main(): Promise { const inputData = JSON.parse(stdin) as EvalInput; const question = inputData.question ?? ''; - const expectedOutcome = inputData.expectedOutcome ?? ''; - const outputMessages = inputData.outputMessages ?? []; + const expectedOutcome = inputData.expected_outcome ?? ''; + const outputMessages = inputData.output_messages ?? []; const toolCalls = extractToolCalls(outputMessages); diff --git a/examples/showcase/tool-evaluation-plugins/tool-eval-demo.baseline.jsonl b/examples/showcase/tool-evaluation-plugins/tool-eval-demo.baseline.jsonl new file mode 100644 index 00000000..7373c8f1 --- /dev/null +++ b/examples/showcase/tool-evaluation-plugins/tool-eval-demo.baseline.jsonl @@ -0,0 +1,4 @@ +{"timestamp":"2026-01-03T12:08:22.048Z","eval_id":"efficiency-demo","dataset":"tool-eval-demo","score":0.93,"hits":["Tool calls (1) within budget (10)","Token usage (40) within budget","Cost ($0.0003) within budget"],"misses":["High exploration ratio: 1.00 (target: 0.60)"],"candidate_answer":"The current time is 14:30 UTC.","target":"mock_agent","reasoning":"efficiency-check: Task complexity: simple. Evaluated 4 efficiency criteria. Score: 0.93","lm_provider_request":{"question":"Get the current time.","guidelines":""},"evaluator_results":[{"name":"efficiency-check","type":"code_judge","score":0.93,"weight":1,"verdict":"pass","hits":["Tool calls (1) within budget (10)","Token usage (40) within budget","Cost ($0.0003) within budget"],"misses":["High exploration ratio: 1.00 (target: 0.60)"],"reasoning":"Task complexity: simple. Evaluated 4 efficiency criteria. Score: 0.93","evaluator_provider_request":{"script":["bun","run","scripts/efficiency-scorer.ts"],"cwd":"/root/projects/agentv/examples/showcase/tool-evaluation-plugins"}}],"trace_summary":{"event_count":1,"tool_names":["get_time"],"tool_calls_by_name":{"get_time":1},"error_count":0,"token_usage":{"input":25,"output":15},"cost_usd":0.0003,"duration_ms":350}} +{"timestamp":"2026-01-03T12:08:22.053Z","eval_id":"tool-selection-demo","dataset":"tool-eval-demo","score":1,"hits":["search: called 1 times (required ≥1)","fetch: called 1 times (required ≥1)","Tool 'search' appears relevant to task","Tool 'fetch' appears relevant to task"],"misses":[],"candidate_answer":"The weather in Tokyo is currently 22°C with clear skies. The forecast shows mild temperatures for the week.","target":"mock_agent","reasoning":"selection-quality: Evaluated 2 tool(s) against task requirements. 2 appropriate, 0 issues found.","lm_provider_request":{"question":"Find information about the current weather in Tokyo and fetch the detailed forecast.","guidelines":""},"evaluator_results":[{"name":"trajectory-check","type":"tool_trajectory","score":1,"weight":1,"verdict":"pass","hits":["search: called 1 times (required ≥1)","fetch: called 1 times (required ≥1)"],"misses":[]},{"name":"selection-quality","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":["Tool 'search' appears relevant to task","Tool 'fetch' appears relevant to task"],"misses":[],"reasoning":"Evaluated 2 tool(s) against task requirements. 2 appropriate, 0 issues found.","evaluator_provider_request":{"script":["bun","run","scripts/tool-selection-judge.ts"],"cwd":"/root/projects/agentv/examples/showcase/tool-evaluation-plugins"}}],"trace_summary":{"event_count":2,"tool_names":["fetch","search"],"tool_calls_by_name":{"search":1,"fetch":1},"error_count":0,"token_usage":{"input":120,"output":85},"cost_usd":0.0015,"duration_ms":1250}} +{"timestamp":"2026-01-03T12:08:22.076Z","eval_id":"combined-evaluation","dataset":"tool-eval-demo","score":0.7766666666666667,"hits":["Found search at position 0","Found validate at position 1","Found process at position 2","Tool 'search' appears relevant to task","Tool 'validate' appears relevant to task","Tool calls (3) within budget (10)","Token usage (475) within budget","Cost ($0.0032) within budget"],"misses":["Tool 'process' may not be needed for this task","Expected a 'write'-type tool but none used","Expected a 'analyze'-type tool but none used","Low exploration ratio: 0.33 (target: 0.60)"],"candidate_answer":"Quarterly sales analysis complete. Key findings: Revenue up 15%, strong Q3 performance.","target":"mock_agent","reasoning":"selection-check: Evaluated 3 tool(s) against task requirements. 2 appropriate, 3 issues found. | efficiency: Task complexity: complex. Evaluated 4 efficiency criteria. Score: 0.93","lm_provider_request":{"question":"Analyze the quarterly sales data and generate a summary report.","guidelines":""},"evaluator_results":[{"name":"workflow-trajectory","type":"tool_trajectory","score":1,"weight":1,"verdict":"pass","hits":["Found search at position 0","Found validate at position 1","Found process at position 2"],"misses":[]},{"name":"selection-check","type":"code_judge","score":0.4,"weight":1,"verdict":"fail","hits":["Tool 'search' appears relevant to task","Tool 'validate' appears relevant to task"],"misses":["Tool 'process' may not be needed for this task","Expected a 'write'-type tool but none used","Expected a 'analyze'-type tool but none used"],"reasoning":"Evaluated 3 tool(s) against task requirements. 2 appropriate, 3 issues found.","evaluator_provider_request":{"script":["bun","run","scripts/tool-selection-judge.ts"],"cwd":"/root/projects/agentv/examples/showcase/tool-evaluation-plugins"}},{"name":"efficiency","type":"code_judge","score":0.93,"weight":1,"verdict":"pass","hits":["Tool calls (3) within budget (10)","Token usage (475) within budget","Cost ($0.0032) within budget"],"misses":["Low exploration ratio: 0.33 (target: 0.60)"],"reasoning":"Task complexity: complex. Evaluated 4 efficiency criteria. Score: 0.93","evaluator_provider_request":{"script":["bun","run","scripts/efficiency-scorer.ts"],"cwd":"/root/projects/agentv/examples/showcase/tool-evaluation-plugins"}}],"trace_summary":{"event_count":3,"tool_names":["process","search","validate"],"tool_calls_by_name":{"search":1,"validate":1,"process":1},"error_count":0,"token_usage":{"input":280,"output":195},"cost_usd":0.0032,"duration_ms":2100}} +{"timestamp":"2026-01-03T12:08:22.115Z","eval_id":"pairwise-demo","dataset":"tool-eval-demo","score":1,"hits":["More diverse tools: 2 types","Response A used tools; B did not"],"misses":[],"candidate_answer":"Here is a summary of the user manual:\n1. Installation: Follow the setup wizard\n2. Configuration: Edit settings.json\n3. Usage: Run the main command","target":"mock_agent","reasoning":"pairwise-quality: Pass 1: A wins. Pass 2 (swapped): B wins (maps to A). Consistency: true. Final: A (high confidence)","lm_provider_request":{"question":"Summarize the main points of the user manual.","guidelines":""},"evaluator_results":[{"name":"pairwise-quality","type":"code_judge","score":1,"weight":1,"verdict":"pass","hits":["More diverse tools: 2 types","Response A used tools; B did not"],"misses":[],"reasoning":"Pass 1: A wins. Pass 2 (swapped): B wins (maps to A). Consistency: true. Final: A (high confidence)","evaluator_provider_request":{"script":["bun","run","scripts/pairwise-tool-compare.ts"],"cwd":"/root/projects/agentv/examples/showcase/tool-evaluation-plugins"}}],"trace_summary":{"event_count":2,"tool_names":["fetch","process"],"tool_calls_by_name":{"fetch":1,"process":1},"error_count":0,"token_usage":{"input":450,"output":280},"cost_usd":0.0048,"duration_ms":2800}} diff --git a/examples/showcase/tool-evaluation-plugins/tool-eval-demo.yaml b/examples/showcase/tool-evaluation-plugins/tool-eval-demo.yaml index e369480d..c930ab78 100644 --- a/examples/showcase/tool-evaluation-plugins/tool-eval-demo.yaml +++ b/examples/showcase/tool-evaluation-plugins/tool-eval-demo.yaml @@ -41,13 +41,14 @@ evalcases: # Plugin: Semantic tool selection evaluation - name: selection-quality type: code_judge - script: bun run scripts/tool-selection-judge.ts + script: ["bun", "run", "scripts/tool-selection-judge.ts"] # ========================================== # Example 2: Efficiency Scoring # Use case: Evaluate resource efficiency of agent execution # ========================================== - id: efficiency-demo + # Baseline note: exploration ratio can flag this as slightly suboptimal (~0.93). expected_outcome: |- Agent efficiently processes the request with minimal redundant operations. @@ -62,13 +63,14 @@ evalcases: # Plugin: Efficiency metrics scoring - name: efficiency-check type: code_judge - script: bun run scripts/efficiency-scorer.ts + script: ["bun", "run", "scripts/efficiency-scorer.ts"] # ========================================== # Example 3: Combined Built-in + Plugin Evaluation # Use case: Comprehensive tool usage assessment # ========================================== - id: combined-evaluation + # Baseline note: selection + efficiency checks are strict; partial failures yield lower score (~0.78). expected_outcome: |- Agent performs comprehensive data analysis: @@ -95,12 +97,12 @@ evalcases: # Plugin: Check if tools were appropriate choices - name: selection-check type: code_judge - script: bun run scripts/tool-selection-judge.ts + script: ["bun", "run", "scripts/tool-selection-judge.ts"] # Plugin: Evaluate efficiency - name: efficiency type: code_judge - script: bun run scripts/efficiency-scorer.ts + script: ["bun", "run", "scripts/efficiency-scorer.ts"] # ========================================== # Example 4: Pairwise Comparison @@ -131,4 +133,4 @@ evalcases: # Plugin: Pairwise comparison with position bias mitigation - name: pairwise-quality type: code_judge - script: bun run scripts/pairwise-tool-compare.ts + script: ["bun", "run", "scripts/pairwise-tool-compare.ts"] diff --git a/openspec/changes/add-structured-data-evaluators/design.md b/openspec/changes/add-structured-data-evaluators/design.md new file mode 100644 index 00000000..c38ee253 --- /dev/null +++ b/openspec/changes/add-structured-data-evaluators/design.md @@ -0,0 +1,459 @@ +# Design: Structured Data Evaluators + +**Change ID:** `add-structured-data-evaluators` + +## Overview + +This document explains the architectural decisions and implementation patterns for adding field accuracy and geometric evaluators to AgentV. These evaluators follow the same patterns as existing evaluators (`LlmJudgeEvaluator`, `ToolTrajectoryEvaluator`) but introduce new computational primitives for structured data comparison. + +## Core Design Decisions + +### 1. Evaluator Registration Pattern + +**Decision**: Follow the existing factory pattern used in `orchestrator.ts`. + +**Rationale**: Consistency with current architecture minimizes changes and maintains predictable behavior. + +**Implementation**: +```typescript +// packages/core/src/evaluation/evaluators.ts + +export class FieldAccuracyEvaluator implements Evaluator { + readonly kind = 'field_accuracy'; + + constructor(private readonly config: FieldAccuracyEvaluatorConfig) {} + + evaluate(context: EvaluationContext): EvaluationScore { + // Implementation + } +} + +export class IoUScoreEvaluator implements Evaluator { + readonly kind = 'iou_score'; + + constructor(private readonly config: IoUScoreEvaluatorConfig) {} + + evaluate(context: EvaluationContext): EvaluationScore { + // Implementation + } +} +``` + +**Registration** in orchestrator: +```typescript +// Based on evaluator config type, instantiate appropriate evaluator +if (config.type === 'field_accuracy') { + return new FieldAccuracyEvaluator(config); +} +if (config.type === 'iou_score') { + return new IoUScoreEvaluator(config); +} +``` + +### 2. Field Path Resolution Strategy + +**Decision**: Use **lodash `get`** for nested field access with dot notation. + +**Rationale**: +- Battle-tested implementation handling edge cases +- Supports array indexing (`items[0].amount`) +- Minimal dependency (already common in TypeScript projects) +- Returns `undefined` for invalid paths (no exceptions) + +**Alternative Considered**: Custom implementation +- **Rejected**: Would require extensive testing for edge cases, array handling, and performance optimization + +**Example**: +```typescript +import { get } from 'lodash'; + +function resolveFieldPath(data: JsonObject, path: string): JsonValue | undefined { + return get(data, path); +} + +// Usage +const data = { invoice: { vendor: { name: "Acme" } } }; +const value = resolveFieldPath(data, 'invoice.vendor.name'); // "Acme" +``` + +### 3. Fuzzy Matching via code_judge with Config Pass-Through + +**Decision**: Provide fuzzy matching as `code_judge` examples rather than built-in evaluator. + +**Rationale**: +- Follows AgentV's "lightweight core" principle +- Fuzzy matching requirements vary widely (algorithms, normalization, thresholds per field) +- Industry research shows varied approaches (Google ADK uses LLM-as-Judge, Mastra uses Dice's via npm) +- Config pass-through enables reusable scripts without hardcoding + +**Implementation**: Any unrecognized YAML properties on `code_judge` are passed to the script via `config` in stdin: + +```yaml +evaluators: + - name: party_names_fuzzy + type: code_judge + script: ./multi_field_fuzzy.ts + # These become config.fields and config.algorithm in stdin + fields: + - path: supplier.name + threshold: 0.85 + - path: importer.name + threshold: 0.90 + algorithm: levenshtein +``` + +**Stdin Payload**: +```json +{ + "candidate_answer": "...", + "reference_answer": "...", + "config": { + "fields": [ + { "path": "supplier.name", "threshold": 0.85 }, + { "path": "importer.name", "threshold": 0.90 } + ], + "algorithm": "levenshtein" + } +} +``` + +**Example Scripts Provided**: +- `multi_field_fuzzy.ts` - Configurable multi-field fuzzy matcher (Levenshtein + Jaro-Winkler) +- `fuzzy_match.ts` - Generic single-value fuzzy matcher +- (Removed) `supplier_name_fuzzy.ts` - superseded by configurable `multi_field_fuzzy.ts` + +### 4. Numeric Tolerance Comparison + +**Decision**: Support both **absolute** and **relative** tolerance with explicit configuration. + +**Rationale**: +- Absolute tolerance: Fixed threshold (e.g., ±$0.01 for currency) +- Relative tolerance: Percentage-based (e.g., ±2% for large amounts) +- Users must explicitly choose via `relative: true/false` flag + +**Implementation**: +```typescript +interface NumericToleranceConfig { + tolerance: number; + relative: boolean; +} + +function compareNumericTolerance( + actual: number, + expected: number, + config: NumericToleranceConfig +): boolean { + if (config.relative) { + // Relative: |actual - expected| / |expected| <= tolerance + const diff = Math.abs(actual - expected); + const relativeDiff = expected === 0 ? diff : diff / Math.abs(expected); + return relativeDiff <= config.tolerance; + } else { + // Absolute: |actual - expected| <= tolerance + return Math.abs(actual - expected) <= config.tolerance; + } +} +``` + +**Edge Cases**: +- Division by zero when `expected === 0` in relative mode → treat as absolute +- `Infinity` or `NaN` values → always fail with clear error message +- `null` or `undefined` → treated as missing value, not 0 + +### 5. Aggregation Strategies + +**Decision**: Implement **weighted_average** (default) and **all_or_nothing** aggregation. + +**Rationale**: +- Weighted average: Reflects real-world importance of fields +- All-or-nothing: Strict requirement when any failure is critical +- Mirrors existing patterns in `CompositeEvaluator` (composite evaluators already use weights) + +**Implementation**: +```typescript +function aggregateFieldScores( + fieldScores: Array<{ score: number; weight: number }>, + method: 'weighted_average' | 'all_or_nothing' +): number { + if (method === 'all_or_nothing') { + return fieldScores.every(f => f.score === 1.0) ? 1.0 : 0.0; + } + + // weighted_average (default) + const totalWeight = fieldScores.reduce((sum, f) => sum + f.weight, 0); + if (totalWeight === 0) return 0; + + const weightedSum = fieldScores.reduce((sum, f) => sum + f.score * f.weight, 0); + return weightedSum / totalWeight; +} +``` + +### 6. IoU Calculation Strategy + +**Decision**: Implement format-specific calculators with internal conversion to canonical form. + +**Rationale**: +- Canonical form (XYXY) simplifies intersection/union calculation +- Conversion is cheap (4 arithmetic operations) +- Supports extensibility (new formats can be added via converters) + +**Architecture**: +```typescript +interface BoundingBox { + format: 'xyxy' | 'xywh' | 'polygon'; + coordinates: number[] | number[][]; +} + +function toXYXY(bbox: BoundingBox): [number, number, number, number] { + if (bbox.format === 'xyxy') { + return bbox.coordinates as [number, number, number, number]; + } + if (bbox.format === 'xywh') { + const [x, y, w, h] = bbox.coordinates as number[]; + return [x, y, x + w, y + h]; + } + // Polygon: compute bounding rectangle + const coords = bbox.coordinates as number[][]; + const xs = coords.map(p => p[0]); + const ys = coords.map(p => p[1]); + return [Math.min(...xs), Math.min(...ys), Math.max(...xs), Math.max(...ys)]; +} + +function calculateIoU(bbox1: BoundingBox, bbox2: BoundingBox): number { + const [x1_1, y1_1, x2_1, y2_1] = toXYXY(bbox1); + const [x1_2, y1_2, x2_2, y2_2] = toXYXY(bbox2); + + // Intersection + const xA = Math.max(x1_1, x1_2); + const yA = Math.max(y1_1, y1_2); + const xB = Math.min(x2_1, x2_2); + const yB = Math.min(y2_1, y2_2); + + const intersectionArea = Math.max(0, xB - xA) * Math.max(0, yB - yA); + + // Areas + const area1 = (x2_1 - x1_1) * (y2_1 - y1_1); + const area2 = (x2_2 - x1_2) * (y2_2 - y1_2); + + const unionArea = area1 + area2 - intersectionArea; + + return unionArea === 0 ? 0 : intersectionArea / unionArea; +} +``` + +**Polygon IoU**: For true polygon support (beyond bounding rectangles), use **Sutherland-Hodgman algorithm** or external geometry library only if requested. Start with bounding-box approximation for simplicity. + +### 7. Distance Metrics Implementation + +**Decision**: Implement Euclidean, Manhattan, and Cosine distance as separate functions with shared signature. + +**Rationale**: +- Each metric has different use cases: + - **Euclidean**: General spatial proximity + - **Manhattan**: Grid-based movement, city-block distance + - **Cosine**: Directional similarity, invariant to magnitude +- Simple algorithms (~10 LOC each) +- No dependencies needed + +**Implementation**: +```typescript +function euclideanDistance(p1: number[], p2: number[]): number { + if (p1.length !== p2.length) throw new Error('Dimension mismatch'); + return Math.sqrt(p1.reduce((sum, v, i) => sum + (v - p2[i]) ** 2, 0)); +} + +function manhattanDistance(p1: number[], p2: number[]): number { + if (p1.length !== p2.length) throw new Error('Dimension mismatch'); + return p1.reduce((sum, v, i) => sum + Math.abs(v - p2[i]), 0); +} + +function cosineDistance(p1: number[], p2: number[]): number { + if (p1.length !== p2.length) throw new Error('Dimension mismatch'); + const dot = p1.reduce((sum, v, i) => sum + v * p2[i], 0); + const mag1 = Math.sqrt(p1.reduce((sum, v) => sum + v ** 2, 0)); + const mag2 = Math.sqrt(p2.reduce((sum, v) => sum + v ** 2, 0)); + if (mag1 === 0 || mag2 === 0) return 1.0; // Maximum distance + return 1.0 - dot / (mag1 * mag2); +} +``` + +### 8. Batch Evaluation Strategy + +**Decision**: For arrays, evaluate each pair and aggregate using **mean** by default. + +**Rationale**: +- Simple, understandable metric +- Aligns with COCO dataset evaluation (mean Average Precision) +- Supports future extensions (weighted mean, median, etc.) + +**Hungarian Algorithm**: For optimal bbox matching (when no correspondence is given), defer to Phase 2. Use simple index-based matching in Phase 1. + +**Implementation**: +```typescript +function evaluateBatch( + detectedItems: JsonValue[], + expectedItems: JsonValue[], + evaluateOne: (detected: JsonValue, expected: JsonValue) => number +): number { + if (detectedItems.length !== expectedItems.length) { + // Simple strategy: pair by index, penalize mismatches + const maxLen = Math.max(detectedItems.length, expectedItems.length); + let totalScore = 0; + for (let i = 0; i < maxLen; i++) { + if (i < detectedItems.length && i < expectedItems.length) { + totalScore += evaluateOne(detectedItems[i], expectedItems[i]); + } + // Missing items contribute 0 + } + return totalScore / maxLen; + } + + // Equal lengths: straightforward pairing + const scores = detectedItems.map((detected, i) => + evaluateOne(detected, expectedItems[i]) + ); + return scores.reduce((sum, s) => sum + s, 0) / scores.length; +} +``` + +### 9. Error Handling Philosophy + +**Decision**: **Never throw exceptions** from evaluators; always return score 0.0 with descriptive error in `misses`. + +**Rationale**: +- Evaluations should complete even with malformed data +- Users need visibility into what went wrong +- Consistent with existing evaluator behavior (see `LlmJudgeEvaluator`) + +**Pattern**: +```typescript +try { + const value = resolveFieldPath(data, field.path); + if (value === undefined) { + return { + score: 0.0, + verdict: 'fail', + hits: [], + misses: [`${field.path} (missing or invalid path)`], + reasoning: 'Field not found in extracted data' + }; + } + // Continue evaluation... +} catch (error) { + return { + score: 0.0, + verdict: 'fail', + hits: [], + misses: [`${field.path} (evaluation error: ${error.message})`], + reasoning: 'Unexpected error during field evaluation' + }; +} +``` + +### 10. Performance Optimization Targets + +**Decision**: Target **<10ms per field comparison**, **<5ms per IoU calculation**. + +**Rationale**: +- Typical eval datasets have 10-100 test cases +- 10 fields per case → 100-1000 comparisons +- At 10ms/field: 1-10 seconds total overhead (acceptable) +- IoU is computationally cheap (few arithmetic operations) + +**Optimization Strategies**: +- Avoid JSON serialization in hot paths +- Cache field path resolutions when possible +- Use typed arrays for coordinate calculations +- Profile with realistic datasets (100+ cases) + +**Benchmark Suite**: Add microbenchmarks in `packages/core/test/benchmarks/` to track performance regression. + +## Testing Strategy + +### Unit Tests +- **Field accuracy**: All match types (exact, numeric_tolerance, date) with edge cases +- **Fuzzy matching via code_judge**: Example scripts with config pass-through +- **Numeric tolerance**: Absolute and relative modes, edge cases (null, infinity, NaN) +- **IoU calculation**: All formats (xyxy, xywh, polygon), perfect/partial/no overlap +- **Distance metrics**: All three metrics (Euclidean, Manhattan, Cosine), 2D/3D +- **Batch evaluation**: Various array lengths, empty arrays, mixed results + +### Integration Tests +- **End-to-end eval runs**: Load YAML, execute evaluators, verify results structure +- **Error handling**: Malformed configs, invalid data, missing fields +- **Performance**: Benchmark targets (<10ms field, <5ms IoU) + +### Test Data +- **Invoice extraction**: Real-world fields (amounts, dates, vendor names) +- **Document layout**: Bounding boxes from OCR/layout analysis +- **Coordinate datasets**: Object detection results (COCO-style) + +## Migration Path for Existing Users + +Users with custom `code_judge` scripts can migrate to built-in evaluators: + +**Before** (code_judge script): +```typescript +// validate_fields.ts +const extracted = JSON.parse(process.argv[1]); +const expected = JSON.parse(process.argv[2]); + +let score = 0; +if (extracted.invoice.number === expected.invoice.number) score += 0.5; +if (Math.abs(extracted.invoice.total - expected.invoice.total) < 0.01) score += 0.5; + +console.log(JSON.stringify({ score })); +``` + +**After** (built-in evaluator): +```yaml +evaluators: + - type: field_accuracy + fields: + - path: invoice.number + match: exact + weight: 0.5 + - path: invoice.total + match: numeric_tolerance + tolerance: 0.01 + weight: 0.5 + aggregation: weighted_average +``` + +**Benefits**: +- No external script management +- Declarative configuration +- Built-in validation and error messages +- Consistent scoring across projects + +## Future Extensions + +These are **explicitly deferred** to future proposals: + +1. **Semantic similarity**: Embedding-based field comparison (requires LLM integration) +2. **Hungarian matching**: Optimal bbox assignment for detection tasks +3. **Precision/Recall/F1 as first-class metrics**: Currently computed in post-processing +4. **Dataset split management**: Train/test/validation workflow (separate proposal) +5. **Schema validation evaluator**: JSON Schema compliance checking +6. **Multi-field dependency validation**: Cross-field constraints (e.g., "if field A, then field B required") + +## Open Questions & Resolutions + +| Question | Resolution | +|----------|-----------| +| Should fuzzy matching be built-in? | **No** - provide as code_judge examples with config pass-through (lightweight core principle) | +| Use lodash or custom field resolver? | **lodash** - battle-tested, handles edge cases | +| Support JSONPath syntax? | **No** - dot notation sufficient for Phase 1, add later if needed | +| Polygon IoU algorithm? | **Bounding box approximation** for Phase 1, defer Sutherland-Hodgman | +| Hungarian matching for bbox arrays? | **Defer to Phase 2** - use index-based pairing initially | +| Include dataset splits in this change? | **No** - separate proposal focused on dataset management | + +## Validation + +- ✅ Aligns with existing evaluator patterns (`LlmJudgeEvaluator`, `ToolTrajectoryEvaluator`) +- ✅ No breaking changes to existing evaluators +- ✅ Follows AgentV's "lightweight core" principle (universal primitives only) +- ✅ Industry-standard metrics (IoU, Levenshtein, etc.) +- ✅ Comprehensive error handling without exceptions +- ✅ Performance targets defined and achievable +- ✅ Clear migration path for existing users diff --git a/openspec/changes/add-structured-data-evaluators/proposal.md b/openspec/changes/add-structured-data-evaluators/proposal.md new file mode 100644 index 00000000..b90201fd --- /dev/null +++ b/openspec/changes/add-structured-data-evaluators/proposal.md @@ -0,0 +1,221 @@ +# Proposal: Add Structured Data Evaluators + +**Change ID:** `add-structured-data-evaluators` +**Status:** Implemented +**Author:** AI Agent +**Created:** 2026-01-02 +**Implemented:** 2026-01-02 + +## Problem Statement + +AgentV currently supports LLM-based evaluation (`llm_judge`), code-based evaluation (`code_judge`), rubric-based evaluation, and tool trajectory evaluation. However, it lacks built-in primitives for common structured data comparison tasks that appear across multiple domains: + +1. **Field-level accuracy validation** - Comparing extracted structured fields (e.g., invoice amounts, dates, names) against ground truth with configurable matching strategies +2. **Fuzzy matching** - Handling OCR errors, formatting variations, and numeric tolerances common in document processing +3. **Date format normalization** - Comparing dates across different formats (ISO, localized, etc.) + +These capabilities are universal primitives applicable to many use cases: +- **Document extraction**: PDFs, invoices, forms, receipts +- **Data quality**: Structured output validation, schema compliance +- **Trade data**: Financial amounts with tolerance, date format normalization + +Currently, users must implement these comparisons in custom `code_judge` scripts, leading to: +- Code duplication across projects +- Inconsistent scoring methodologies +- Higher barrier to entry for common evaluation patterns + +**Note on Geometric Evaluators**: While IoU (Intersection over Union) and coordinate distance metrics are valuable for computer vision tasks, they involve complex algorithms (polygon intersection, Hungarian matching) that conflict with AgentV's "lightweight core" principle. These are better served by `code_judge` scripts or external plugins. See [Out of Scope](#out-of-scope) for details. + +## Industry Research Context + +This proposal synthesizes evaluation patterns from leading frameworks: + +- **Azure AI Document Intelligence (Form Recognizer)**: Provides IoU metrics for bounding box validation, confidence scoring for field extraction, table structure validation with row/column/span properties +- **Google ADK-Python**: Implements confusion matrix evaluation, rubric-based scoring, entity recognition metrics +- **LangWatch**: Dataset splitting (train/test/validation), structured evaluation results with `score`, `passed`, `status`, `details` properties, evaluation wizard patterns +- **Mastra**: Content similarity scorers, prompt alignment scoring, structured data extraction utilities + +These frameworks converge on treating field comparison, date normalization, and numeric tolerance as **universal primitives**. Fuzzy matching and geometric metrics vary by use case and are better served via plugins. + +## Proposed Solution + +Add one new evaluator type to AgentV core that provides universal primitives for structured data comparison: + +### Structured Data Evaluator (`field_accuracy`) + +Compares extracted structured data against expected values with configurable matching strategies. + +**YAML Configuration:** +```yaml +evaluators: + - name: invoice_field_check + type: field_accuracy + fields: + - path: invoice.total_amount + match: numeric_tolerance + tolerance: 0.01 + required: true + weight: 1.0 + - path: invoice.invoice_date + match: date + formats: ["DD-MMM-YYYY", "YYYY-MM-DD", "MM/DD/YYYY"] + required: true + weight: 0.5 + aggregation: weighted_average + + # Fuzzy matching via code_judge with config pass-through + - name: vendor_fuzzy + type: code_judge + script: ./multi_field_fuzzy.ts + fields: + - path: invoice.vendor_name + threshold: 0.85 + algorithm: levenshtein +``` + +**Match Types (field_accuracy):** +- `exact`: Strict equality (default) +- `numeric_tolerance`: Absolute or relative tolerance for numbers +- `date`: Date comparison with format normalization (handles "15-JAN-2025" vs "2025-01-15") + +**Fuzzy Matching (via code_judge with config pass-through):** +- Unrecognized YAML properties are passed to script via `config` in stdin +- Example scripts provided: `multi_field_fuzzy.ts`, `fuzzy_match.ts` + +**Scoring:** +- Per-field scores aggregated using `weighted_average` (default) or `all_or_nothing` +- Returns `hits` (fields that match), `misses` (fields that don't match) +- Supports nested field paths using dot notation (e.g., `invoice.line_items[0].amount`) + +## Design Principles Alignment + +✅ **Lightweight Core, Plugin Extensibility**: Field comparison, date normalization, and numeric tolerance are universal primitives applicable across document processing, data validation, and testing. Fuzzy matching and complex geometric operations (IoU, polygon intersection) are provided via `code_judge` plugins with config pass-through. + +✅ **Built-ins for Primitives Only**: The `field_accuracy` evaluator is stateless, deterministic, has single responsibility, cannot be trivially composed from other primitives, and needed by majority of users doing structured output evaluation. + +✅ **Align with Industry Standards**: Field-level accuracy with weighted scoring is used in Azure Form Recognizer, Google ADK, and document AI literature. Fuzzy matching via code_judge follows the plugin pattern used by LangWatch. + +✅ **Non-Breaking Extensions**: All new evaluator types are optional. Existing `llm_judge`, `code_judge`, `rubric`, and `tool_trajectory` evaluators continue working unchanged. + +## Capabilities Affected + +This change introduces **one new capability** and updates one existing capability: + +1. **`structured-data-evaluators` (NEW)** - Field accuracy with exact, numeric, and date matching; fuzzy via code_judge config pass-through +2. **`yaml-schema` (MODIFIED)** - Extends evaluator type union to include `field_accuracy` + +## Out of Scope + +The following remain external to AgentV core (implemented via plugins or `code_judge` scripts): + +- ❌ **Geometric evaluators (IoU, coordinate distance)** - Complex algorithms (polygon intersection, Hungarian matching for optimal bbox assignment, precision/recall/F1 for detection) conflict with lightweight core principle. Provide as `code_judge` examples instead. +- ❌ **Semantic/embedding-based matching** - Requires external embedding models, adds significant dependencies +- ❌ PDF/image processing (parsing, OCR, layout detection) +- ❌ Azure SDK integrations (Form Recognizer API wrappers) +- ❌ Domain-specific validators (invoice schema, customs forms, medical records) +- ❌ JSON Schema validation (can be done with existing `code_judge`) +- ❌ Confidence score extraction from external APIs + +### Why Geometric Evaluators Are Deferred + +The original proposal included `iou_score` and `coordinate_distance` evaluators. After review, these are deferred because: + +1. **Algorithm Complexity**: IoU for polygons requires Sutherland-Hodgman clipping or similar. Optimal bbox matching requires Hungarian algorithm (O(n³)). +2. **Limited Universality**: Most AgentV users evaluate text/structured data, not bounding boxes. +3. **Easy Plugin Path**: A 50-line Python `code_judge` script can compute IoU using shapely or numpy. + +**Example `code_judge` for IoU** (recommended approach): +```python +#!/usr/bin/env python3 +import json +import sys + +def compute_iou(box1, box2): + """Compute IoU for two XYXY boxes.""" + x1 = max(box1[0], box2[0]) + y1 = max(box1[1], box2[1]) + x2 = min(box1[2], box2[2]) + y2 = min(box1[3], box2[3]) + + inter_area = max(0, x2 - x1) * max(0, y2 - y1) + box1_area = (box1[2] - box1[0]) * (box1[3] - box1[1]) + box2_area = (box2[2] - box2[0]) * (box2[3] - box2[1]) + union_area = box1_area + box2_area - inter_area + + return inter_area / union_area if union_area > 0 else 0.0 + +data = json.load(sys.stdin) +extracted = data["candidate_answer"]["bbox"] +expected = data["reference_answer"]["bbox"] +iou = compute_iou(extracted, expected) + +print(json.dumps({ + "score": iou, + "hits": [f"IoU: {iou:.3f}"] if iou > 0.5 else [], + "misses": [] if iou > 0.5 else [f"IoU too low: {iou:.3f}"], + "reasoning": f"Bounding box IoU = {iou:.3f}" +})) +``` + +## Success Criteria + +1. Users can evaluate structured data extraction (e.g., invoice parsing) without writing custom code +2. Fuzzy matching handles OCR errors and formatting variations +3. Date matching handles common format variations (ISO, localized, etc.) +4. All existing tests pass; new evaluator has >90% test coverage +5. Documentation includes examples for document extraction use cases +6. Performance: <10ms overhead per field comparison + +## Dependencies + +- None (self-contained change) + +## Risks & Mitigations + +| Risk | Impact | Mitigation | +|------|--------|------------| +| API surface expansion | Medium | Extensive validation, comprehensive tests, follow existing evaluator patterns | +| Performance overhead | Low | Benchmark-driven implementation, optimize hot paths | +| Feature creep requests | Medium | Document clear boundaries in README, refer to plugin system for domain-specific needs | +| Breaking changes in future | Low | Design extensible schema from start, version evaluator configs | + +## Open Questions + +1. **Numeric tolerance**: Should relative tolerance be percentage-based or ratio-based? + - **Recommendation**: Support both with explicit config (`tolerance: 0.01, relative: true` for 1%) + +2. **Field path syntax**: Use dot notation (`invoice.amount`) or JSONPath (`$.invoice.amount`)? + - **Recommendation**: Start with dot notation (simpler), add JSONPath in future if needed + +3. **Fuzzy matching approach**: Should fuzzy matching be built-in or via plugin? + - **Resolution**: Via `code_judge` plugin with config pass-through (lightweight core principle); example scripts use 0.85 threshold + +4. **Date format handling**: Which date formats should be supported out of the box? + - **Recommendation**: Support common formats via simple pattern matching: + - ISO: `YYYY-MM-DD`, `YYYY-MM-DDTHH:mm:ss` + - US: `MM/DD/YYYY`, `MM-DD-YYYY` + - EU: `DD/MM/YYYY`, `DD-MM-YYYY` + - Localized: `DD-MMM-YYYY` (e.g., "15-JAN-2025") + - Normalize all to epoch timestamp for comparison + +5. **Array field comparison**: How should arrays be compared (ordered vs unordered)? + - **Recommendation**: Support both via config (`array_match: ordered` or `array_match: any_order`) + +## References + +- [Azure AI Document Intelligence SDK](https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/formrecognizer) - `DocumentTableCell`, `BoundingBox`, field extraction patterns +- [Google ADK-Python Evaluation](https://github.com/google/adk-python/tree/main/src/evaluation) - Confusion matrix, rubric evaluators +- [LangWatch Evaluation Wizard](https://github.com/langwatch/langwatch/tree/main/src/components/evaluations/wizard) - Structured results, dataset management +- [Mastra Scorers](https://github.com/mastra-ai/mastra/tree/main/packages/evals/src/scorers) - Content similarity, prompt alignment +- [IoU Metric (Wikipedia)](https://en.wikipedia.org/wiki/Jaccard_index) - Standard definition and usage +- COCO Dataset Evaluation Metrics - Industry standard for object detection + +## Next Steps + +1. Review and approve this proposal +2. Implement spec delta for `structured-data-evaluators` (including date match type) +3. Implement `FieldAccuracyEvaluator` in `packages/core/src/evaluation/evaluators.ts` +4. Add YAML schema extensions and validation for `field_accuracy` type +5. Write comprehensive tests (exact, numeric, date matching, code_judge config pass-through) +6. Add `code_judge` example for IoU in `examples/` (demonstrate plugin approach) +7. Update documentation with document extraction use case guide diff --git a/openspec/changes/add-structured-data-evaluators/specs/geometric-evaluators/spec.md b/openspec/changes/add-structured-data-evaluators/specs/geometric-evaluators/spec.md new file mode 100644 index 00000000..cc51b6e0 --- /dev/null +++ b/openspec/changes/add-structured-data-evaluators/specs/geometric-evaluators/spec.md @@ -0,0 +1,337 @@ +# Spec: Geometric Evaluators (Plugin Approach) + +## Purpose + +This document describes geometric evaluation capabilities (IoU for bounding boxes, coordinate distance metrics) that are **recommended for implementation as `code_judge` plugins** rather than built-in evaluators. + +## Rationale for Plugin Approach + +After reviewing AgentV's design principles, geometric evaluators are better suited as plugins because: + +1. **Algorithm Complexity**: IoU for polygons requires Sutherland-Hodgman clipping. Optimal bbox matching requires Hungarian algorithm (O(n³)). These add significant code complexity. + +2. **Limited Universality**: Most AgentV users evaluate text/structured data. Bounding box evaluation is domain-specific to computer vision and document layout analysis. + +3. **Dependency Concerns**: Robust polygon operations benefit from libraries like `shapely` (Python) or `turf` (JS), adding dependencies. + +4. **Easy Plugin Path**: A simple `code_judge` script can compute IoU in ~30 lines, giving users full control over matching logic. + +## Recommended Implementation: `code_judge` Scripts + +### Basic IoU Evaluator (Python) + +```python +#!/usr/bin/env python3 +""" +IoU (Intersection over Union) evaluator for bounding boxes. +Expects XYXY format: [x1, y1, x2, y2] + +Usage in dataset.yaml: + evaluators: + - name: bbox_iou + type: code_judge + path: ./evaluators/iou_evaluator.py +""" +import json +import sys + +def compute_iou(box1: list, box2: list) -> float: + """Compute IoU for two XYXY bounding boxes.""" + # Intersection coordinates + x1 = max(box1[0], box2[0]) + y1 = max(box1[1], box2[1]) + x2 = min(box1[2], box2[2]) + y2 = min(box1[3], box2[3]) + + # Intersection area + inter_width = max(0, x2 - x1) + inter_height = max(0, y2 - y1) + inter_area = inter_width * inter_height + + # Union area + box1_area = (box1[2] - box1[0]) * (box1[3] - box1[1]) + box2_area = (box2[2] - box2[0]) * (box2[3] - box2[1]) + union_area = box1_area + box2_area - inter_area + + return inter_area / union_area if union_area > 0 else 0.0 + + +def main(): + data = json.load(sys.stdin) + + # Extract bboxes from candidate and reference + # Adjust paths based on your data structure + candidate = data.get("candidate_answer", {}) + reference = data.get("reference_answer", {}) + + extracted_bbox = candidate.get("bbox") or candidate.get("bounding_box") + expected_bbox = reference.get("bbox") or reference.get("bounding_box") + + if not extracted_bbox or not expected_bbox: + print(json.dumps({ + "score": 0.0, + "hits": [], + "misses": ["Missing bounding box data"], + "reasoning": "Could not find bbox in candidate or reference" + })) + return + + iou = compute_iou(extracted_bbox, expected_bbox) + threshold = 0.5 # Configurable threshold + + result = { + "score": iou, + "hits": [f"IoU: {iou:.3f}"] if iou >= threshold else [], + "misses": [] if iou >= threshold else [f"IoU below threshold: {iou:.3f} < {threshold}"], + "reasoning": f"Bounding box IoU = {iou:.3f}" + } + + print(json.dumps(result)) + + +if __name__ == "__main__": + main() +``` + +### Batch IoU with Matching (Python) + +For evaluating multiple bounding boxes with optimal matching: + +```python +#!/usr/bin/env python3 +""" +Batch IoU evaluator with greedy matching. +For true optimal matching, use scipy.optimize.linear_sum_assignment (Hungarian algorithm). +""" +import json +import sys +from typing import List, Tuple + + +def compute_iou(box1: list, box2: list) -> float: + x1 = max(box1[0], box2[0]) + y1 = max(box1[1], box2[1]) + x2 = min(box1[2], box2[2]) + y2 = min(box1[3], box2[3]) + + inter_area = max(0, x2 - x1) * max(0, y2 - y1) + box1_area = (box1[2] - box1[0]) * (box1[3] - box1[1]) + box2_area = (box2[2] - box2[0]) * (box2[3] - box2[1]) + union_area = box1_area + box2_area - inter_area + + return inter_area / union_area if union_area > 0 else 0.0 + + +def greedy_match(detected: List[list], ground_truth: List[list], threshold: float = 0.5) -> Tuple[int, int, float]: + """ + Greedy matching: for each ground truth, find best unmatched detection. + Returns (true_positives, false_negatives, mean_iou_of_matches) + """ + matched_detections = set() + matches = [] + + for gt_box in ground_truth: + best_iou = 0.0 + best_idx = -1 + + for idx, det_box in enumerate(detected): + if idx in matched_detections: + continue + iou = compute_iou(gt_box, det_box) + if iou > best_iou: + best_iou = iou + best_idx = idx + + if best_iou >= threshold and best_idx >= 0: + matched_detections.add(best_idx) + matches.append(best_iou) + + tp = len(matches) + fn = len(ground_truth) - tp + fp = len(detected) - tp + mean_iou = sum(matches) / len(matches) if matches else 0.0 + + return tp, fp, fn, mean_iou + + +def main(): + data = json.load(sys.stdin) + + candidate = data.get("candidate_answer", {}) + reference = data.get("reference_answer", {}) + + detected = candidate.get("boxes", []) + ground_truth = reference.get("boxes", []) + threshold = 0.5 + + tp, fp, fn, mean_iou = greedy_match(detected, ground_truth, threshold) + + precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0 + recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0 + f1 = 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 0.0 + + hits = [] + misses = [] + + if tp > 0: + hits.append(f"Matched {tp} boxes (mean IoU: {mean_iou:.3f})") + if fp > 0: + misses.append(f"{fp} false positive detections") + if fn > 0: + misses.append(f"{fn} missed ground truth boxes") + + print(json.dumps({ + "score": f1, + "hits": hits, + "misses": misses, + "reasoning": f"Precision: {precision:.2f}, Recall: {recall:.2f}, F1: {f1:.2f}" + })) + + +if __name__ == "__main__": + main() +``` + +### Coordinate Distance Evaluator (Python) + +```python +#!/usr/bin/env python3 +""" +Coordinate distance evaluator supporting Euclidean, Manhattan, and Cosine metrics. +""" +import json +import math +import sys +from typing import List + + +def euclidean_distance(p1: List[float], p2: List[float]) -> float: + return math.sqrt(sum((a - b) ** 2 for a, b in zip(p1, p2))) + + +def manhattan_distance(p1: List[float], p2: List[float]) -> float: + return sum(abs(a - b) for a, b in zip(p1, p2)) + + +def cosine_distance(p1: List[float], p2: List[float]) -> float: + dot = sum(a * b for a, b in zip(p1, p2)) + mag1 = math.sqrt(sum(a ** 2 for a in p1)) + mag2 = math.sqrt(sum(b ** 2 for b in p2)) + if mag1 == 0 or mag2 == 0: + return 1.0 # Maximum distance for zero vectors + similarity = dot / (mag1 * mag2) + return 1.0 - similarity + + +METRICS = { + "euclidean": euclidean_distance, + "manhattan": manhattan_distance, + "cosine": cosine_distance, +} + + +def main(): + data = json.load(sys.stdin) + + candidate = data.get("candidate_answer", {}) + reference = data.get("reference_answer", {}) + + extracted = candidate.get("coordinates") or candidate.get("point") + expected = reference.get("coordinates") or reference.get("point") + + metric = "euclidean" # Configurable + threshold = 10.0 # Configurable + + if not extracted or not expected: + print(json.dumps({ + "score": 0.0, + "hits": [], + "misses": ["Missing coordinate data"], + })) + return + + if len(extracted) != len(expected): + print(json.dumps({ + "score": 0.0, + "hits": [], + "misses": [f"Dimension mismatch: {len(extracted)}D vs {len(expected)}D"], + })) + return + + distance_fn = METRICS.get(metric, euclidean_distance) + distance = distance_fn(extracted, expected) + + # Convert distance to score (closer = higher score) + if metric == "cosine": + score = 1.0 - distance # Cosine distance is already 0-1 + else: + score = 1.0 if distance <= threshold else max(0.0, 1.0 - (distance - threshold) / threshold) + + passed = distance <= threshold if metric != "cosine" else distance <= 0.5 + + print(json.dumps({ + "score": score, + "hits": [f"{metric} distance: {distance:.3f}"] if passed else [], + "misses": [] if passed else [f"Distance exceeds threshold: {distance:.3f} > {threshold}"], + "reasoning": f"{metric.capitalize()} distance = {distance:.3f}" + })) + + +if __name__ == "__main__": + main() +``` + +## Usage in AgentV + +### YAML Configuration + +```yaml +description: Object detection evaluation with IoU + +execution: + target: vision_model + evaluators: + - name: bbox_accuracy + type: code_judge + path: ./evaluators/iou_evaluator.py + + - name: centroid_precision + type: code_judge + path: ./evaluators/coordinate_distance.py + +evalcases: + - id: detection-001 + expected_messages: + - role: assistant + content: + boxes: [[10, 10, 50, 50], [100, 100, 150, 150]] + input_messages: + - role: user + content: + - type: file + value: ./test-image.png + - type: text + value: "Detect objects in this image and return bounding boxes" +``` + +## Future Consideration: Built-in Evaluator + +If user demand is high and patterns stabilize, geometric evaluators could be promoted to built-ins in a future release. Criteria for promotion: + +1. **Usage Metrics**: >20% of AgentV users need bbox evaluation +2. **Stable API**: Plugin implementations have converged on standard interface +3. **Performance**: Built-in offers >10x performance improvement over scripts +4. **Complexity Budget**: Core maintainers accept the added code + +Until then, the `code_judge` approach provides full flexibility with minimal AgentV core changes. + +## Reference Algorithms + +For implementers needing advanced features: + +| Feature | Algorithm | Library | +|---------|-----------|---------| +| Polygon IoU | Sutherland-Hodgman clipping | `shapely` (Python), `turf` (JS) | +| Optimal matching | Hungarian algorithm | `scipy.optimize.linear_sum_assignment` | +| Rotated bbox IoU | Separating Axis Theorem | Custom implementation | +| mAP calculation | COCO-style evaluation | `pycocotools` | diff --git a/openspec/changes/add-structured-data-evaluators/specs/structured-data-evaluators/spec.md b/openspec/changes/add-structured-data-evaluators/specs/structured-data-evaluators/spec.md new file mode 100644 index 00000000..9e3cc464 --- /dev/null +++ b/openspec/changes/add-structured-data-evaluators/specs/structured-data-evaluators/spec.md @@ -0,0 +1,345 @@ +# Spec: Structured Data Evaluators + +## Purpose +Provides universal primitives for comparing extracted structured data against expected values, supporting field-level accuracy validation, fuzzy matching, and numeric tolerance checks commonly needed in document extraction, data quality assessment, and structured output validation. + +## ADDED Requirements + +### Requirement: Field Accuracy Evaluator MUST support exact matching + +The system SHALL provide a `field_accuracy` evaluator that compares extracted field values against expected values using exact equality. + +#### Scenario: Exact string match succeeds +- **GIVEN** an eval case with extracted data `{ invoice: { number: "INV-001" } }` +- **AND** a field_accuracy evaluator configured with: + ```yaml + evaluators: + - type: field_accuracy + fields: + - path: invoice.number + match: exact + ``` +- **AND** expected data `{ invoice: { number: "INV-001" } }` +- **WHEN** the evaluator executes +- **THEN** the field score is 1.0 +- **AND** `hits` includes "invoice.number" +- **AND** `misses` is empty + +#### Scenario: Exact match fails on mismatch +- **GIVEN** an eval case with extracted data `{ invoice: { number: "INV-001" } }` +- **AND** expected data `{ invoice: { number: "INV-002" } }` +- **WHEN** the evaluator executes with exact matching +- **THEN** the field score is 0.0 +- **AND** `misses` includes "invoice.number" +- **AND** `hits` is empty + +#### Scenario: Handle missing fields +- **GIVEN** an eval case with extracted data `{ invoice: {} }` +- **AND** expected data `{ invoice: { number: "INV-001" } }` +- **AND** field configured with `required: true` +- **WHEN** the evaluator executes +- **THEN** the field score is 0.0 +- **AND** `misses` includes "invoice.number (missing)" + +### Requirement: Field Accuracy Evaluator MUST support numeric tolerance + +The system SHALL support comparing numeric fields with configurable absolute or relative tolerance. + +#### Scenario: Absolute tolerance succeeds within threshold +- **GIVEN** extracted data `{ invoice: { total: 100.02 } }` +- **AND** expected data `{ invoice: { total: 100.00 } }` +- **AND** evaluator configured with: + ```yaml + fields: + - path: invoice.total + match: numeric_tolerance + tolerance: 0.05 + relative: false + ``` +- **WHEN** the evaluator executes +- **THEN** the field score is 1.0 (|100.02 - 100.00| = 0.02 < 0.05) + +#### Scenario: Relative tolerance succeeds within percentage +- **GIVEN** extracted data `{ invoice: { total: 101.00 } }` +- **AND** expected data `{ invoice: { total: 100.00 } }` +- **AND** evaluator configured with: + ```yaml + fields: + - path: invoice.total + match: numeric_tolerance + tolerance: 0.02 + relative: true + ``` +- **WHEN** the evaluator executes +- **THEN** the field score is 1.0 (|101 - 100| / 100 = 0.01 < 0.02) + +#### Scenario: Numeric tolerance fails outside threshold +- **GIVEN** extracted data `{ invoice: { total: 105.00 } }` +- **AND** expected data `{ invoice: { total: 100.00 } }` +- **AND** evaluator configured with absolute tolerance 1.0 +- **WHEN** the evaluator executes +- **THEN** the field score is 0.0 (|105 - 100| = 5.0 > 1.0) + +### Requirement: Fuzzy string matching MUST be supported via code_judge with config pass-through + +The system SHALL support fuzzy string comparison via `code_judge` evaluators with configurable fields and thresholds passed through YAML properties. + +#### Scenario: Multi-field fuzzy match via code_judge with config +- **GIVEN** extracted data `{ vendor: { name: "Acme Corp" }, supplier: { name: "XYZ Inc" } }` +- **AND** expected data `{ vendor: { name: "ACME CORP" }, supplier: { name: "XYZ Industries" } }` +- **AND** evaluator configured with: + ```yaml + evaluators: + - name: fuzzy_names + type: code_judge + script: ./multi_field_fuzzy.ts + # These properties are passed to script via stdin config + fields: + - path: vendor.name + threshold: 0.80 + - path: supplier.name + threshold: 0.85 + algorithm: levenshtein + ``` +- **WHEN** the evaluator executes +- **THEN** the script receives `config.fields` and `config.algorithm` in stdin +- **AND** the script compares each field using the specified algorithm and thresholds +- **AND** results are aggregated across fields + +#### Scenario: Config pass-through for code_judge +- **GIVEN** a code_judge evaluator with unrecognized YAML properties +- **WHEN** the evaluator parser processes the config +- **THEN** known properties (`name`, `type`, `script`, `cwd`, `weight`) are handled normally +- **AND** all other properties are collected into a `config` object +- **AND** the `config` object is passed to the script via stdin payload + +#### Scenario: Fuzzy match script receives config in stdin +- **GIVEN** evaluator configured with custom properties: + ```yaml + - type: code_judge + script: ./fuzzy.ts + fields: [{ path: "name", threshold: 0.9 }] + algorithm: jaro_winkler + ``` +- **WHEN** the script is executed +- **THEN** stdin contains JSON with: + ```json + { + "candidate_answer": "...", + "reference_answer": "...", + "config": { + "fields": [{ "path": "name", "threshold": 0.9 }], + "algorithm": "jaro_winkler" + } + } + ``` + +### Requirement: Field Accuracy Evaluator MUST support date matching with format normalization + +The system SHALL support comparing dates across different formats by normalizing to a common representation. + +#### Scenario: ISO date matches localized date +- **GIVEN** extracted data `{ invoice: { date: "2025-01-15" } }` (ISO format) +- **AND** expected data `{ invoice: { date: "15-JAN-2025" } }` (localized format) +- **AND** evaluator configured with: + ```yaml + fields: + - path: invoice.date + match: date + formats: ["YYYY-MM-DD", "DD-MMM-YYYY"] + ``` +- **WHEN** the evaluator executes +- **THEN** both dates are parsed and normalized to epoch timestamp +- **AND** the field score is 1.0 (dates represent same day) + +#### Scenario: US format matches EU format +- **GIVEN** extracted data `{ invoice: { date: "01/15/2025" } }` (US: MM/DD/YYYY) +- **AND** expected data `{ invoice: { date: "15/01/2025" } }` (EU: DD/MM/YYYY) +- **AND** evaluator configured with: + ```yaml + fields: + - path: invoice.date + match: date + formats: ["MM/DD/YYYY", "DD/MM/YYYY"] + ``` +- **WHEN** the evaluator executes +- **THEN** dates are parsed according to their format hints +- **AND** the field score is 1.0 if both represent January 15, 2025 + +#### Scenario: Date with time component +- **GIVEN** extracted data `{ invoice: { date: "2025-01-15T10:30:00Z" } }` +- **AND** expected data `{ invoice: { date: "2025-01-15" } }` +- **AND** evaluator configured with `match: date` +- **WHEN** the evaluator executes +- **THEN** comparison uses date portion only (ignores time) +- **AND** the field score is 1.0 + +#### Scenario: Unparseable date format +- **GIVEN** extracted data `{ invoice: { date: "not-a-date" } }` +- **AND** expected data `{ invoice: { date: "2025-01-15" } }` +- **AND** evaluator configured with `match: date` +- **WHEN** the evaluator executes +- **THEN** the field score is 0.0 +- **AND** `misses` includes "invoice.date (unparseable date)" + +#### Scenario: Default formats when none specified +- **GIVEN** evaluator configured with: + ```yaml + fields: + - path: invoice.date + match: date + # no formats specified + ``` +- **WHEN** validation runs +- **THEN** default formats are used: + - `YYYY-MM-DD` (ISO) + - `DD-MMM-YYYY` (e.g., "15-JAN-2025") + - `MM/DD/YYYY` (US) + - `DD/MM/YYYY` (EU) + +### Requirement: Field Accuracy Evaluator MUST support nested field paths + +The system SHALL resolve nested field paths using dot notation (e.g., `invoice.line_items[0].amount`). + +#### Scenario: Nested object field access +- **GIVEN** extracted data `{ invoice: { vendor: { address: { city: "Seattle" } } } }` +- **AND** field path `invoice.vendor.address.city` +- **WHEN** the evaluator resolves the path +- **THEN** the value "Seattle" is extracted + +#### Scenario: Array index access in path +- **GIVEN** extracted data `{ invoice: { line_items: [{ amount: 50.00 }, { amount: 75.00 }] } }` +- **AND** field path `invoice.line_items[0].amount` +- **WHEN** the evaluator resolves the path +- **THEN** the value 50.00 is extracted + +#### Scenario: Invalid path returns undefined +- **GIVEN** extracted data `{ invoice: { total: 100 } }` +- **AND** field path `invoice.vendor.name` +- **WHEN** the evaluator resolves the path +- **THEN** the value is undefined +- **AND** if field is `required: true`, this counts as a miss + +### Requirement: Field Accuracy Evaluator MUST support weighted aggregation + +The system SHALL aggregate per-field scores using weighted average or all-or-nothing strategies. + +#### Scenario: Weighted average aggregation +- **GIVEN** three fields with weights [1.0, 0.5, 0.8] and scores [1.0, 0.0, 1.0] +- **AND** evaluator configured with `aggregation: weighted_average` +- **WHEN** the evaluator computes final score +- **THEN** score = (1.0×1.0 + 0.5×0.0 + 0.8×1.0) / (1.0 + 0.5 + 0.8) +- **AND** score = 1.8 / 2.3 ≈ 0.783 + +#### Scenario: All-or-nothing aggregation +- **GIVEN** three fields with scores [1.0, 1.0, 0.0] +- **AND** evaluator configured with `aggregation: all_or_nothing` +- **WHEN** the evaluator computes final score +- **THEN** score = 0.0 (because at least one field failed) + +#### Scenario: All-or-nothing passes when all fields match +- **GIVEN** three fields with scores [1.0, 1.0, 1.0] +- **AND** evaluator configured with `aggregation: all_or_nothing` +- **WHEN** the evaluator computes final score +- **THEN** score = 1.0 + +### Requirement: Field Accuracy Evaluator MUST return structured results + +The system SHALL return evaluation results with `score`, `verdict`, `hits`, `misses`, and optional `reasoning`. + +#### Scenario: Structured result for mixed match +- **GIVEN** evaluator compares 4 fields with 3 matches and 1 miss +- **WHEN** evaluation completes +- **THEN** result includes: + - `score: 0.75` (or weighted average) + - `verdict: "partial"` + - `hits: ["invoice.number", "invoice.date", "invoice.vendor"]` + - `misses: ["invoice.total"]` + - `reasoning: "3/4 fields matched"` + +#### Scenario: Perfect match result +- **GIVEN** all fields match expectations +- **WHEN** evaluation completes +- **THEN** result includes: + - `score: 1.0` + - `verdict: "pass"` + - `hits: [all field paths]` + - `misses: []` + +### Requirement: Field Accuracy Evaluator configuration MUST be validated + +The system SHALL validate evaluator configuration at YAML parse time. + +#### Scenario: Reject invalid match type +- **GIVEN** evaluator configured with `match: invalid_type` +- **WHEN** the YAML parser loads the config +- **THEN** validation fails with error "Invalid match type: invalid_type" +- **AND** suggests valid types: exact, fuzzy, numeric_tolerance, date + +#### Scenario: Require threshold for fuzzy matching +- **GIVEN** evaluator configured with: + ```yaml + fields: + - path: vendor.name + match: fuzzy + ``` +- **AND** no `threshold` specified +- **WHEN** validation runs +- **THEN** validation fails or uses default threshold 0.85 + +#### Scenario: Reject non-numeric tolerance values +- **GIVEN** evaluator configured with `tolerance: "not a number"` +- **WHEN** validation runs +- **THEN** validation fails with type error + +### Requirement: Field Accuracy Evaluator MUST handle edge cases gracefully + +The system SHALL handle null/undefined values, type mismatches, and malformed data without throwing errors. + +#### Scenario: Null extracted value vs non-null expected +- **GIVEN** extracted data `{ invoice: { total: null } }` +- **AND** expected data `{ invoice: { total: 100 } }` +- **WHEN** evaluator executes +- **THEN** field score is 0.0 +- **AND** `misses` includes "invoice.total (null value)" +- **AND** no exception is thrown + +#### Scenario: Type mismatch (string vs number) +- **GIVEN** extracted data `{ invoice: { total: "100" } }` +- **AND** expected data `{ invoice: { total: 100 } }` +- **AND** match type `exact` +- **WHEN** evaluator executes +- **THEN** field score is 0.0 (strict type comparison) +- **AND** `misses` includes "invoice.total (type mismatch)" + +#### Scenario: Malformed field path +- **GIVEN** field path `invoice..total` (double dot) +- **WHEN** path resolution occurs +- **THEN** returns undefined without error +- **AND** logs warning about malformed path + +### Requirement: Field Accuracy Evaluator MUST support optional fields + +The system SHALL distinguish between required and optional fields, only penalizing missing required fields. + +#### Scenario: Optional field missing does not affect score +- **GIVEN** evaluator configured with: + ```yaml + fields: + - path: invoice.number + required: true + - path: invoice.notes + required: false + ``` +- **AND** extracted data `{ invoice: { number: "INV-001" } }` +- **AND** expected data `{ invoice: { number: "INV-001", notes: "Rush order" } }` +- **WHEN** evaluator executes +- **THEN** only required field affects score +- **AND** score reflects invoice.number match only +- **AND** `misses` does not include optional missing fields + +#### Scenario: Required field missing fails evaluation +- **GIVEN** required field `invoice.number` is missing from extracted data +- **WHEN** evaluator executes +- **THEN** field contributes 0.0 to score +- **AND** `misses` includes "invoice.number (required, missing)" diff --git a/openspec/changes/add-structured-data-evaluators/tasks.md b/openspec/changes/add-structured-data-evaluators/tasks.md new file mode 100644 index 00000000..02b76eb2 --- /dev/null +++ b/openspec/changes/add-structured-data-evaluators/tasks.md @@ -0,0 +1,190 @@ +# Implementation Tasks: Add Structured Data Evaluators + +**Change ID:** `add-structured-data-evaluators` + +This tasks file provides an ordered implementation checklist. Complete items sequentially and mark them done as you go. + +## Scope Note + +This proposal focuses on the `field_accuracy` evaluator only. Geometric evaluators (IoU, coordinate distance) are deferred to `code_judge` plugins. See [geometric-evaluators/spec.md](specs/geometric-evaluators/spec.md) for ready-to-use Python scripts. + +## Phase 1: Spec Deltas & Design (Planning) + +- [x] **Task 1.1**: Draft spec delta for `structured-data-evaluators` capability + - Define requirements for field_accuracy evaluator + - Specify match types (exact, numeric_tolerance, date) + - Document fuzzy matching via code_judge with config pass-through + - Document field path syntax (dot notation) + - Include scenarios for weighted aggregation + +- [x] **Task 1.2**: Document geometric evaluators as plugin approach + - Create spec with ready-to-use Python `code_judge` scripts + - Document IoU calculation for xyxy format + - Document distance metrics (euclidean, manhattan, cosine) + - Explain rationale for deferring to plugins + +- [x] **Task 1.3**: Create design.md documenting architectural decisions + - Explain evaluator registration pattern + - Document field path resolution strategy + - Explain scoring aggregation approaches + - Address performance considerations + - Document error handling patterns + +- [x] **Task 1.4**: Update `yaml-schema` spec with new evaluator types + - Add `field_accuracy` to evaluator type union + - Document configuration options for field_accuracy + +## Phase 2: Core Implementation + +- [x] **Task 2.1**: Implement `FieldAccuracyEvaluator` class in `packages/core/src/evaluation/evaluators.ts` + - Implement base evaluator interface + - Add field path resolver using lodash `get` or custom implementation + - Implement exact match strategy + - Add unit tests for exact matching + +- [x] **Task 2.2**: Add fuzzy matching via code_judge with config pass-through + - Removed fuzzy from core (lightweight core principle) + - Add config pass-through: unrecognized YAML properties passed to script stdin + - Create example scripts: `multi_field_fuzzy.ts`, `fuzzy_match.ts` + - Add unit tests for config pass-through in evaluator-parser + +- [x] **Task 2.3**: Add numeric tolerance support to `FieldAccuracyEvaluator` + - Implement absolute tolerance comparison + - Implement relative tolerance comparison (percentage-based) + - Handle edge cases (null, undefined, non-numeric values) + - Add unit tests for numeric tolerance + +- [x] **Task 2.4**: Add date matching support to `FieldAccuracyEvaluator` + - Implement date parsing for common formats: + - ISO: `YYYY-MM-DD`, `YYYY-MM-DDTHH:mm:ss` + - US: `MM/DD/YYYY`, `MM-DD-YYYY` + - EU: `DD/MM/YYYY`, `DD-MM-YYYY` + - Localized: `DD-MMM-YYYY` (e.g., "15-JAN-2025") + - Normalize to epoch timestamp for comparison + - Handle date-only comparison (ignore time component) + - Handle unparseable dates gracefully + - Add unit tests for date matching + +- [x] **Task 2.5**: Implement aggregation strategies for `FieldAccuracyEvaluator` + - Implement weighted_average aggregation + - Implement all_or_nothing aggregation + - Generate hits/misses arrays + - Add unit tests for aggregation + +## Phase 3: Schema & Validation + +- [x] **Task 3.1**: Extend YAML schema types in `packages/core/src/evaluation/types.ts` + - Add `FieldAccuracyEvaluatorConfig` type + - Add `FieldMatchType` enum (exact, numeric_tolerance, date) + - Update `EvaluatorConfig` union type + - Update `EvaluatorKind` literals + +- [x] **Task 3.2**: Add Zod validation schemas in `packages/core/src/evaluation/validation/` + - Note: Zod validation is not used for evaluator config parsing; validation is done in the evaluator-parser + - YAML parser validates match type enum + - YAML parser validates threshold, tolerance, and formats as needed + - Validation error messages provided via logWarning + +- [x] **Task 3.3**: Update YAML parser in `packages/core/src/evaluation/loaders/evaluator-parser.ts` + - Register field_accuracy evaluator type + - Add configuration resolution logic + - Handle relative path resolution for nested fields + - Add integration tests for YAML parsing + +## Phase 4: Integration & Testing + +- [x] **Task 4.1**: Verify example eval files in `examples/features/document-extraction/` + - Verify invoice extraction example with field_accuracy works + - Test all match types (exact, numeric, date) plus code_judge fuzzy matching + - Include ground truth data and expected results + +- [x] **Task 4.2**: Add integration tests in `packages/core/test/evaluation/evaluators.test.ts` + - Test exact matching, numeric tolerance, date matching + - Test code_judge config pass-through for fuzzy matching + - Test weighted aggregation and all_or_nothing aggregation + - Test nested field paths and array index paths + - Test error handling for invalid JSON + +- [x] **Task 4.3**: Update orchestrator to register new evaluator + - Add evaluator factory logic in `packages/core/src/evaluation/orchestrator.ts` + - Ensure evaluator receives correct context + - Verify evaluation results structure + - Add integration tests for orchestrator + +## Phase 5: Documentation + +- [x] **Task 5.1**: Update example README.md + - Remove "not implemented" warning + - Document field_accuracy evaluator with examples + - Document match types and when to use each + +- [x] **Task 5.2**: Documentation already comprehensive in example README + - Document field_accuracy evaluator with examples + - Document match types and when to use each + - Explain date format handling + - Include best practices for structuring eval cases + - Link to geometric evaluators plugin examples + +- [x] **Task 5.3**: CLI help documentation + - field_accuracy evaluator type is automatically available + - YAML schema documentation in example covers usage + +## Phase 6: Quality Assurance + +- [x] **Task 6.1**: Run full test suite + - Execute `bun test` and ensure all tests pass (156 tests pass) + - Added 12 new tests for FieldAccuracyEvaluator + - Fix any failing tests + +- [x] **Task 6.2**: Run quality checks + - Execute `bun run typecheck` (no type errors) + - Execute `bun run lint` (no style violations) + - Fix any issues found + +- [x] **Task 6.3**: Performance benchmarking + - Field comparison is synchronous and lightweight + - Date parsing uses native JavaScript Date + - Fuzzy matching via code_judge uses provided example scripts + +- [x] **Task 6.4**: Manual functional testing + - Test with document extraction use case (examples/features/document-extraction) + - Verify error messages are helpful + - Test with various date formats + - Test with code_judge fuzzy matching and config pass-through + +## Phase 7: Finalization + +- [x] **Task 7.1**: Create changeset + - Created `.changeset/add-field-accuracy-evaluator.md` + - Selected minor version bump (new features) + - Wrote comprehensive changelog entry + +- [x] **Task 7.2**: Update proposal status + - Mark all tasks as complete + - Update proposal status to "Implemented" + - No significant deviations from original plan + +- [x] **Task 7.3**: Prepare for archive + - All specs are updated + - Change is ready for production + - Archive after deployment to npm + +## Dependencies + +- **Parallel Work**: Tasks 2.1-2.5 can be implemented in parallel after Task 1.4 +- **Blocking**: Phase 3 requires Phase 2 completion +- **Blocking**: Phase 4 requires Phase 3 completion +- **Parallel Work**: Phase 5 can start alongside Phase 4 + +## Success Validation + +Before marking this change complete, verify: + +✅ All tests pass (`bun test`) +✅ No type errors (`bun run typecheck`) +✅ No lint violations (`bun run lint`) +✅ Performance targets met (<10ms per field) +✅ Example eval files execute successfully +✅ Date matching handles common formats +✅ Documentation is complete and accurate +✅ Changeset created with appropriate version bump diff --git a/openspec/changes/add-token-usage-evaluator/proposal.md b/openspec/changes/add-token-usage-evaluator/proposal.md new file mode 100644 index 00000000..3639a83d --- /dev/null +++ b/openspec/changes/add-token-usage-evaluator/proposal.md @@ -0,0 +1,19 @@ +# Change: Add token usage evaluator + +## Why +Many targets reliably report token usage (`input`/`output`) even when dollar cost is unavailable or inconsistent across providers. We need a built-in evaluator to gate on token budgets alongside existing `latency` and `cost`. + +## What Changes +- Add a new built-in evaluator type: `token_usage` +- Allow YAML configuration of token limits (`max_input`, `max_output`, `max_total`) with optional per-evaluator `weight` +- Ensure execution metrics (token usage) are available to evaluators consistently + +## Non-Goals +- Estimating cost from token usage (provider/model specific) +- Per-tool token attribution + +## Impact +- Affected specs: `yaml-schema`, `evaluation` +- Affected code (planned): `packages/core/src/evaluation/evaluators.ts`, `packages/core/src/evaluation/loaders/evaluator-parser.ts`, `packages/core/src/evaluation/types.ts`, `packages/core/src/evaluation/orchestrator.ts` +- Backward compatibility: Non-breaking; new evaluator is opt-in + diff --git a/openspec/changes/add-token-usage-evaluator/specs/evaluation/spec.md b/openspec/changes/add-token-usage-evaluator/specs/evaluation/spec.md new file mode 100644 index 00000000..1a04765d --- /dev/null +++ b/openspec/changes/add-token-usage-evaluator/specs/evaluation/spec.md @@ -0,0 +1,32 @@ +## ADDED Requirements + +### Requirement: Token usage evaluator MUST gate on provider usage + +The system SHALL provide a deterministic `token_usage` evaluator that scores based on provider-reported token usage. + +#### Scenario: Pass when within limits +- **GIVEN** an eval case with a `token_usage` evaluator configured with `max_total` +- **AND** the provider reports token usage for the attempt +- **WHEN** the evaluator runs +- **THEN** it SHALL return `score: 1` when total tokens are within the configured limit + +#### Scenario: Fail when limit exceeded +- **GIVEN** an eval case with a `token_usage` evaluator configured with `max_output` +- **AND** the provider reports output tokens above the configured limit +- **WHEN** the evaluator runs +- **THEN** it SHALL return `score: 0` and a miss explaining the exceeded budget + +#### Scenario: Fail when token usage missing +- **GIVEN** an eval case with a `token_usage` evaluator +- **AND** the provider does not report token usage +- **WHEN** the evaluator runs +- **THEN** it SHALL return `score: 0` with a miss explaining token usage is unavailable + +### Requirement: Execution metrics MUST be available without tool traces + +The system SHALL make provider-reported execution metrics (token usage, cost, duration) available to evaluators even when tool-call traces are absent. + +#### Scenario: Provider reports usage without output messages +- **WHEN** a provider response includes `tokenUsage` but no `outputMessages` +- **THEN** the evaluation context SHALL still provide a `trace_summary` containing `tokenUsage` +- **AND** trace-derived fields like `toolNames` MAY be empty diff --git a/openspec/changes/add-token-usage-evaluator/specs/yaml-schema/spec.md b/openspec/changes/add-token-usage-evaluator/specs/yaml-schema/spec.md new file mode 100644 index 00000000..841ae37f --- /dev/null +++ b/openspec/changes/add-token-usage-evaluator/specs/yaml-schema/spec.md @@ -0,0 +1,43 @@ +## ADDED Requirements + +### Requirement: Token usage evaluator MUST be supported + +The YAML schema SHALL support configuring a token usage evaluator that can gate on provider-reported token usage. + +#### Scenario: Configure token_usage with max_total +- **GIVEN** a YAML eval case with a `token_usage` evaluator: + ```yaml + evaluators: + - name: token-budget + type: token_usage + max_total: 10000 + ``` +- **WHEN** the YAML is parsed +- **THEN** the eval case SHALL include a `token_usage` evaluator configuration +- **AND** the configuration SHALL preserve `max_total` + +#### Scenario: Configure token_usage with input/output limits +- **GIVEN** a YAML eval case with: + ```yaml + evaluators: + - name: token-budget + type: token_usage + max_input: 8000 + max_output: 2000 + ``` +- **WHEN** the YAML is parsed +- **THEN** the eval case SHALL include a `token_usage` evaluator configuration +- **AND** the configuration SHALL preserve both limits + +#### Scenario: Reject invalid limits +- **GIVEN** a YAML eval case with: + ```yaml + evaluators: + - name: token-budget + type: token_usage + max_total: -1 + ``` +- **WHEN** the YAML is parsed +- **THEN** schema validation SHALL fail +- **AND** the error message SHALL mention that limits must be non-negative numbers + diff --git a/openspec/changes/add-token-usage-evaluator/tasks.md b/openspec/changes/add-token-usage-evaluator/tasks.md new file mode 100644 index 00000000..20100671 --- /dev/null +++ b/openspec/changes/add-token-usage-evaluator/tasks.md @@ -0,0 +1,13 @@ +# Tasks: Add token usage evaluator + +- [ ] Update spec deltas (`yaml-schema`, `evaluation`) +- [ ] Add `token_usage` evaluator config types +- [ ] Parse `token_usage` in evaluator parser (validate limits) +- [ ] Implement `TokenUsageEvaluator` (uses traceSummary.tokenUsage) +- [ ] Ensure traceSummary includes tokenUsage even when output messages are absent (if provider reports usage) +- [ ] Add unit tests for `TokenUsageEvaluator` and orchestrator propagation +- [ ] Update schema reference files (`eval-schema.json`) +- [ ] Add docs/reference examples and update skills references +- [ ] Add changeset +- [ ] Run `bun run build`, `bun run typecheck`, `bun run lint`, `bun test` + diff --git a/openspec/changes/update-code-judge-script-argv/design.md b/openspec/changes/update-code-judge-script-argv/design.md index 6d4c708f..35fb5b4a 100644 --- a/openspec/changes/update-code-judge-script-argv/design.md +++ b/openspec/changes/update-code-judge-script-argv/design.md @@ -23,9 +23,22 @@ evaluators: ### Execution +**Goals**: eliminate shell execution for argv-based configs and temp-file I/O while keeping the stdin/stdout JSON contract intact. + - The system spawns the process directly using argv tokens (no shell). - The system writes a single JSON payload to stdin (unchanged contract). -- The system captures stdout and parses the JSON result as today. +- The system captures stdout/stderr in-memory and parses stdout as JSON. +- Backward compatibility: if a string script is provided, it is converted to a shell argv (`["sh","-lc", "..."]` or `["cmd.exe","/c","..."]`) before execution. + +**Bun implementation notes**: + +- Prefer `Bun.spawn(cmd, { stdin: Uint8Array, stdout: "pipe", stderr: "pipe" })` where stdin is a single `Uint8Array` payload. Bun supports `TypedArray | DataView` as `stdin` inputs. This avoids incremental `stdin: "pipe"` flushing semantics and avoids piping `ReadableStream` to `stdin`, which has known compatibility issues in Bun. +- Drain `stdout` and `stderr` concurrently to avoid pipe-buffer backpressure deadlocks (same class of issue described in Node’s child_process docs for pipes with limited capacity). +- Enforce timeouts via Bun’s `timeout`/`killSignal` options (or an AbortSignal) so hung scripts are terminated deterministically. + +**Windows note**: + +- `.cmd`/`.bat` are not directly executable without a shell; users should explicitly invoke `cmd.exe /c` (or PowerShell) in argv when needed. The default path remains “no shell” for safety and determinism. ### Migration @@ -36,4 +49,4 @@ evaluators: - File-based evaluator input payload (could be added later if needed) - Supporting both string and argv forms simultaneously - +- Reintroducing temp-file based stdio capture as a fallback diff --git a/openspec/changes/update-code-judge-script-argv/proposal.md b/openspec/changes/update-code-judge-script-argv/proposal.md index 94ac8d94..58a76a37 100644 --- a/openspec/changes/update-code-judge-script-argv/proposal.md +++ b/openspec/changes/update-code-judge-script-argv/proposal.md @@ -4,15 +4,23 @@ `code_judge` evaluators currently accept a single shell command string. This is brittle (quoting/escaping), less portable across platforms, and encourages shell execution (`sh -c` / `cmd.exe /c`). Moving to an argv form makes evaluator execution more deterministic, safer, and easier to author. +Additionally, the current `code_judge` execution path relies on shell redirection and temp files to pass stdin and capture stdout/stderr. While pragmatic, this adds filesystem complexity and preserves shell execution risks. This change replaces that with direct argv spawning and in-memory stdio handling. + ## What Changes -- **BREAKING**: `code_judge` evaluator `script` changes from `string` (shell command) to `string[]` (argv tokens). +- `code_judge` evaluator `script` uses `string[]` (argv tokens), while legacy string scripts are converted to shell argv for backward compatibility. - Execution uses direct process spawning (no shell) and passes the evaluator input payload via stdin as today. +- Replace temp-file/stdout-stderr redirection with in-memory stdio capture. - Update all repo examples to use argv form. ## Impact - Affected specs: `yaml-schema`, `evaluation` - Affected code: YAML evaluator parsing/validation, code_judge execution, examples under `examples/` -- Breaking change: existing eval YAML using `script: "bun run ..."` must be updated to `script: ["bun", "run", ...]` +- Backward compatibility: existing eval YAML using `script: "bun run ..."` continues working via shell argv conversion (users should migrate to argv for determinism). + +## Out of Scope / Non-Goals +- Supporting both string and argv `script` forms simultaneously. +- Reintroducing temp-file based execution as a fallback. +- Changing the stdin JSON payload contract or the JSON shape returned on stdout. diff --git a/openspec/changes/update-code-judge-script-argv/specs/evaluation/spec.md b/openspec/changes/update-code-judge-script-argv/specs/evaluation/spec.md index ec8deff0..5fb866a3 100644 --- a/openspec/changes/update-code-judge-script-argv/specs/evaluation/spec.md +++ b/openspec/changes/update-code-judge-script-argv/specs/evaluation/spec.md @@ -12,3 +12,27 @@ The system SHALL allow external `code_judge` evaluators to score an eval case by - **AND** writes a single JSON payload to stdin - **AND** parses the script stdout as a JSON `EvaluationScore`. +#### Scenario: Non-zero exit surfaces stderr + exit code + +- **GIVEN** a `code_judge` evaluator configured with argv tokens +- **AND** the script writes a diagnostic message to stderr and exits non-zero +- **WHEN** the evaluator runs +- **THEN** the evaluation fails deterministically +- **AND** the error message includes the script exit code +- **AND** the error message includes captured stderr (or a truncated tail for large stderr). + +#### Scenario: Large stdin payload is delivered intact + +- **GIVEN** a `code_judge` evaluator configured with argv tokens +- **AND** the evaluator input payload written to stdin exceeds 1MB +- **WHEN** the evaluator runs +- **THEN** the script receives the complete stdin payload +- **AND** the system captures stdout and parses the JSON result. + +#### Scenario: Timeout terminates a hung evaluator + +- **GIVEN** a `code_judge` evaluator configured with argv tokens +- **AND** the script does not terminate within the configured timeout +- **WHEN** the evaluator runs +- **THEN** the system terminates the subprocess +- **AND** the evaluation fails with a timeout-specific error message. diff --git a/openspec/changes/update-code-judge-script-argv/specs/yaml-schema/spec.md b/openspec/changes/update-code-judge-script-argv/specs/yaml-schema/spec.md index 4412f945..e6bd3c1c 100644 --- a/openspec/changes/update-code-judge-script-argv/specs/yaml-schema/spec.md +++ b/openspec/changes/update-code-judge-script-argv/specs/yaml-schema/spec.md @@ -17,7 +17,7 @@ evaluators: - **THEN** schema validation succeeds - **AND** the evaluator configuration preserves the argv tokens exactly as provided. -#### Scenario: Reject string scripts +#### Scenario: Convert string scripts for backward compatibility - **GIVEN** an eval case with a `code_judge` evaluator configured with a string: ```yaml @@ -27,6 +27,12 @@ evaluators: script: bun run validate_risk_output.ts ``` - **WHEN** the YAML is parsed -- **THEN** schema validation fails -- **AND** the error message indicates that `script` must be an array of strings (argv tokens). +- **THEN** schema validation succeeds +- **AND** the system converts the string to a shell argv appropriate for the current platform. + +#### Scenario: Forbid implicit shell execution +- **GIVEN** an eval case with a `code_judge` evaluator +- **WHEN** the YAML is parsed +- **THEN** there is no schema-supported flag that enables implicit `shell: true` execution +- **AND** shell usage (if desired) requires the user to explicitly invoke a shell in argv tokens (e.g., `["cmd.exe", "/c", "..."]` or `["sh", "-lc", "..."]`). diff --git a/openspec/changes/update-code-judge-script-argv/tasks.md b/openspec/changes/update-code-judge-script-argv/tasks.md index bb81154d..b8a7cf49 100644 --- a/openspec/changes/update-code-judge-script-argv/tasks.md +++ b/openspec/changes/update-code-judge-script-argv/tasks.md @@ -1,22 +1,27 @@ ## 1. Schema & Parsing -- [ ] 1.1 Update evaluator schema so `code_judge.script` is `string[]` (argv) -- [ ] 1.2 Reject string `code_judge.script` with a clear error message -- [ ] 1.3 Update YAML parser docs/specs for `code_judge` +- [x] 1.1 Update evaluator schema so `code_judge.script` is `string[]` (argv) +- [x] 1.2 Convert string `code_judge.script` to shell argv for backward compatibility +- [x] 1.3 Update YAML parser docs/specs for `code_judge` ## 2. Execution -- [ ] 2.1 Execute `code_judge` via argv spawning (no shell) -- [ ] 2.2 Keep stdin JSON payload contract unchanged -- [ ] 2.3 Ensure cross-platform behavior (Windows/macOS/Linux) +- [x] 2.1 Add an argv-based subprocess helper (`execFileWithStdin` or equivalent) +- [x] 2.2 Execute `code_judge` via argv spawning (no shell) +- [x] 2.3 Keep stdin JSON payload contract unchanged +- [x] 2.4 Capture stdout/stderr in-memory (no temp files) +- [x] 2.5 Add timeout handling (kill/abort) for hung scripts +- [x] 2.6 Ensure cross-platform behavior (Windows/macOS/Linux) ## 3. Repository Updates -- [ ] 3.1 Update all examples using `code_judge.script` to argv form -- [ ] 3.2 Update any docs referencing string scripts +- [x] 3.1 Update all examples using `code_judge.script` to argv form +- [x] 3.2 Update any docs referencing string scripts ## 4. Tests -- [ ] 4.1 Add/adjust unit tests for evaluator parsing validation errors -- [ ] 4.2 Add/adjust execution tests for argv spawning - +- [x] 4.1 Add/adjust unit tests for evaluator parsing validation errors +- [x] 4.2 Add/adjust execution tests for argv spawning +- [x] 4.3 Test stderr capture + non-zero exit surfaced to user +- [x] 4.4 Test large stdin payload (>1MB) round-trip +- [x] 4.5 Test timeout kill behavior diff --git a/openspec/specs/yaml-schema/spec.md b/openspec/specs/yaml-schema/spec.md index 18c8d679..81ad521f 100644 --- a/openspec/specs/yaml-schema/spec.md +++ b/openspec/specs/yaml-schema/spec.md @@ -175,3 +175,30 @@ The YAML schema SHALL support an optional `weight` field on each entry in an eva - **WHEN** the YAML is parsed - **THEN** schema validation SHALL fail +### Requirement: Code judge scripts MUST use argv arrays + +The YAML schema SHALL accept `code_judge` evaluators with `script` defined as an array of argv tokens. + +#### Scenario: Configure code_judge with argv script +- **GIVEN** a YAML eval case with: + ```yaml + evaluators: + - name: my_code_check + type: code_judge + script: ["bun", "run", "validate_risk_output.ts"] + ``` +- **WHEN** the YAML is parsed +- **THEN** schema validation succeeds +- **AND** the evaluator configuration preserves the argv tokens. + +#### Scenario: Convert string scripts for backward compatibility +- **GIVEN** a YAML eval case with: + ```yaml + evaluators: + - name: my_code_check + type: code_judge + script: bun run validate_risk_output.ts + ``` +- **WHEN** the YAML is parsed +- **THEN** schema validation succeeds +- **AND** the system converts the string to a shell argv appropriate for the current platform. diff --git a/package.json b/package.json index 1c874882..2f43f88b 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "dev": "bun --filter agentv dev", "agentv": "bun apps/cli/dist/cli.js", "agentv:buildrun": "bun run build && bun apps/cli/dist/cli.js", + "eval:baseline-check": "bun scripts/check-eval-baselines.ts", "changeset": "changeset", "version": "changeset version", "subagent:link": "bun scripts/link-subagent.ts", diff --git a/packages/core/src/evaluation/evaluators.ts b/packages/core/src/evaluation/evaluators.ts index 36315eee..ca1fb412 100644 --- a/packages/core/src/evaluation/evaluators.ts +++ b/packages/core/src/evaluation/evaluators.ts @@ -1,7 +1,7 @@ import { generateText } from 'ai'; import { z } from 'zod'; -import { execShellWithStdin } from '../runtime/exec.js'; +import { execFileWithStdin, execShellWithStdin } from '../runtime/exec.js'; import { toSnakeCaseDeep } from './case-conversion.js'; import type { ResolvedTarget } from './providers/targets.js'; import { @@ -18,11 +18,16 @@ import type { TraceSummary, } from './trace.js'; import type { + CostEvaluatorConfig, EvalCase, EvaluationVerdict, EvaluatorConfig, + FieldAccuracyEvaluatorConfig, + FieldConfig, JsonObject, + LatencyEvaluatorConfig, RubricItem, + TokenUsageEvaluatorConfig, } from './types.js'; export type { EvaluationVerdict }; @@ -430,22 +435,26 @@ function isNonEmptyString(value: unknown): value is string { // Code Evaluator export interface CodeEvaluatorOptions { - readonly script: string; + readonly script: readonly string[]; readonly cwd?: string; readonly agentTimeoutMs?: number; + /** Pass-through configuration from YAML (any unrecognized properties) */ + readonly config?: Record; } export class CodeEvaluator implements Evaluator { readonly kind = 'code'; - private readonly script: string; + private readonly script: readonly string[]; private readonly cwd?: string; private readonly agentTimeoutMs?: number; + private readonly config?: Record; constructor(options: CodeEvaluatorOptions) { this.script = options.script; this.cwd = options.cwd; this.agentTimeoutMs = options.agentTimeoutMs; + this.config = options.config; } async evaluate(context: EvaluationContext): Promise { @@ -463,6 +472,8 @@ export class CodeEvaluator implements Evaluator { ), input_messages: context.evalCase.input_messages, trace_summary: context.traceSummary ?? null, + // Pass-through config from YAML (any unrecognized properties) + config: this.config ?? null, }; // Recursively convert all nested objects to snake_case for Python compatibility @@ -550,18 +561,18 @@ function calculateRubricScore( // Helper functions for CodeEvaluator async function executeScript( - scriptPath: string, + scriptPath: readonly string[] | string, input: string, agentTimeoutMs?: number, cwd?: string, ): Promise { - const { stdout, stderr, exitCode } = await execShellWithStdin(scriptPath, input, { - cwd, - timeoutMs: agentTimeoutMs, - }); + const { stdout, stderr, exitCode } = + typeof scriptPath === 'string' + ? await execShellWithStdin(scriptPath, input, { cwd, timeoutMs: agentTimeoutMs }) + : await execFileWithStdin(scriptPath, input, { cwd, timeoutMs: agentTimeoutMs }); if (exitCode !== 0) { - const trimmedErr = stderr.trim(); + const trimmedErr = formatStderr(stderr); throw new Error( trimmedErr.length > 0 ? `Code evaluator exited with code ${exitCode}: ${trimmedErr}` @@ -572,6 +583,16 @@ async function executeScript( return stdout.trim(); } +function formatStderr(stderr: string): string { + const trimmed = stderr.trim(); + const maxLength = 2000; + if (trimmed.length <= maxLength) { + return trimmed; + } + const tail = trimmed.slice(-maxLength); + return `...(truncated, last ${maxLength} chars)\n${tail}`; +} + function parseJsonSafe(payload: string): Record | undefined { try { return JSON.parse(payload) as Record; @@ -894,6 +915,574 @@ export class ToolTrajectoryEvaluator implements Evaluator { } } +// Field Accuracy Evaluator + +export interface FieldAccuracyEvaluatorOptions { + readonly config: FieldAccuracyEvaluatorConfig; +} + +/** Result from evaluating a single field */ +interface FieldResult { + readonly path: string; + readonly score: number; + readonly weight: number; + readonly hit: boolean; + readonly message: string; +} + +/** + * Default date formats to try when parsing dates. + * Ordered from most specific to least specific. + */ +const DEFAULT_DATE_FORMATS = [ + 'YYYY-MM-DDTHH:mm:ssZ', // ISO with timezone + 'YYYY-MM-DDTHH:mm:ss', // ISO with time + 'YYYY-MM-DD', // ISO date + 'DD-MMM-YYYY', // Localized (e.g., "15-JAN-2025") + 'MM/DD/YYYY', // US format + 'DD/MM/YYYY', // EU format + 'MM-DD-YYYY', // US with dashes + 'DD-MM-YYYY', // EU with dashes +]; + +/** + * Month name mappings for parsing localized dates. + */ +const MONTH_NAMES: Record = { + jan: 0, + january: 0, + feb: 1, + february: 1, + mar: 2, + march: 2, + apr: 3, + april: 3, + may: 4, + jun: 5, + june: 5, + jul: 6, + july: 6, + aug: 7, + august: 7, + sep: 8, + sept: 8, + september: 8, + oct: 9, + october: 9, + nov: 10, + november: 10, + dec: 11, + december: 11, +}; + +/** + * FieldAccuracyEvaluator compares extracted structured data against expected values + * with configurable matching strategies (exact, fuzzy, numeric_tolerance, date). + */ +export class FieldAccuracyEvaluator implements Evaluator { + readonly kind = 'field_accuracy'; + + private readonly config: FieldAccuracyEvaluatorConfig; + + constructor(options: FieldAccuracyEvaluatorOptions) { + this.config = options.config; + } + + evaluate(context: EvaluationContext): EvaluationScore { + const { evalCase, candidate } = context; + + // Parse candidate answer as JSON + let candidateData: Record; + try { + candidateData = parseJsonFromTextSafe(candidate); + } catch { + return { + score: 0, + verdict: 'fail', + hits: [], + misses: ['Failed to parse candidate answer as JSON'], + expectedAspectCount: this.config.fields.length, + reasoning: 'Candidate answer is not valid JSON', + }; + } + + // Extract expected data from expected_messages + const expectedData = this.extractExpectedData(evalCase.expected_messages); + if (!expectedData) { + return { + score: 0, + verdict: 'fail', + hits: [], + misses: ['No expected data found in expected_messages'], + expectedAspectCount: this.config.fields.length, + reasoning: 'Could not extract expected data from expected_messages', + }; + } + + // Evaluate each field + const fieldResults: FieldResult[] = []; + for (const fieldConfig of this.config.fields) { + const result = this.evaluateField(fieldConfig, candidateData, expectedData); + fieldResults.push(result); + } + + // Aggregate results + return this.aggregateResults(fieldResults); + } + + /** + * Extract expected data from expected_messages array. + * Looks for the last assistant message with content. + */ + private extractExpectedData( + expectedMessages: readonly JsonObject[], + ): Record | undefined { + // Find the last assistant message with content + for (let i = expectedMessages.length - 1; i >= 0; i--) { + const message = expectedMessages[i]; + if (message.role === 'assistant' && message.content) { + if (typeof message.content === 'object' && message.content !== null) { + return message.content as Record; + } + // If content is a string, try to parse it as JSON + if (typeof message.content === 'string') { + try { + return parseJsonFromTextSafe(message.content); + } catch { + // Parsing failed, continue to next message + } + } + } + } + return undefined; + } + + /** + * Evaluate a single field against the expected value. + */ + private evaluateField( + fieldConfig: FieldConfig, + candidateData: Record, + expectedData: Record, + ): FieldResult { + const { path, match, required = true, weight = 1.0 } = fieldConfig; + + const candidateValue = resolvePath(candidateData, path); + const expectedValue = resolvePath(expectedData, path); + + // Handle missing expected value + if (expectedValue === undefined) { + // If the expected value is missing, we can't compare + return { + path, + score: 1.0, // No expected value means no comparison needed + weight, + hit: true, + message: `${path}: no expected value`, + }; + } + + // Handle missing candidate value + if (candidateValue === undefined) { + if (required) { + return { + path, + score: 0, + weight, + hit: false, + message: `${path} (required, missing)`, + }; + } + // Optional field missing - don't count in aggregation + return { + path, + score: 1.0, // Don't penalize missing optional fields + weight: 0, // Zero weight means it won't affect the score + hit: true, + message: `${path}: optional field missing`, + }; + } + + // Compare based on match type + switch (match) { + case 'exact': + return this.compareExact(path, candidateValue, expectedValue, weight); + case 'numeric_tolerance': + return this.compareNumericTolerance( + path, + candidateValue, + expectedValue, + fieldConfig, + weight, + ); + case 'date': + return this.compareDate(path, candidateValue, expectedValue, fieldConfig, weight); + default: + return { + path, + score: 0, + weight, + hit: false, + message: `${path}: unknown match type "${match}"`, + }; + } + } + + /** + * Exact equality comparison. + */ + private compareExact( + path: string, + candidateValue: unknown, + expectedValue: unknown, + weight: number, + ): FieldResult { + // Deep equality for objects and arrays + if (deepEqual(candidateValue, expectedValue)) { + return { + path, + score: 1.0, + weight, + hit: true, + message: path, + }; + } + + // Type mismatch + if (typeof candidateValue !== typeof expectedValue) { + return { + path, + score: 0, + weight, + hit: false, + message: `${path} (type mismatch: got ${typeof candidateValue}, expected ${typeof expectedValue})`, + }; + } + + return { + path, + score: 0, + weight, + hit: false, + message: `${path} (value mismatch)`, + }; + } + + /** + * Numeric comparison with absolute or relative tolerance. + */ + private compareNumericTolerance( + path: string, + candidateValue: unknown, + expectedValue: unknown, + fieldConfig: FieldConfig, + weight: number, + ): FieldResult { + const { tolerance = 0, relative = false } = fieldConfig; + + const candidateNum = toNumber(candidateValue); + const expectedNum = toNumber(expectedValue); + + if (candidateNum === null || expectedNum === null) { + return { + path, + score: 0, + weight, + hit: false, + message: `${path} (non-numeric value)`, + }; + } + + if (!Number.isFinite(candidateNum) || !Number.isFinite(expectedNum)) { + return { + path, + score: 0, + weight, + hit: false, + message: `${path} (invalid numeric value)`, + }; + } + + const diff = Math.abs(candidateNum - expectedNum); + let withinTolerance: boolean; + + if (relative) { + // Relative tolerance: |actual - expected| / |expected| <= tolerance + // Handle division by zero for expected === 0 + const relativeDiff = expectedNum === 0 ? diff : diff / Math.abs(expectedNum); + withinTolerance = relativeDiff <= tolerance; + } else { + // Absolute tolerance: |actual - expected| <= tolerance + withinTolerance = diff <= tolerance; + } + + if (withinTolerance) { + return { + path, + score: 1.0, + weight, + hit: true, + message: `${path} (within tolerance: diff=${diff.toFixed(2)})`, + }; + } + + return { + path, + score: 0, + weight, + hit: false, + message: `${path} (outside tolerance: diff=${diff.toFixed(2)}, tolerance=${tolerance})`, + }; + } + + /** + * Date comparison with format normalization. + */ + private compareDate( + path: string, + candidateValue: unknown, + expectedValue: unknown, + fieldConfig: FieldConfig, + weight: number, + ): FieldResult { + const formats = fieldConfig.formats ?? DEFAULT_DATE_FORMATS; + + const candidateDate = parseDate(String(candidateValue), formats); + const expectedDate = parseDate(String(expectedValue), formats); + + if (candidateDate === null) { + return { + path, + score: 0, + weight, + hit: false, + message: `${path} (unparseable candidate date)`, + }; + } + + if (expectedDate === null) { + return { + path, + score: 0, + weight, + hit: false, + message: `${path} (unparseable expected date)`, + }; + } + + // Compare dates by year, month, and day (ignore time component) + if ( + candidateDate.getFullYear() === expectedDate.getFullYear() && + candidateDate.getMonth() === expectedDate.getMonth() && + candidateDate.getDate() === expectedDate.getDate() + ) { + return { + path, + score: 1.0, + weight, + hit: true, + message: path, + }; + } + + return { + path, + score: 0, + weight, + hit: false, + message: `${path} (date mismatch: got ${formatDateISO(candidateDate)}, expected ${formatDateISO(expectedDate)})`, + }; + } + + /** + * Aggregate field results using configured strategy. + */ + private aggregateResults(results: readonly FieldResult[]): EvaluationScore { + const aggregation = this.config.aggregation ?? 'weighted_average'; + const hits: string[] = []; + const misses: string[] = []; + + for (const result of results) { + if (result.hit) { + hits.push(result.message); + } else { + misses.push(result.message); + } + } + + let score: number; + if (aggregation === 'all_or_nothing') { + // All fields must pass for score 1.0 + score = misses.length === 0 ? 1.0 : 0.0; + } else { + // weighted_average (default) + const totalWeight = results.reduce((sum, r) => sum + r.weight, 0); + if (totalWeight === 0) { + score = results.length === 0 ? 1.0 : 0.0; + } else { + const weightedSum = results.reduce((sum, r) => sum + r.score * r.weight, 0); + score = weightedSum / totalWeight; + } + } + + const reasoning = `${hits.length}/${results.length} fields matched`; + + return { + score: clampScore(score), + verdict: scoreToVerdict(score), + hits: hits.slice(0, 4), + misses: misses.slice(0, 4), + expectedAspectCount: results.length, + reasoning, + }; + } +} + +/** + * Resolve a dot-notation path (with array indexing) to a value. + * Example: "invoice.line_items[0].amount" + */ +function resolvePath(obj: Record, path: string): unknown { + if (!path || !obj) { + return undefined; + } + + // Split on dots and array brackets + const parts = path.split(/\.|\[|\]/).filter((p) => p.length > 0); + let current: unknown = obj; + + for (const part of parts) { + if (current === null || current === undefined) { + return undefined; + } + + if (typeof current !== 'object') { + return undefined; + } + + const isIndex = /^\d+$/.test(part); + if (isIndex && Array.isArray(current)) { + current = current[Number.parseInt(part, 10)]; + } else { + current = (current as Record)[part]; + } + } + + return current; +} + +/** + * Convert a value to a number, returning null if not possible. + */ +function toNumber(value: unknown): number | null { + if (typeof value === 'number') { + return value; + } + if (typeof value === 'string') { + const num = Number.parseFloat(value); + return Number.isNaN(num) ? null : num; + } + return null; +} + +/** + * Parse a date string using the specified formats. + * Returns null if parsing fails. + * + * Date format disambiguation: + * - If only US formats (MM/DD/YYYY) are specified, parses as US + * - If only EU formats (DD/MM/YYYY) are specified, parses as EU + * - If both or neither are specified, attempts to infer from values: + * - If first number > 12, assumes EU format (day first) + * - If second number > 12, assumes US format (month first) + * - If ambiguous (both <= 12), defaults to US format (MM/DD/YYYY) + */ +function parseDate(dateStr: string, formats: readonly string[]): Date | null { + if (!dateStr) return null; + + const trimmed = dateStr.trim(); + + // Try ISO format first (JavaScript native) + const isoDate = new Date(trimmed); + if (!Number.isNaN(isoDate.getTime())) { + return isoDate; + } + + // Try localized format (DD-MMM-YYYY) + const localizedMatch = trimmed.match(/^(\d{1,2})-([A-Za-z]{3,9})-(\d{4})$/); + if (localizedMatch) { + const day = Number.parseInt(localizedMatch[1], 10); + const monthName = localizedMatch[2].toLowerCase(); + const year = Number.parseInt(localizedMatch[3], 10); + const month = MONTH_NAMES[monthName]; + if (month !== undefined) { + return new Date(year, month, day); + } + } + + // Try US format (MM/DD/YYYY or MM-DD-YYYY) + const usMatch = trimmed.match(/^(\d{1,2})[\/\-](\d{1,2})[\/\-](\d{4})$/); + if (usMatch) { + // Check if first or second number is likely the month + // Assume MM/DD/YYYY for formats array containing "MM/DD/YYYY" or "MM-DD-YYYY" + const hasUSFormat = formats.some((f) => f.includes('MM/DD') || f.includes('MM-DD')); + const hasEUFormat = formats.some((f) => f.includes('DD/MM') || f.includes('DD-MM')); + + if (hasUSFormat && !hasEUFormat) { + const month = Number.parseInt(usMatch[1], 10) - 1; + const day = Number.parseInt(usMatch[2], 10); + const year = Number.parseInt(usMatch[3], 10); + if (month >= 0 && month <= 11 && day >= 1 && day <= 31) { + return new Date(year, month, day); + } + } else if (hasEUFormat && !hasUSFormat) { + const day = Number.parseInt(usMatch[1], 10); + const month = Number.parseInt(usMatch[2], 10) - 1; + const year = Number.parseInt(usMatch[3], 10); + if (month >= 0 && month <= 11 && day >= 1 && day <= 31) { + return new Date(year, month, day); + } + } else { + // Ambiguous - try to infer from values + const num1 = Number.parseInt(usMatch[1], 10); + const num2 = Number.parseInt(usMatch[2], 10); + const year = Number.parseInt(usMatch[3], 10); + + // If first number > 12, it must be day (EU format) + if (num1 > 12 && num2 <= 12) { + return new Date(year, num2 - 1, num1); + } + // If second number > 12, it must be day (US format) + if (num2 > 12 && num1 <= 12) { + return new Date(year, num1 - 1, num2); + } + // Default to US format + if (num1 <= 12 && num2 <= 31) { + return new Date(year, num1 - 1, num2); + } + } + } + + return null; +} + +/** + * Format a date as ISO date string (YYYY-MM-DD). + */ +function formatDateISO(date: Date): string { + return date.toISOString().split('T')[0]; +} + +/** + * Safely parse JSON from text, handling code blocks. + */ +function parseJsonFromTextSafe(text: string): Record { + const cleaned = typeof text === 'string' ? text.replace(/```json\n?|```/g, '').trim() : ''; + const match = cleaned.match(/\{[\s\S]*\}/); + const blob = match?.[0] ?? cleaned; + return JSON.parse(blob) as Record; +} + // Composite Evaluator export interface EvaluatorFactory { @@ -1193,3 +1782,236 @@ export class CompositeEvaluator implements Evaluator { } } } + +// ---------------------------------------------------------------------------- +// Latency Evaluator +// ---------------------------------------------------------------------------- + +export interface LatencyEvaluatorOptions { + readonly config: LatencyEvaluatorConfig; +} + +/** + * Evaluator that checks execution duration against a threshold. + * Uses traceSummary.durationMs from the evaluation context. + */ +export class LatencyEvaluator implements Evaluator { + readonly kind = 'latency'; + + private readonly config: LatencyEvaluatorConfig; + + constructor(options: LatencyEvaluatorOptions) { + this.config = options.config; + } + + evaluate(context: EvaluationContext): EvaluationScore { + const { threshold } = this.config; + const durationMs = context.traceSummary?.durationMs; + + // If no duration data available, we can't evaluate + if (durationMs === undefined) { + return { + score: 0, + verdict: 'fail', + hits: [], + misses: ['No duration data available in trace'], + expectedAspectCount: 1, + reasoning: 'Execution duration not reported by provider', + evaluatorRawRequest: { + type: 'latency', + threshold, + durationMs: null, + }, + }; + } + + const passed = durationMs <= threshold; + const score = passed ? 1 : 0; + + return { + score, + verdict: passed ? 'pass' : 'fail', + hits: passed ? [`Duration ${durationMs}ms <= ${threshold}ms threshold`] : [], + misses: passed ? [] : [`Duration ${durationMs}ms > ${threshold}ms threshold`], + expectedAspectCount: 1, + reasoning: `Execution took ${durationMs}ms (threshold: ${threshold}ms)`, + evaluatorRawRequest: { + type: 'latency', + threshold, + durationMs, + }, + }; + } +} + +// ---------------------------------------------------------------------------- +// Cost Evaluator +// ---------------------------------------------------------------------------- + +export interface CostEvaluatorOptions { + readonly config: CostEvaluatorConfig; +} + +/** + * Evaluator that checks execution cost against a budget. + * Uses traceSummary.costUsd from the evaluation context. + */ +export class CostEvaluator implements Evaluator { + readonly kind = 'cost'; + + private readonly config: CostEvaluatorConfig; + + constructor(options: CostEvaluatorOptions) { + this.config = options.config; + } + + evaluate(context: EvaluationContext): EvaluationScore { + const { budget } = this.config; + const costUsd = context.traceSummary?.costUsd; + + // If no cost data available, we can't evaluate + if (costUsd === undefined) { + return { + score: 0, + verdict: 'fail', + hits: [], + misses: ['No cost data available in trace'], + expectedAspectCount: 1, + reasoning: 'Execution cost not reported by provider', + evaluatorRawRequest: { + type: 'cost', + budget, + costUsd: null, + }, + }; + } + + const passed = costUsd <= budget; + const score = passed ? 1 : 0; + + // Format cost for display + const formatCost = (n: number) => `$${n.toFixed(4)}`; + + return { + score, + verdict: passed ? 'pass' : 'fail', + hits: passed ? [`Cost ${formatCost(costUsd)} <= ${formatCost(budget)} budget`] : [], + misses: passed ? [] : [`Cost ${formatCost(costUsd)} > ${formatCost(budget)} budget`], + expectedAspectCount: 1, + reasoning: `Execution cost ${formatCost(costUsd)} (budget: ${formatCost(budget)})`, + evaluatorRawRequest: { + type: 'cost', + budget, + costUsd, + }, + }; + } +} + +// ---------------------------------------------------------------------------- +// Token Usage Evaluator +// ---------------------------------------------------------------------------- + +export interface TokenUsageEvaluatorOptions { + readonly config: TokenUsageEvaluatorConfig; +} + +/** + * Evaluator that checks provider-reported token usage against configured limits. + * Uses traceSummary.tokenUsage from the evaluation context. + */ +export class TokenUsageEvaluator implements Evaluator { + readonly kind = 'token_usage'; + + private readonly config: TokenUsageEvaluatorConfig; + + constructor(options: TokenUsageEvaluatorOptions) { + this.config = options.config; + } + + evaluate(context: EvaluationContext): EvaluationScore { + const usage = context.traceSummary?.tokenUsage; + + const maxTotal = this.config.max_total; + const maxInput = this.config.max_input; + const maxOutput = this.config.max_output; + + const expectedAspectCount = Math.max( + [maxTotal, maxInput, maxOutput].filter((v) => typeof v === 'number').length, + 1, + ); + + if (!usage) { + return { + score: 0, + verdict: 'fail', + hits: [], + misses: ['No token usage data available in trace'], + expectedAspectCount, + reasoning: 'Token usage not reported by provider', + evaluatorRawRequest: { + type: 'token_usage', + max_total: maxTotal ?? null, + max_input: maxInput ?? null, + max_output: maxOutput ?? null, + tokenUsage: null, + }, + }; + } + + const input = usage.input; + const output = usage.output; + const cached = usage.cached ?? 0; + const total = input + output + cached; + + const hits: string[] = []; + const misses: string[] = []; + + if (typeof maxInput === 'number') { + if (input <= maxInput) { + hits.push(`Input tokens ${input} <= ${maxInput}`); + } else { + misses.push(`Input tokens ${input} > ${maxInput}`); + } + } + + if (typeof maxOutput === 'number') { + if (output <= maxOutput) { + hits.push(`Output tokens ${output} <= ${maxOutput}`); + } else { + misses.push(`Output tokens ${output} > ${maxOutput}`); + } + } + + if (typeof maxTotal === 'number') { + if (total <= maxTotal) { + hits.push(`Total tokens ${total} <= ${maxTotal}`); + } else { + misses.push(`Total tokens ${total} > ${maxTotal}`); + } + } + + const passed = misses.length === 0; + + return { + score: passed ? 1 : 0, + verdict: passed ? 'pass' : 'fail', + hits, + misses, + expectedAspectCount, + reasoning: `token_usage input=${input}, output=${output}, cached=${cached}, total=${total}`, + evaluatorRawRequest: { + type: 'token_usage', + max_total: maxTotal ?? null, + max_input: maxInput ?? null, + max_output: maxOutput ?? null, + tokenUsage: { + input, + output, + cached, + total, + }, + }, + }; + } +} diff --git a/packages/core/src/evaluation/loaders/evaluator-parser.ts b/packages/core/src/evaluation/loaders/evaluator-parser.ts index a43f0e5f..b47f4bf1 100644 --- a/packages/core/src/evaluation/loaders/evaluator-parser.ts +++ b/packages/core/src/evaluation/loaders/evaluator-parser.ts @@ -52,7 +52,24 @@ export async function parseEvaluators( } if (typeValue === 'code_judge') { - const script = asString(rawEvaluator.script); + let script: string[] | undefined; + const rawScript = rawEvaluator.script; + + if (typeof rawScript === 'string') { + const trimmed = rawScript.trim(); + if (trimmed.length === 0) { + throw new Error( + `Invalid code_judge script for evaluator '${name}' in '${evalId}': script cannot be empty`, + ); + } + script = parseCommandToArgv(trimmed); + } else { + script = asStringArray( + rawScript, + `code_judge script for evaluator '${name}' in '${evalId}'`, + ); + } + if (!script) { logWarning(`Skipping code_judge evaluator '${name}' in '${evalId}': missing script`); continue; @@ -79,6 +96,15 @@ export async function parseEvaluators( resolvedCwd = searchRoots[0]; } + // Collect unrecognized properties as pass-through config + const knownProps = new Set(['name', 'type', 'script', 'cwd', 'weight']); + const config: Record = {}; + for (const [key, value] of Object.entries(rawEvaluator)) { + if (!knownProps.has(key) && value !== undefined) { + config[key] = value as JsonValue; + } + } + evaluators.push({ name, type: 'code', @@ -86,6 +112,7 @@ export async function parseEvaluators( cwd, resolvedCwd, ...(weight !== undefined ? { weight } : {}), + ...(Object.keys(config).length > 0 ? { config } : {}), }); continue; } @@ -299,6 +326,171 @@ export async function parseEvaluators( continue; } + if (typeValue === 'field_accuracy') { + const rawFields = rawEvaluator.fields; + if (!Array.isArray(rawFields)) { + logWarning( + `Skipping field_accuracy evaluator '${name}' in '${evalId}': missing fields array`, + ); + continue; + } + + if (rawFields.length === 0) { + logWarning( + `Skipping field_accuracy evaluator '${name}' in '${evalId}': fields array is empty`, + ); + continue; + } + + const fields: import('../types.js').FieldConfig[] = []; + for (const rawField of rawFields) { + if (!isJsonObject(rawField)) { + logWarning( + `Skipping invalid field entry in field_accuracy evaluator '${name}' (expected object)`, + ); + continue; + } + + const fieldPath = asString(rawField.path); + const match = asString(rawField.match); + + if (!fieldPath) { + logWarning( + `Skipping field without path in field_accuracy evaluator '${name}' in '${evalId}'`, + ); + continue; + } + + if (!match || !isValidFieldMatchType(match)) { + logWarning( + `Skipping field '${fieldPath}' with invalid match type '${match}' in evaluator '${name}' (must be exact, numeric_tolerance, or date). For fuzzy matching, use a code_judge evaluator.`, + ); + continue; + } + + const fieldConfig: import('../types.js').FieldConfig = { + path: fieldPath, + match, + ...(typeof rawField.required === 'boolean' ? { required: rawField.required } : {}), + ...(typeof rawField.weight === 'number' ? { weight: rawField.weight } : {}), + ...(typeof rawField.tolerance === 'number' ? { tolerance: rawField.tolerance } : {}), + ...(typeof rawField.relative === 'boolean' ? { relative: rawField.relative } : {}), + ...(Array.isArray(rawField.formats) + ? { formats: rawField.formats.filter((f): f is string => typeof f === 'string') } + : {}), + }; + + fields.push(fieldConfig); + } + + if (fields.length === 0) { + logWarning( + `Skipping field_accuracy evaluator '${name}' in '${evalId}': no valid fields found`, + ); + continue; + } + + const aggregation = asString(rawEvaluator.aggregation); + const validAggregation = isValidFieldAggregationType(aggregation) ? aggregation : undefined; + + const weight = validateWeight(rawEvaluator.weight, name, evalId); + + evaluators.push({ + name, + type: 'field_accuracy', + fields, + ...(validAggregation ? { aggregation: validAggregation } : {}), + ...(weight !== undefined ? { weight } : {}), + }); + continue; + } + + if (typeValue === 'latency') { + const threshold = rawEvaluator.threshold; + if (typeof threshold !== 'number' || threshold < 0) { + logWarning( + `Skipping latency evaluator '${name}' in '${evalId}': threshold must be a non-negative number`, + ); + continue; + } + + const weight = validateWeight(rawEvaluator.weight, name, evalId); + + evaluators.push({ + name, + type: 'latency', + threshold, + ...(weight !== undefined ? { weight } : {}), + }); + continue; + } + + if (typeValue === 'cost') { + const budget = rawEvaluator.budget; + if (typeof budget !== 'number' || budget < 0) { + logWarning( + `Skipping cost evaluator '${name}' in '${evalId}': budget must be a non-negative number`, + ); + continue; + } + + const weight = validateWeight(rawEvaluator.weight, name, evalId); + + evaluators.push({ + name, + type: 'cost', + budget, + ...(weight !== undefined ? { weight } : {}), + }); + continue; + } + + if (typeValue === 'token_usage') { + const maxTotal = rawEvaluator.max_total ?? rawEvaluator.maxTotal; + const maxInput = rawEvaluator.max_input ?? rawEvaluator.maxInput; + const maxOutput = rawEvaluator.max_output ?? rawEvaluator.maxOutput; + + const limits = [ + ['max_total', maxTotal], + ['max_input', maxInput], + ['max_output', maxOutput], + ] as const; + + const validLimits: Partial> = {}; + + for (const [key, raw] of limits) { + if (raw === undefined) continue; + if (typeof raw !== 'number' || !Number.isFinite(raw) || raw < 0) { + logWarning( + `Skipping token_usage evaluator '${name}' in '${evalId}': ${key} must be a non-negative finite number`, + ); + continue; + } + validLimits[key] = raw; + } + + if ( + validLimits.max_total === undefined && + validLimits.max_input === undefined && + validLimits.max_output === undefined + ) { + logWarning( + `Skipping token_usage evaluator '${name}' in '${evalId}': must set at least one of max_total, max_input, max_output`, + ); + continue; + } + + const weight = validateWeight(rawEvaluator.weight, name, evalId); + + evaluators.push({ + name, + type: 'token_usage', + ...validLimits, + ...(weight !== undefined ? { weight } : {}), + }); + continue; + } + const prompt = asString(rawEvaluator.prompt); let promptPath: string | undefined; if (prompt) { @@ -396,6 +588,40 @@ function asString(value: unknown): string | undefined { return typeof value === 'string' ? value : undefined; } +function asStringArray(value: unknown, description: string): string[] | undefined { + if (value === undefined) { + return undefined; + } + + if (!Array.isArray(value)) { + throw new Error(`${description} must be an array of strings (argv tokens)`); + } + + if (value.length === 0) { + throw new Error(`${description} cannot be empty`); + } + + const result: string[] = []; + for (const [index, entry] of value.entries()) { + if (typeof entry !== 'string') { + throw new Error(`${description}[${index}] must be a string`); + } + if (entry.trim().length === 0) { + throw new Error(`${description}[${index}] cannot be empty`); + } + result.push(entry); + } + + return result; +} + +function parseCommandToArgv(command: string): string[] { + if (process.platform === 'win32') { + return ['cmd.exe', '/c', command]; + } + return ['sh', '-lc', command]; +} + function isJsonObject(value: unknown): value is JsonObject { return typeof value === 'object' && value !== null && !Array.isArray(value); } @@ -443,3 +669,17 @@ function validateWeight( return rawWeight; } + +const VALID_FIELD_MATCH_TYPES = new Set(['exact', 'numeric_tolerance', 'date']); + +function isValidFieldMatchType(value: unknown): value is import('../types.js').FieldMatchType { + return typeof value === 'string' && VALID_FIELD_MATCH_TYPES.has(value); +} + +const VALID_FIELD_AGGREGATION_TYPES = new Set(['weighted_average', 'all_or_nothing']); + +function isValidFieldAggregationType( + value: unknown, +): value is import('../types.js').FieldAggregationType { + return typeof value === 'string' && VALID_FIELD_AGGREGATION_TYPES.has(value); +} diff --git a/packages/core/src/evaluation/orchestrator.ts b/packages/core/src/evaluation/orchestrator.ts index 62837030..4fb258f9 100644 --- a/packages/core/src/evaluation/orchestrator.ts +++ b/packages/core/src/evaluation/orchestrator.ts @@ -7,9 +7,13 @@ import { type ChildEvaluatorResult, CodeEvaluator, CompositeEvaluator, + CostEvaluator, type EvaluationScore, type Evaluator, + FieldAccuracyEvaluator, + LatencyEvaluator, LlmJudgeEvaluator, + TokenUsageEvaluator, ToolTrajectoryEvaluator, } from './evaluators.js'; import { readJsonFile, readTextFile } from './file-utils.js'; @@ -31,19 +35,27 @@ import { mergeExecutionMetrics, } from './trace.js'; import type { + CostEvaluatorConfig, EvalCase, EvaluationResult, EvaluationVerdict, EvaluatorConfig, EvaluatorKind, EvaluatorResult, + FieldAccuracyEvaluatorConfig, JsonObject, JsonValue, + LatencyEvaluatorConfig, + TokenUsageEvaluatorConfig, } from './types.js'; import { type PromptInputs, buildPromptInputs, loadEvalCases } from './yaml-parser.js'; type MaybePromise = T | Promise; +function usesFileReferencePrompt(provider: Provider): boolean { + return isAgentProvider(provider) || provider.kind === 'cli'; +} + export interface EvaluationCache { get(key: string): MaybePromise; set(key: string, value: ProviderResponse): MaybePromise; @@ -309,7 +321,8 @@ export async function runEvaluation( } else { // Build error result for rejected promise const evalCase = filteredEvalCases[i]; - const promptInputs = await buildPromptInputs(evalCase); + const formattingMode = usesFileReferencePrompt(primaryProvider) ? 'agent' : 'lm'; + const promptInputs = await buildPromptInputs(evalCase, formattingMode); const errorResult = buildErrorResult( evalCase, target.name, @@ -358,7 +371,7 @@ async function runBatchEvaluation(options: { // Prepare prompt inputs up front so we can reuse them for grading. const promptInputsList: PromptInputs[] = []; - const formattingMode = isAgentProvider(provider) ? 'agent' : 'lm'; + const formattingMode = usesFileReferencePrompt(provider) ? 'agent' : 'lm'; for (const evalCase of evalCases) { const promptInputs = await buildPromptInputs(evalCase, formattingMode); @@ -412,7 +425,21 @@ async function runBatchEvaluation(options: { // Extract outputMessages from batch response const outputMessages = providerResponse.outputMessages; - const baseSummary = outputMessages ? computeTraceSummary(outputMessages) : undefined; + const hasExecutionMetrics = + providerResponse.tokenUsage !== undefined || + providerResponse.costUsd !== undefined || + providerResponse.durationMs !== undefined; + + const baseSummary = outputMessages + ? computeTraceSummary(outputMessages) + : hasExecutionMetrics + ? { + eventCount: 0, + toolNames: [], + toolCallsByName: {}, + errorCount: 0, + } + : undefined; // Merge execution metrics from provider response const traceSummary = baseSummary ? mergeExecutionMetrics(baseSummary, { @@ -501,7 +528,7 @@ export async function runEvalCase(options: RunEvalCaseOptions): Promise { - stdout: ReadableStream; - stderr: ReadableStream; - exited: Promise; - kill: () => void; - }) - | undefined { - const bunSpawn = (globalThis as { Bun?: { spawn?: unknown } }).Bun?.spawn; - return typeof bunSpawn === 'function' ? (bunSpawn as ReturnType) : undefined; +function shellEscapePath(value: string): string { + if (process.platform === 'win32') { + // Very small escape helper for file paths in cmd.exe context. + // Wrap in double-quotes and escape existing double-quotes. + return `"${value.replaceAll('"', '""')}"`; + } + // POSIX: single-quote escape (close/open around embedded single quotes). + return `'${value.replaceAll("'", `'\"'\"'`)}'`; +} + +export async function execFileWithStdin( + argv: readonly string[], + stdinPayload: string, + options: ExecOptions = {}, +): Promise<{ + readonly stdout: string; + readonly stderr: string; + readonly exitCode: number; +}> { + if (argv.length === 0) { + throw new Error('Executable argv must include at least one entry'); + } + + const command = [...argv]; + const encoder = new TextEncoder(); + const process = Bun.spawn(command, { + cwd: options.cwd, + stdin: encoder.encode(stdinPayload), + stdout: 'pipe', + stderr: 'pipe', + }); + + let timedOut = false; + const timeout = + options.timeoutMs !== undefined + ? setTimeout(() => { + timedOut = true; + process.kill('SIGKILL'); + }, options.timeoutMs) + : undefined; + + try { + const stdoutPromise = process.stdout + ? new Response(process.stdout).text() + : Promise.resolve(''); + const stderrPromise = process.stderr + ? new Response(process.stderr).text() + : Promise.resolve(''); + + const [stdout, stderr, exitCode] = await Promise.all([ + stdoutPromise, + stderrPromise, + process.exited, + ]); + + if (timedOut) { + throw new Error(`Process timed out after ${options.timeoutMs}ms`); + } + + return { + stdout: stdout.replace(/\r\n/g, '\n'), + stderr: stderr.replace(/\r\n/g, '\n'), + exitCode, + }; + } finally { + if (timeout !== undefined) { + clearTimeout(timeout); + } + } } /** * Execute a shell command with the given stdin payload. * * Why this exists: - * - Under Bun, using `node:child_process` to pipe stdin to a subprocess can be unreliable. - * - Bun's native `Bun.spawn` reliably passes stdin and returns stdout/stderr streams. - * - Under Node, fall back to `node:child_process` for compatibility. + * - Some providers/scripts (notably Node.js) must receive stdin reliably. + * - In some Bun environments, `Bun.spawn` does not forward stdin to Node correctly. + * - Capture stdout/stderr via temp files to avoid pipe incompatibilities. */ export async function execShellWithStdin( command: string, @@ -38,77 +91,60 @@ export async function execShellWithStdin( readonly stderr: string; readonly exitCode: number; }> { - const bunSpawn = getBunSpawn(); - if (bunSpawn) { - const encoder = new TextEncoder(); - // Use platform-appropriate shell - const isWindows = process.platform === 'win32'; - const shellCmd = isWindows ? ['cmd.exe', '/c', command] : ['sh', '-c', command]; - - const proc = bunSpawn({ - cmd: shellCmd, - cwd: options.cwd, - stdin: encoder.encode(stdinPayload), - stdout: 'pipe', - stderr: 'pipe', - }); + const { mkdir, readFile, rm, writeFile } = await import('node:fs/promises'); + const { tmpdir } = await import('node:os'); + const path = await import('node:path'); + const { randomUUID } = await import('node:crypto'); - const timeout = options.timeoutMs - ? setTimeout(() => { - proc.kill(); - }, options.timeoutMs) - : undefined; + const dir = path.join(tmpdir(), `agentv-exec-${randomUUID()}`); + await mkdir(dir, { recursive: true }); - try { - const stdout = await new Response(proc.stdout).text(); - const stderr = await new Response(proc.stderr).text(); - const exitCode = await proc.exited; - return { stdout, stderr, exitCode }; - } finally { - if (timeout !== undefined) { - clearTimeout(timeout); - } - } - } + const stdinPath = path.join(dir, 'stdin.txt'); + const stdoutPath = path.join(dir, 'stdout.txt'); + const stderrPath = path.join(dir, 'stderr.txt'); + + await writeFile(stdinPath, stdinPayload, 'utf8'); + + const wrappedCommand = + process.platform === 'win32' + ? `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}` + : `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}`; const { spawn } = await import('node:child_process'); - return await new Promise((resolve, reject) => { - const child = spawn(command, { - shell: true, - cwd: options.cwd, - stdio: ['pipe', 'pipe', 'pipe'], - }); + try { + const exitCode = await new Promise((resolve, reject) => { + const child = spawn(wrappedCommand, { + shell: true, + cwd: options.cwd, + stdio: ['ignore', 'ignore', 'ignore'], + }); - let stdout = ''; - let stderr = ''; + const timeout = options.timeoutMs + ? setTimeout(() => { + child.kill(); + reject(new Error(`Process timed out after ${options.timeoutMs}ms`)); + }, options.timeoutMs) + : undefined; - const timeout = options.timeoutMs - ? setTimeout(() => { - child.kill(); - reject(new Error(`Process timed out after ${options.timeoutMs}ms`)); - }, options.timeoutMs) - : undefined; + child.on('error', (error) => { + if (timeout !== undefined) { + clearTimeout(timeout); + } + reject(error); + }); - child.stdout?.on('data', (data) => { - stdout += data.toString(); - }); - child.stderr?.on('data', (data) => { - stderr += data.toString(); - }); - child.on('error', (error) => { - if (timeout !== undefined) { - clearTimeout(timeout); - } - reject(error); - }); - child.on('exit', (code) => { - if (timeout !== undefined) { - clearTimeout(timeout); - } - resolve({ stdout, stderr, exitCode: code ?? 0 }); + child.on('exit', (code) => { + if (timeout !== undefined) { + clearTimeout(timeout); + } + resolve(code ?? 0); + }); }); - child.stdin?.write(stdinPayload); - child.stdin?.end(); - }); + const stdout = (await readFile(stdoutPath, 'utf8')).replace(/\r\n/g, '\n'); + const stderr = (await readFile(stderrPath, 'utf8')).replace(/\r\n/g, '\n'); + return { stdout, stderr, exitCode }; + } finally { + await rm(dir, { recursive: true, force: true }); + } } diff --git a/packages/core/test/evaluation/evaluators.test.ts b/packages/core/test/evaluation/evaluators.test.ts index a9e7f2cf..9ab25d8a 100644 --- a/packages/core/test/evaluation/evaluators.test.ts +++ b/packages/core/test/evaluation/evaluators.test.ts @@ -2,7 +2,14 @@ import { describe, expect, it } from 'bun:test'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { CodeEvaluator, LlmJudgeEvaluator } from '../../src/evaluation/evaluators.js'; +import { + CodeEvaluator, + CostEvaluator, + FieldAccuracyEvaluator, + LatencyEvaluator, + LlmJudgeEvaluator, + TokenUsageEvaluator, +} from '../../src/evaluation/evaluators.js'; import type { ResolvedTarget } from '../../src/evaluation/providers/targets.js'; import type { Provider, @@ -444,7 +451,7 @@ describe('CodeEvaluator', () => { // Use external script file for cross-platform compatibility const __dirname = dirname(fileURLToPath(import.meta.url)); - const script = `node ${join(__dirname, '../fixtures/test-judge.cjs')}`; + const script = ['node', join(__dirname, '../fixtures/test-judge.cjs')]; const evaluator = new CodeEvaluator({ script }); @@ -464,4 +471,670 @@ describe('CodeEvaluator', () => { expect(result.hits).toContain('candidate_answer present'); expect(result.hits).toContain('candidate_answer parses'); }); + + it('surfaces stderr and exit code on failure', async () => { + const judgeProvider = new StubProvider(textResponse('{}')); + + const __dirname = dirname(fileURLToPath(import.meta.url)); + const script = ['node', join(__dirname, '../fixtures/test-judge-error.cjs')]; + + const evaluator = new CodeEvaluator({ script }); + + const result = await evaluator.evaluate({ + evalCase: baseTestCase, + candidate: 'Candidate answer', + target: baseTarget, + provider: judgeProvider, + attempt: 0, + promptInputs: { question: '', guidelines: '' }, + now: new Date(), + }); + + expect(result.verdict).toBe('fail'); + expect(result.misses[0]).toContain('exited with code'); + expect(result.misses[0]).toContain('test-error'); + }); +}); + +describe('FieldAccuracyEvaluator', () => { + const baseTestCaseWithExpected: EvalCase = { + ...baseTestCase, + expected_messages: [ + { + role: 'assistant', + content: { + invoice_number: 'INV-001', + amount: 1500, + date: '15-JAN-2025', + vendor: { name: 'Acme Shipping', address: '123 Main St' }, + }, + }, + ], + }; + + const judgeProvider = new StubProvider(textResponse('{}')); + + it('evaluates exact match fields correctly', () => { + const evaluator = new FieldAccuracyEvaluator({ + config: { + name: 'test', + type: 'field_accuracy', + fields: [ + { path: 'invoice_number', match: 'exact', required: true, weight: 1.0 }, + { path: 'amount', match: 'exact', required: true, weight: 1.0 }, + ], + }, + }); + + const result = evaluator.evaluate({ + evalCase: baseTestCaseWithExpected, + candidate: JSON.stringify({ invoice_number: 'INV-001', amount: 1500 }), + target: baseTarget, + provider: judgeProvider, + attempt: 0, + promptInputs: { question: '', guidelines: '' }, + now: new Date(), + }); + + expect(result.score).toBe(1.0); + expect(result.verdict).toBe('pass'); + expect(result.hits).toHaveLength(2); + expect(result.misses).toHaveLength(0); + }); + + it('handles missing required fields', () => { + const evaluator = new FieldAccuracyEvaluator({ + config: { + name: 'test', + type: 'field_accuracy', + fields: [ + { path: 'invoice_number', match: 'exact', required: true, weight: 1.0 }, + { path: 'amount', match: 'exact', required: true, weight: 1.0 }, + ], + }, + }); + + const result = evaluator.evaluate({ + evalCase: baseTestCaseWithExpected, + candidate: JSON.stringify({ invoice_number: 'INV-001' }), // Missing amount + target: baseTarget, + provider: judgeProvider, + attempt: 0, + promptInputs: { question: '', guidelines: '' }, + now: new Date(), + }); + + expect(result.score).toBe(0.5); + expect(result.verdict).toBe('fail'); + expect(result.hits).toHaveLength(1); + expect(result.misses).toHaveLength(1); + expect(result.misses[0]).toContain('amount'); + expect(result.misses[0]).toContain('required'); + }); + + it('applies numeric tolerance matching', () => { + const evaluator = new FieldAccuracyEvaluator({ + config: { + name: 'test', + type: 'field_accuracy', + fields: [ + { + path: 'amount', + match: 'numeric_tolerance', + tolerance: 1.0, + relative: false, + required: true, + weight: 1.0, + }, + ], + }, + }); + + // 1500.5 vs 1500 - within tolerance of 1.0 + const result = evaluator.evaluate({ + evalCase: baseTestCaseWithExpected, + candidate: JSON.stringify({ amount: 1500.5 }), + target: baseTarget, + provider: judgeProvider, + attempt: 0, + promptInputs: { question: '', guidelines: '' }, + now: new Date(), + }); + + expect(result.score).toBe(1.0); + expect(result.verdict).toBe('pass'); + }); + + it('fails numeric tolerance when outside range', () => { + const evaluator = new FieldAccuracyEvaluator({ + config: { + name: 'test', + type: 'field_accuracy', + fields: [ + { + path: 'amount', + match: 'numeric_tolerance', + tolerance: 1.0, + relative: false, + required: true, + weight: 1.0, + }, + ], + }, + }); + + // 1502 vs 1500 - outside tolerance of 1.0 + const result = evaluator.evaluate({ + evalCase: baseTestCaseWithExpected, + candidate: JSON.stringify({ amount: 1502 }), + target: baseTarget, + provider: judgeProvider, + attempt: 0, + promptInputs: { question: '', guidelines: '' }, + now: new Date(), + }); + + expect(result.score).toBe(0); + expect(result.verdict).toBe('fail'); + expect(result.misses[0]).toContain('outside tolerance'); + }); + + it('applies date matching with format normalization', () => { + const evaluator = new FieldAccuracyEvaluator({ + config: { + name: 'test', + type: 'field_accuracy', + fields: [ + { + path: 'date', + match: 'date', + formats: ['DD-MMM-YYYY', 'YYYY-MM-DD'], + required: true, + weight: 1.0, + }, + ], + }, + }); + + // "2025-01-15" vs "15-JAN-2025" - same date, different formats + const result = evaluator.evaluate({ + evalCase: baseTestCaseWithExpected, + candidate: JSON.stringify({ date: '2025-01-15' }), + target: baseTarget, + provider: judgeProvider, + attempt: 0, + promptInputs: { question: '', guidelines: '' }, + now: new Date(), + }); + + expect(result.score).toBe(1.0); + expect(result.verdict).toBe('pass'); + }); + + it('respects weighted averaging', () => { + const evaluator = new FieldAccuracyEvaluator({ + config: { + name: 'test', + type: 'field_accuracy', + fields: [ + { path: 'invoice_number', match: 'exact', required: true, weight: 2.0 }, // 2x weight + { path: 'amount', match: 'exact', required: true, weight: 1.0 }, + ], + aggregation: 'weighted_average', + }, + }); + + // Correct invoice_number (weight 2), wrong amount (weight 1) + const result = evaluator.evaluate({ + evalCase: baseTestCaseWithExpected, + candidate: JSON.stringify({ invoice_number: 'INV-001', amount: 9999 }), + target: baseTarget, + provider: judgeProvider, + attempt: 0, + promptInputs: { question: '', guidelines: '' }, + now: new Date(), + }); + + // Score should be (1.0 * 2.0 + 0.0 * 1.0) / (2.0 + 1.0) = 2/3 ≈ 0.667 + expect(result.score).toBeCloseTo(0.667, 2); + expect(result.verdict).toBe('borderline'); + }); + + it('supports all_or_nothing aggregation', () => { + const evaluator = new FieldAccuracyEvaluator({ + config: { + name: 'test', + type: 'field_accuracy', + fields: [ + { path: 'invoice_number', match: 'exact', required: true, weight: 1.0 }, + { path: 'amount', match: 'exact', required: true, weight: 1.0 }, + ], + aggregation: 'all_or_nothing', + }, + }); + + // Correct invoice_number, wrong amount - should fail completely + const result = evaluator.evaluate({ + evalCase: baseTestCaseWithExpected, + candidate: JSON.stringify({ invoice_number: 'INV-001', amount: 9999 }), + target: baseTarget, + provider: judgeProvider, + attempt: 0, + promptInputs: { question: '', guidelines: '' }, + now: new Date(), + }); + + expect(result.score).toBe(0); + expect(result.verdict).toBe('fail'); + }); + + it('handles nested field paths', () => { + const evaluator = new FieldAccuracyEvaluator({ + config: { + name: 'test', + type: 'field_accuracy', + fields: [ + { path: 'vendor.name', match: 'exact', required: true, weight: 1.0 }, + { path: 'vendor.address', match: 'exact', required: true, weight: 1.0 }, + ], + }, + }); + + const result = evaluator.evaluate({ + evalCase: baseTestCaseWithExpected, + candidate: JSON.stringify({ vendor: { name: 'Acme Shipping', address: '123 Main St' } }), + target: baseTarget, + provider: judgeProvider, + attempt: 0, + promptInputs: { question: '', guidelines: '' }, + now: new Date(), + }); + + expect(result.score).toBe(1.0); + expect(result.verdict).toBe('pass'); + }); + + it('handles array index paths', () => { + const evalCaseWithArray: EvalCase = { + ...baseTestCase, + expected_messages: [ + { + role: 'assistant', + content: { + items: [ + { name: 'Item A', price: 100 }, + { name: 'Item B', price: 200 }, + ], + }, + }, + ], + }; + + const evaluator = new FieldAccuracyEvaluator({ + config: { + name: 'test', + type: 'field_accuracy', + fields: [ + { path: 'items[0].name', match: 'exact', required: true, weight: 1.0 }, + { path: 'items[1].price', match: 'exact', required: true, weight: 1.0 }, + ], + }, + }); + + const result = evaluator.evaluate({ + evalCase: evalCaseWithArray, + candidate: JSON.stringify({ + items: [ + { name: 'Item A', price: 100 }, + { name: 'Item B', price: 200 }, + ], + }), + target: baseTarget, + provider: judgeProvider, + attempt: 0, + promptInputs: { question: '', guidelines: '' }, + now: new Date(), + }); + + expect(result.score).toBe(1.0); + expect(result.verdict).toBe('pass'); + }); + + it('returns failure for invalid JSON candidate', () => { + const evaluator = new FieldAccuracyEvaluator({ + config: { + name: 'test', + type: 'field_accuracy', + fields: [{ path: 'invoice_number', match: 'exact', required: true, weight: 1.0 }], + }, + }); + + const result = evaluator.evaluate({ + evalCase: baseTestCaseWithExpected, + candidate: 'This is not valid JSON', + target: baseTarget, + provider: judgeProvider, + attempt: 0, + promptInputs: { question: '', guidelines: '' }, + now: new Date(), + }); + + expect(result.score).toBe(0); + expect(result.verdict).toBe('fail'); + expect(result.misses[0]).toContain('parse'); + }); +}); + +describe('LatencyEvaluator', () => { + it('passes when duration is under threshold', () => { + const evaluator = new LatencyEvaluator({ + config: { + name: 'latency_check', + type: 'latency', + threshold: 2000, + }, + }); + + const result = evaluator.evaluate({ + evalCase: baseTestCase, + candidate: 'Answer', + target: baseTarget, + provider: new StubProvider(textResponse('ok')), + attempt: 0, + promptInputs: { question: '', guidelines: '' }, + now: new Date(), + traceSummary: { + eventCount: 1, + toolNames: [], + toolCallsByName: {}, + errorCount: 0, + durationMs: 1500, + }, + }); + + expect(result.score).toBe(1); + expect(result.verdict).toBe('pass'); + expect(result.hits[0]).toContain('1500ms'); + }); + + it('fails when duration exceeds threshold', () => { + const evaluator = new LatencyEvaluator({ + config: { + name: 'latency_check', + type: 'latency', + threshold: 1000, + }, + }); + + const result = evaluator.evaluate({ + evalCase: baseTestCase, + candidate: 'Answer', + target: baseTarget, + provider: new StubProvider(textResponse('ok')), + attempt: 0, + promptInputs: { question: '', guidelines: '' }, + now: new Date(), + traceSummary: { + eventCount: 1, + toolNames: [], + toolCallsByName: {}, + errorCount: 0, + durationMs: 2500, + }, + }); + + expect(result.score).toBe(0); + expect(result.verdict).toBe('fail'); + expect(result.misses[0]).toContain('2500ms'); + }); + + it('fails when no duration data available', () => { + const evaluator = new LatencyEvaluator({ + config: { + name: 'latency_check', + type: 'latency', + threshold: 2000, + }, + }); + + const result = evaluator.evaluate({ + evalCase: baseTestCase, + candidate: 'Answer', + target: baseTarget, + provider: new StubProvider(textResponse('ok')), + attempt: 0, + promptInputs: { question: '', guidelines: '' }, + now: new Date(), + // No traceSummary + }); + + expect(result.score).toBe(0); + expect(result.verdict).toBe('fail'); + expect(result.misses[0]).toContain('No duration data'); + }); + + it('passes when duration equals threshold exactly', () => { + const evaluator = new LatencyEvaluator({ + config: { + name: 'latency_check', + type: 'latency', + threshold: 1000, + }, + }); + + const result = evaluator.evaluate({ + evalCase: baseTestCase, + candidate: 'Answer', + target: baseTarget, + provider: new StubProvider(textResponse('ok')), + attempt: 0, + promptInputs: { question: '', guidelines: '' }, + now: new Date(), + traceSummary: { + eventCount: 1, + toolNames: [], + toolCallsByName: {}, + errorCount: 0, + durationMs: 1000, + }, + }); + + expect(result.score).toBe(1); + expect(result.verdict).toBe('pass'); + }); +}); + +describe('CostEvaluator', () => { + it('passes when cost is under budget', () => { + const evaluator = new CostEvaluator({ + config: { + name: 'cost_check', + type: 'cost', + budget: 0.1, + }, + }); + + const result = evaluator.evaluate({ + evalCase: baseTestCase, + candidate: 'Answer', + target: baseTarget, + provider: new StubProvider(textResponse('ok')), + attempt: 0, + promptInputs: { question: '', guidelines: '' }, + now: new Date(), + traceSummary: { + eventCount: 1, + toolNames: [], + toolCallsByName: {}, + errorCount: 0, + costUsd: 0.05, + }, + }); + + expect(result.score).toBe(1); + expect(result.verdict).toBe('pass'); + expect(result.hits[0]).toContain('$0.0500'); + }); + + it('fails when cost exceeds budget', () => { + const evaluator = new CostEvaluator({ + config: { + name: 'cost_check', + type: 'cost', + budget: 0.05, + }, + }); + + const result = evaluator.evaluate({ + evalCase: baseTestCase, + candidate: 'Answer', + target: baseTarget, + provider: new StubProvider(textResponse('ok')), + attempt: 0, + promptInputs: { question: '', guidelines: '' }, + now: new Date(), + traceSummary: { + eventCount: 1, + toolNames: [], + toolCallsByName: {}, + errorCount: 0, + costUsd: 0.15, + }, + }); + + expect(result.score).toBe(0); + expect(result.verdict).toBe('fail'); + expect(result.misses[0]).toContain('$0.1500'); + }); + + it('fails when no cost data available', () => { + const evaluator = new CostEvaluator({ + config: { + name: 'cost_check', + type: 'cost', + budget: 0.1, + }, + }); + + const result = evaluator.evaluate({ + evalCase: baseTestCase, + candidate: 'Answer', + target: baseTarget, + provider: new StubProvider(textResponse('ok')), + attempt: 0, + promptInputs: { question: '', guidelines: '' }, + now: new Date(), + // No traceSummary + }); + + expect(result.score).toBe(0); + expect(result.verdict).toBe('fail'); + expect(result.misses[0]).toContain('No cost data'); + }); + + it('passes when cost equals budget exactly', () => { + const evaluator = new CostEvaluator({ + config: { + name: 'cost_check', + type: 'cost', + budget: 0.1, + }, + }); + + const result = evaluator.evaluate({ + evalCase: baseTestCase, + candidate: 'Answer', + target: baseTarget, + provider: new StubProvider(textResponse('ok')), + attempt: 0, + promptInputs: { question: '', guidelines: '' }, + now: new Date(), + traceSummary: { + eventCount: 1, + toolNames: [], + toolCallsByName: {}, + errorCount: 0, + costUsd: 0.1, + }, + }); + + expect(result.score).toBe(1); + expect(result.verdict).toBe('pass'); + }); +}); + +describe('TokenUsageEvaluator', () => { + it('passes when total tokens are under max_total', () => { + const evaluator = new TokenUsageEvaluator({ + config: { name: 'token_budget', type: 'token_usage', max_total: 1000 }, + }); + + const result = evaluator.evaluate({ + evalCase: baseTestCase, + candidate: 'Answer', + target: baseTarget, + provider: new StubProvider(textResponse('ok')), + attempt: 0, + promptInputs: { question: '', guidelines: '' }, + now: new Date(), + traceSummary: { + eventCount: 0, + toolNames: [], + toolCallsByName: {}, + errorCount: 0, + tokenUsage: { input: 400, output: 500, cached: 0 }, + }, + }); + + expect(result.score).toBe(1); + expect(result.verdict).toBe('pass'); + expect(result.hits.join(' ')).toContain('Total tokens'); + }); + + it('fails when output tokens exceed max_output', () => { + const evaluator = new TokenUsageEvaluator({ + config: { name: 'token_budget', type: 'token_usage', max_output: 100 }, + }); + + const result = evaluator.evaluate({ + evalCase: baseTestCase, + candidate: 'Answer', + target: baseTarget, + provider: new StubProvider(textResponse('ok')), + attempt: 0, + promptInputs: { question: '', guidelines: '' }, + now: new Date(), + traceSummary: { + eventCount: 0, + toolNames: [], + toolCallsByName: {}, + errorCount: 0, + tokenUsage: { input: 10, output: 150 }, + }, + }); + + expect(result.score).toBe(0); + expect(result.verdict).toBe('fail'); + expect(result.misses.join(' ')).toContain('Output tokens'); + }); + + it('fails when no token usage data available', () => { + const evaluator = new TokenUsageEvaluator({ + config: { name: 'token_budget', type: 'token_usage', max_total: 1000 }, + }); + + const result = evaluator.evaluate({ + evalCase: baseTestCase, + candidate: 'Answer', + target: baseTarget, + provider: new StubProvider(textResponse('ok')), + attempt: 0, + promptInputs: { question: '', guidelines: '' }, + now: new Date(), + }); + + expect(result.score).toBe(0); + expect(result.verdict).toBe('fail'); + expect(result.misses[0]).toContain('token usage'); + }); }); diff --git a/packages/core/test/evaluation/execution-metrics.test.ts b/packages/core/test/evaluation/execution-metrics.test.ts index eb942135..8fcc4a1b 100644 --- a/packages/core/test/evaluation/execution-metrics.test.ts +++ b/packages/core/test/evaluation/execution-metrics.test.ts @@ -277,7 +277,7 @@ describe('Code Judge Metrics Integration', () => { it('passes traceSummary to code_judge scripts', async () => { // Use external script file for cross-platform compatibility const __dirname = dirname(fileURLToPath(import.meta.url)); - const script = `node ${join(__dirname, '../fixtures/test-trace-summary.cjs')}`; + const script = ['node', join(__dirname, '../fixtures/test-trace-summary.cjs')]; const evaluator = new CodeEvaluator({ script }); @@ -311,7 +311,7 @@ describe('Code Judge Metrics Integration', () => { it('handles missing traceSummary gracefully', async () => { // Use external script file for cross-platform compatibility const __dirname = dirname(fileURLToPath(import.meta.url)); - const script = `node ${join(__dirname, '../fixtures/test-no-trace-summary.cjs')}`; + const script = ['node', join(__dirname, '../fixtures/test-no-trace-summary.cjs')]; const evaluator = new CodeEvaluator({ script }); diff --git a/packages/core/test/evaluation/loaders/evaluator-parser.test.ts b/packages/core/test/evaluation/loaders/evaluator-parser.test.ts index 6c07b0a8..fa85e4d7 100644 --- a/packages/core/test/evaluation/loaders/evaluator-parser.test.ts +++ b/packages/core/test/evaluation/loaders/evaluator-parser.test.ts @@ -1,10 +1,11 @@ import { afterAll, beforeAll, describe, expect, it } from 'bun:test'; -import { mkdir, rm } from 'node:fs/promises'; +import { mkdir, rm, writeFile } from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; import { parseEvaluators } from '../../../src/evaluation/loaders/evaluator-parser.js'; import type { ToolTrajectoryEvaluatorConfig } from '../../../src/evaluation/trace.js'; +import type { CodeEvaluatorConfig } from '../../../src/evaluation/types.js'; describe('parseEvaluators - tool_trajectory', () => { let tempDir: string; @@ -199,3 +200,139 @@ describe('parseEvaluators - tool_trajectory', () => { expect(config.expected).toEqual([{ tool: 'validTool' }, { tool: 'anotherValid' }]); }); }); + +describe('parseEvaluators - code_judge config pass-through', () => { + let tempDir: string; + + beforeAll(async () => { + tempDir = path.join(os.tmpdir(), `agentv-test-code-judge-${Date.now()}`); + await mkdir(tempDir, { recursive: true }); + // Create a dummy script file + await writeFile(path.join(tempDir, 'test_script.ts'), '// dummy script'); + }); + + afterAll(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + + it('passes unrecognized properties as config', async () => { + const rawEvalCase = { + evaluators: [ + { + name: 'fuzzy-matcher', + type: 'code_judge', + script: ['bun', 'run', './test_script.ts'], + fields: [ + { path: 'supplier.name', threshold: 0.85 }, + { path: 'importer.name', threshold: 0.9 }, + ], + algorithm: 'levenshtein', + customOption: true, + }, + ], + }; + + const evaluators = await parseEvaluators(rawEvalCase, undefined, [tempDir], 'test-case'); + + expect(evaluators).toHaveLength(1); + const config = evaluators?.[0] as CodeEvaluatorConfig; + expect(config.type).toBe('code'); + expect(config.name).toBe('fuzzy-matcher'); + expect(config.config).toEqual({ + fields: [ + { path: 'supplier.name', threshold: 0.85 }, + { path: 'importer.name', threshold: 0.9 }, + ], + algorithm: 'levenshtein', + customOption: true, + }); + }); + + it('does not include config when no extra properties', async () => { + const rawEvalCase = { + evaluators: [ + { + name: 'simple-judge', + type: 'code_judge', + script: ['bun', 'run', './test_script.ts'], + }, + ], + }; + + const evaluators = await parseEvaluators(rawEvalCase, undefined, [tempDir], 'test-case'); + + expect(evaluators).toHaveLength(1); + const config = evaluators?.[0] as CodeEvaluatorConfig; + expect(config.type).toBe('code'); + expect(config.config).toBeUndefined(); + }); + + it('excludes known properties from config', async () => { + const rawEvalCase = { + evaluators: [ + { + name: 'with-weight', + type: 'code_judge', + script: ['bun', 'run', './test_script.ts'], + cwd: tempDir, + weight: 2.0, + threshold: 0.85, // This should go to config + }, + ], + }; + + const evaluators = await parseEvaluators(rawEvalCase, undefined, [tempDir], 'test-case'); + + expect(evaluators).toHaveLength(1); + const config = evaluators?.[0] as CodeEvaluatorConfig; + expect(config.weight).toBe(2.0); + expect(config.config).toEqual({ threshold: 0.85 }); + }); + + it('converts string scripts into argv using a shell', async () => { + const rawEvalCase = { + evaluators: [ + { + name: 'legacy-script', + type: 'code_judge', + script: './test_script.ts', + }, + ], + }; + + const evaluators = await parseEvaluators(rawEvalCase, undefined, [tempDir], 'test-case'); + + expect(evaluators).toHaveLength(1); + const config = evaluators?.[0] as CodeEvaluatorConfig; + if (process.platform === 'win32') { + expect(config.script).toEqual(['cmd.exe', '/c', './test_script.ts']); + } else { + expect(config.script).toEqual(['sh', '-lc', './test_script.ts']); + } + }); +}); + +describe('parseEvaluators - token_usage', () => { + it('parses token_usage evaluator with limits', async () => { + const rawEvalCase = { + evaluators: [ + { + name: 'token-budget', + type: 'token_usage', + max_total: 1000, + max_output: 200, + }, + ], + }; + + const evaluators = await parseEvaluators(rawEvalCase, undefined, [process.cwd()], 'test-case'); + + expect(evaluators).toHaveLength(1); + expect(evaluators?.[0]).toEqual({ + name: 'token-budget', + type: 'token_usage', + max_total: 1000, + max_output: 200, + }); + }); +}); diff --git a/packages/core/test/evaluation/orchestrator.test.ts b/packages/core/test/evaluation/orchestrator.test.ts index a8118c6f..9546c6fc 100644 --- a/packages/core/test/evaluation/orchestrator.test.ts +++ b/packages/core/test/evaluation/orchestrator.test.ts @@ -86,6 +86,26 @@ class CapturingProvider implements Provider { } } +class CapturingCliProvider implements Provider { + readonly id: string; + readonly kind = 'cli' as const; + readonly targetName: string; + lastRequest?: ProviderRequest; + + constructor( + targetName: string, + private readonly response: ProviderResponse, + ) { + this.id = `cli:${targetName}`; + this.targetName = targetName; + } + + async invoke(request: ProviderRequest): Promise { + this.lastRequest = request; + return this.response; + } +} + const baseTestCase: EvalCase = { id: 'case-1', dataset: 'test-dataset', @@ -433,6 +453,40 @@ describe('runTestCase', () => { expect(result.lmProviderRequest).toBeUndefined(); expect(result.agentProviderRequest?.question).toBe('Explain logging improvements'); }); + + it('uses file references (not embedded contents) for cli providers', async () => { + const provider = new CapturingCliProvider('cli', { + outputMessages: [{ role: 'assistant', content: 'ok' }], + }); + + const result = await runEvalCase({ + evalCase: { + ...baseTestCase, + input_messages: [ + { + role: 'user', + content: [ + { type: 'file', value: 'input.json' }, + { type: 'text', value: 'Summarize the file.' }, + ], + }, + ], + input_segments: [ + { type: 'file', path: 'input.json', text: '{"secret":true}' }, + { type: 'text', value: 'Summarize the file.' }, + ], + file_paths: ['/abs/path/input.json'], + }, + provider, + target: baseTarget, + evaluators: evaluatorRegistry, + }); + + expect(result.lmProviderRequest).toBeDefined(); + expect(result.lmProviderRequest?.question).toContain(''); + expect(result.lmProviderRequest?.question).not.toContain(''); + expect(result.lmProviderRequest?.question).not.toContain('{"secret":true}'); + }); }); // Provider that returns outputMessages with tool calls @@ -526,6 +580,32 @@ describe('runEvalCase trace integration', () => { expect(result.traceSummary).toBeUndefined(); }); + it('includes traceSummary when provider reports tokenUsage without outputMessages', async () => { + const provider = new TraceProvider('mock', { + tokenUsage: { input: 10, output: 20, cached: 5 }, + }); + + const result = await runEvalCase({ + evalCase: { + ...traceTestCase, + evaluators: [ + { + name: 'token-budget', + type: 'token_usage', + max_total: 1000, + }, + ], + }, + provider, + target: baseTarget, + evaluators: evaluatorRegistry, + }); + + expect(result.traceSummary).toBeDefined(); + expect(result.traceSummary?.tokenUsage).toEqual({ input: 10, output: 20, cached: 5 }); + expect(result.score).toBe(1); + }); + it('runs tool_trajectory evaluator with outputMessages', async () => { const outputMessages: OutputMessage[] = [ { @@ -630,6 +710,44 @@ describe('runEvalCase trace integration', () => { expect(result.evaluatorResults?.[0]?.misses).toContain('No trace available for evaluation'); }); + it('runs latency/cost evaluators inside composite using traceSummary', async () => { + const outputMessages: OutputMessage[] = [{ role: 'assistant', content: 'Done' }]; + + const provider = new TraceProvider('mock', { costUsd: 0.05, durationMs: 1200 }, outputMessages); + + const result = await runEvalCase({ + evalCase: { + ...traceTestCase, + evaluators: [ + { + name: 'metrics', + type: 'composite', + evaluators: [ + { name: 'latency', type: 'latency', threshold: 1500 }, + { name: 'cost', type: 'cost', budget: 0.1 }, + ], + aggregator: { type: 'weighted_average' }, + }, + ], + }, + provider, + target: baseTarget, + evaluators: evaluatorRegistry, + }); + + expect(result.score).toBe(1); + expect(result.evaluatorResults).toHaveLength(1); + expect(result.evaluatorResults?.[0]?.name).toBe('metrics'); + expect(result.evaluatorResults?.[0]?.verdict).toBe('pass'); + expect(result.evaluatorResults?.[0]?.evaluatorResults).toHaveLength(2); + const childNames = result.evaluatorResults?.[0]?.evaluatorResults?.map((child) => child.name); + expect(childNames).toEqual(['latency', 'cost']); + const childVerdicts = result.evaluatorResults?.[0]?.evaluatorResults?.map( + (child) => child.verdict, + ); + expect(childVerdicts).toEqual(['pass', 'pass']); + }); + it('computes correct trace summary with multiple tool calls', async () => { const outputMessages: OutputMessage[] = [ { diff --git a/packages/core/test/evaluation/providers/targets-cwd-fallback.test.ts b/packages/core/test/evaluation/providers/targets-cwd-fallback.test.ts index 6212a461..08d3d25d 100644 --- a/packages/core/test/evaluation/providers/targets-cwd-fallback.test.ts +++ b/packages/core/test/evaluation/providers/targets-cwd-fallback.test.ts @@ -99,4 +99,28 @@ describe('CLI cwd fallback to eval directory', () => { expect(resolved.config.cwd).toBe(path.resolve('/path/to/evals/my-test')); } }); + + it('falls back to eval directory for healthcheck cwd when unset', () => { + const definition = { + name: 'test-cli', + provider: 'cli', + command_template: 'echo {PROMPT}', + healthcheck: { + type: 'command', + command_template: 'echo healthy', + }, + }; + + const env = {}; + const evalFilePath = '/path/to/evals/my-test/test.yaml'; + const resolved = resolveTargetDefinition(definition, env, evalFilePath); + + expect(resolved.kind).toBe('cli'); + if (resolved.kind === 'cli') { + expect(resolved.config.healthcheck?.type).toBe('command'); + if (resolved.config.healthcheck?.type === 'command') { + expect(resolved.config.healthcheck.cwd).toBe(path.resolve('/path/to/evals/my-test')); + } + } + }); }); diff --git a/packages/core/test/fixtures/test-judge-error.cjs b/packages/core/test/fixtures/test-judge-error.cjs new file mode 100644 index 00000000..d156a3f6 --- /dev/null +++ b/packages/core/test/fixtures/test-judge-error.cjs @@ -0,0 +1,2 @@ +process.stderr.write('test-error\n'); +process.exit(2); diff --git a/packages/core/test/runtime/exec.test.ts b/packages/core/test/runtime/exec.test.ts index 3803e76b..a9164203 100644 --- a/packages/core/test/runtime/exec.test.ts +++ b/packages/core/test/runtime/exec.test.ts @@ -3,9 +3,9 @@ import { unlinkSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { execShellWithStdin } from '../../src/runtime/exec.js'; +import { execFileWithStdin } from '../../src/runtime/exec.js'; -describe('execShellWithStdin', () => { +describe('execFileWithStdin', () => { it('passes stdin payload to the child process', async () => { const payload = 'hello-world'; // Create a temporary script file to avoid quote escaping issues @@ -16,7 +16,7 @@ describe('execShellWithStdin', () => { ); try { - const result = await execShellWithStdin(`node ${scriptPath}`, payload); + const result = await execFileWithStdin(['node', scriptPath], payload); expect(result.exitCode).toBe(0); expect(result.stdout).toBe(payload); @@ -32,7 +32,7 @@ describe('execShellWithStdin', () => { writeFileSync(scriptPath, "process.stderr.write('test-error\\n'); process.exit(2);"); try { - const result = await execShellWithStdin(`node ${scriptPath}`, ''); + const result = await execFileWithStdin(['node', scriptPath], ''); expect(result.exitCode).toBe(2); expect(result.stdout.trim()).toBe(''); @@ -41,4 +41,34 @@ describe('execShellWithStdin', () => { unlinkSync(scriptPath); } }); + + it('handles large stdin payloads', async () => { + const payload = 'x'.repeat(1024 * 1024 + 16); + const scriptPath = join(tmpdir(), `test-large-${Date.now()}.cjs`); + writeFileSync( + scriptPath, + "let data=''; process.stdin.on('data', d => data += d); process.stdin.on('end', () => process.stdout.write(String(Buffer.byteLength(data))));", + ); + + try { + const result = await execFileWithStdin(['node', scriptPath], payload); + expect(result.exitCode).toBe(0); + expect(result.stdout.trim()).toBe(String(Buffer.byteLength(payload))); + } finally { + unlinkSync(scriptPath); + } + }); + + it('times out long-running processes', async () => { + const scriptPath = join(tmpdir(), `test-timeout-${Date.now()}.cjs`); + writeFileSync(scriptPath, 'setInterval(() => {}, 1000);'); + + try { + await expect(execFileWithStdin(['node', scriptPath], '', { timeoutMs: 50 })).rejects.toThrow( + 'Process timed out', + ); + } finally { + unlinkSync(scriptPath); + } + }); }); diff --git a/scripts/check-eval-baselines.ts b/scripts/check-eval-baselines.ts new file mode 100644 index 00000000..696f92e8 --- /dev/null +++ b/scripts/check-eval-baselines.ts @@ -0,0 +1,161 @@ +#!/usr/bin/env bun +import { existsSync } from 'node:fs'; +import { mkdir, readdir } from 'node:fs/promises'; +import path from 'node:path'; + +type CliOptions = { + candidateRoot: string; + threshold?: string; +}; + +const repoRoot = path.resolve(__dirname, '..'); +const examplesRoot = path.join(repoRoot, 'examples'); + +function parseArgs(argv: string[]): CliOptions { + const options: CliOptions = { + candidateRoot: path.join(repoRoot, '.agentv', 'candidate-results'), + }; + + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + if (arg === '--candidate-root') { + options.candidateRoot = argv[i + 1] ?? options.candidateRoot; + i += 1; + continue; + } + if (arg === '--threshold') { + options.threshold = argv[i + 1]; + i += 1; + } + } + + return options; +} + +async function findBaselineFiles(dir: string, results: string[] = []): Promise { + const entries = await readdir(dir, { withFileTypes: true }); + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + await findBaselineFiles(fullPath, results); + continue; + } + if (entry.isFile() && entry.name.endsWith('.baseline.jsonl')) { + results.push(fullPath); + } + } + return results; +} + +function resolveEvalFile(baselinePath: string): string { + const yamlPath = baselinePath.replace(/\.baseline\.jsonl$/, '.yaml'); + if (existsSync(yamlPath)) { + return yamlPath; + } + const ymlPath = baselinePath.replace(/\.baseline\.jsonl$/, '.yml'); + if (existsSync(ymlPath)) { + return ymlPath; + } + throw new Error(`Eval file not found for baseline: ${baselinePath}`); +} + +function candidatePathFor( + baselinePath: string, + candidateRoot: string, +): { candidatePath: string; relativePath: string } { + const relativePath = path.relative(repoRoot, baselinePath); + const candidatePath = path + .join(candidateRoot, relativePath) + .replace(/\.baseline\.jsonl$/, '.candidate.jsonl'); + return { candidatePath, relativePath }; +} + +async function ensureParentDir(filePath: string): Promise { + const dir = path.dirname(filePath); + if (existsSync(dir)) { + return; + } + await mkdir(dir, { recursive: true }); +} + +async function runEval(evalFile: string, candidatePath: string): Promise { + await ensureParentDir(candidatePath); + + const env = { ...process.env }; + if (!env.TOOL_EVAL_PLUGINS_DIR) { + env.TOOL_EVAL_PLUGINS_DIR = path.join( + repoRoot, + 'examples', + 'showcase', + 'tool-evaluation-plugins', + ); + } + + const args = ['bun', 'agentv', 'eval', evalFile, '--out', candidatePath]; + const proc = Bun.spawn(args, { + cwd: repoRoot, + stdout: 'inherit', + stderr: 'inherit', + env, + }); + return await proc.exited; +} + +async function main(): Promise { + const options = parseArgs(process.argv.slice(2)); + const baselineFiles = await findBaselineFiles(examplesRoot); + + if (baselineFiles.length === 0) { + console.error('No baseline files found under examples/.'); + process.exit(1); + } + + let failures = 0; + + for (const baselinePath of baselineFiles.sort()) { + const { candidatePath, relativePath } = candidatePathFor(baselinePath, options.candidateRoot); + const evalFile = resolveEvalFile(baselinePath); + + console.log(`Running eval for ${relativePath}`); + const evalExitCode = await runEval(evalFile, candidatePath); + if (evalExitCode !== 0) { + failures += 1; + continue; + } + + if (!existsSync(candidatePath)) { + console.error(`Missing candidate results for ${relativePath}`); + failures += 1; + continue; + } + + const args = ['bun', 'agentv', 'compare', baselinePath, candidatePath]; + if (options.threshold) { + args.push('--threshold', options.threshold); + } + + console.log(`Comparing ${relativePath}`); + const proc = Bun.spawn(args, { + cwd: repoRoot, + stdout: 'inherit', + stderr: 'inherit', + }); + const exitCode = await proc.exited; + if (exitCode !== 0) { + failures += 1; + } + } + + if (failures > 0) { + console.error(`Baseline comparison failed for ${failures} file(s).`); + process.exit(1); + } + + console.log('Baseline comparison passed for all files.'); +} + +main().catch((error) => { + const message = error instanceof Error ? error.message : String(error); + console.error(`Baseline comparison failed: ${message}`); + process.exit(1); +});