feat(evals): serialize values accessed via nested selectors (#10317)

* feat(evals): serialize values accessed via nested selectors

* add proxy

Author: mikeldking

js/packages/phoenix-evals/src/llm/ClassificationEvaluator.ts

@@ -3,6 +3,7 @@ import {
   CreateClassificationEvaluatorArgs,
   EvaluatorFn,
   Template,
+  WithPromptTemplate,
 } from "../types";
 
 import { createClassifierFn } from "./createClassifierFn";
@@ -11,9 +12,10 @@ import { LLMEvaluator } from "./LLMEvaluator";
 /**
  * An LLM evaluator that performs evaluation via classification
  */
-export class ClassificationEvaluator<
-  RecordType extends Record<string, unknown>,
-> extends LLMEvaluator<RecordType> {
+export class ClassificationEvaluator<RecordType extends Record<string, unknown>>
+  extends LLMEvaluator<RecordType>
+  implements WithPromptTemplate
+{
   readonly evaluatorFn: EvaluatorFn<RecordType>;
   readonly promptTemplate: Template;
   private _promptTemplateVariables: string[] | undefined;

js/packages/phoenix-evals/src/template/applyTemplate.ts

@@ -1,5 +1,7 @@
 import { Template } from "../types/templating";
 
+import { createTemplateVariablesProxy } from "./createTemplateVariablesProxy";
+
 import Mustache from "mustache";
 
 /**
@@ -11,6 +13,12 @@ export function formatTemplate(args: {
   variables: Record<string, unknown>;
 }) {
   const { template, variables } = args;
+  const variablesProxy = createTemplateVariablesProxy(variables);
   // Disable HTML escaping by providing a custom escape function that returns text as-is
-  return Mustache.render(template, variables, {}, { escape: (text) => text });
+  return Mustache.render(
+    template,
+    variablesProxy,
+    {},
+    { escape: (text) => text }
+  );
 }

js/packages/phoenix-evals/src/template/createTemplateVariablesProxy.ts

@@ -0,0 +1,69 @@
+/**
+ * Creates a Proxy that wraps an object to stringify nested object values when accessed directly.
+ * This allows Mustache to access properties of objects (e.g., {{user.name}}) while
+ * stringifying objects that are accessed as leaf values (e.g., {{user.profile}}).
+ */
+export function createTemplateVariablesProxy(obj: unknown): unknown {
+  if (obj === null || obj === undefined) {
+    return obj;
+  }
+
+  if (Array.isArray(obj)) {
+    return obj.map(createTemplateVariablesProxy);
+  }
+
+  if (typeof obj === "object") {
+    return new Proxy(obj as Record<string, unknown>, {
+      get(target, prop: string | symbol) {
+        // Handle toString and valueOf to stringify the object when accessed directly
+        if (prop === "toString") {
+          return () => JSON.stringify(target);
+        }
+        if (prop === "valueOf") {
+          return () => JSON.stringify(target);
+        }
+
+        if (typeof prop !== "string") {
+          return Reflect.get(target, prop);
+        }
+
+        const value = Reflect.get(target, prop);
+
+        // If the value is an object (not array, not null), wrap it in a proxy
+        // so it can be stringified if accessed directly, or have its properties accessed
+        if (
+          value !== null &&
+          typeof value === "object" &&
+          !Array.isArray(value)
+        ) {
+          return createTemplateVariablesProxy(value);
+        }
+
+        return value;
+      },
+      // Override valueOf and toString to stringify the object when Mustache tries to render it directly
+      // Mustache will call toString() when it needs to render an object as a string
+      has(target, prop) {
+        if (prop === "toString" || prop === "valueOf") {
+          return true;
+        }
+        return Reflect.has(target, prop);
+      },
+      ownKeys(target) {
+        return Reflect.ownKeys(target);
+      },
+      getOwnPropertyDescriptor(target, prop) {
+        if (prop === "toString") {
+          return {
+            enumerable: false,
+            configurable: true,
+            value: () => JSON.stringify(target),
+          };
+        }
+        return Reflect.getOwnPropertyDescriptor(target, prop);
+      },
+    });
+  }
+
+  return obj;
+}

js/packages/phoenix-evals/src/types/evals.ts

@@ -1,4 +1,5 @@
 import { WithTelemetry } from "./otel";
+import { Template } from "./templating";
 
 import { LanguageModel } from "ai";
 
@@ -77,7 +78,7 @@ export interface CreateClassifierArgs extends WithTelemetry {
   /**
    * The prompt template to use for classification
    */
-  promptTemplate: string;
+  promptTemplate: Template;
 }
 
 export interface CreateEvaluatorArgs {

js/packages/phoenix-evals/src/types/templating.ts

@@ -1,2 +1,10 @@
 export type Template = string;
 export type TemplateVariables = Record<string, unknown>;
+
+/**
+ * A class or object that has a prompt template
+ */
+export interface WithPromptTemplate {
+  readonly promptTemplate: Template;
+  get promptTemplateVariables(): string[];
+}

js/packages/phoenix-evals/test/template/applyTemplate.test.ts

@@ -131,4 +131,24 @@ describe("formatTemplate", () => {
       "[Query]: How do I check if x > 5?\n[Answer]: Use: if (x > 5) { }"
     );
   });
+  it("should handle template with nested object variables", () => {
+    const result = formatTemplate({
+      template: "Hello {{user.name}}, your email is {{user.email}}.",
+      variables: {
+        user: { name: "Bob", email: "[email protected]" },
+      },
+    });
+
+    expect(result).toBe("Hello Bob, your email is [email protected].");
+  });
+  it("should stringify nested object variables when accessed directly", () => {
+    const result = formatTemplate({
+      template: "Hello {{user}}",
+      variables: {
+        user: { name: "Bob", email: "[email protected]" },
+      },
+    });
+
+    expect(result).toBe('Hello {"name":"Bob","email":"[email protected]"}');
+  });
 });