diff --git a/.gitignore b/.gitignore index ece0ddaa..ec966b94 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,9 @@ # macOS .DS_Store +# LiveShare cache files +*.lscache + # User-specific files *.rsuser *.suo diff --git a/samples/python/external-agents/observability/.env.example b/samples/python/external-agents/observability/.env.example new file mode 100644 index 00000000..c76ee217 --- /dev/null +++ b/samples/python/external-agents/observability/.env.example @@ -0,0 +1,11 @@ +AZURE_SUBSCRIPTION_ID= +FOUNDRY_PROJECT_ENDPOINT=https://.services.ai.azure.com/api/projects/ +RESOURCE_GROUP= +LOCATION=eastus2 +ACA_ENV= +ACR_NAME= +APPLICATIONINSIGHTS_CONNECTION_STRING=InstrumentationKey=;IngestionEndpoint= +AZURE_OPENAI_ENDPOINT=https://.openai.azure.com +AZURE_OPENAI_DEPLOYMENT=gpt-4o-mini +AZURE_OPENAI_API_VERSION=2024-10-21 +AZURE_OPENAI_API_KEY= diff --git a/samples/python/external-agents/observability/Dockerfile b/samples/python/external-agents/observability/Dockerfile new file mode 100644 index 00000000..c5d3e3c4 --- /dev/null +++ b/samples/python/external-agents/observability/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.11-slim + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PORT=8000 + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY weather_agent.py . + +EXPOSE 8000 + +CMD ["python", "weather_agent.py"] diff --git a/samples/python/external-agents/observability/README.md b/samples/python/external-agents/observability/README.md new file mode 100644 index 00000000..72d53315 --- /dev/null +++ b/samples/python/external-agents/observability/README.md @@ -0,0 +1,169 @@ +# External Agent Observability — Weather Agent on ACA + +This sample shows the **end-to-end story for a Foundry "external" agent**: +a third-party agent runtime that lives **outside** Foundry, registered +into Foundry purely so its OpenTelemetry traces and Foundry-side +evaluations light up in the portal. + +The runtime here is a tiny [LangChain](https://python.langchain.com/) +weather agent (LangGraph ReAct), instrumented with the +[Microsoft OpenTelemetry distro](https://github.com/microsoft/opentelemetry-distro-python) +so its spans flow into the Application Insights connected to your +Foundry project. We deploy it to **Azure Container Apps** to play the +role of "agent hosted somewhere outside Foundry" (it could just as +easily be GCP Cloud Run, AWS, or on-prem). + +> **Preview note.** External agents are gated behind +> `Foundry-Features: ExternalAgents=V1Preview` while in public preview. +> The SDK calls below opt in via `allow_preview=True`. + +> **Distro note.** The Microsoft OTel distro does not yet accept an +> explicit `otel_agent_id` input +> ([microsoft/opentelemetry-distro-python#148](https://github.com/microsoft/opentelemetry-distro-python/issues/148)). +> Until that ships, this sample relies on the default `gen_ai.agent.id` +> emitted by the LangChain instrumentation. The Foundry registration +> uses the matching agent name so the Foundry trace view will resolve +> once the distro fix is available. + +## Microsoft OpenTelemetry distro — references + +To learn more about the distro or to find samples in another language, +start here: + +- **Docs:** [Microsoft OpenTelemetry overview](https://learn.microsoft.com/en-us/azure/microsoft-opentelemetry/overview) +- **Samples by language:** + - .NET — [microsoft/opentelemetry-distro-dotnet](https://github.com/microsoft/opentelemetry-distro-dotnet) + - Python — [microsoft/opentelemetry-distro-python](https://github.com/microsoft/opentelemetry-distro-python) + - JavaScript — [microsoft/opentelemetry-distro-javascript](https://github.com/microsoft/opentelemetry-distro-javascript) + +## What's in this folder + +| File | Purpose | +| --- | --- | +| [weather_agent.py](weather_agent.py) | LangChain weather agent + Microsoft OTel distro, exposed as a FastAPI HTTP service. This is the "external runtime". | +| [.env.example](.env.example) | Placeholder environment template for local configuration. | +| [Dockerfile](Dockerfile) | Container image for the weather agent. | +| [deploy.sh](deploy.sh) / [deploy.ps1](deploy.ps1) | Build, push to ACR, deploy to ACA, and generate traffic. | +| [generate_traffic.py](generate_traffic.py) | Hits the deployed agent with a handful of weather questions. | +| [register_external_agent.py](register_external_agent.py) | Registers the runtime in Foundry as `kind=external` via the `azure-ai-projects` SDK. | +| [run_trace_eval.py](run_trace_eval.py) | Runs a one-off trace-based eval over the registered agent and prints scores. | +| [requirements.txt](requirements.txt) | Python deps for both the runtime and the helper scripts. | + +## Architecture + +``` + ┌────────────────────────┐ OTel spans ┌─────────────────────┐ + │ Weather agent (ACA) │ ─────────────────────────▶ │ Application Insights │ + │ LangChain + MS distro │ gen_ai.agent.id = │ (linked to project) │ + └────────────────────────┘ "weather-agent" └─────────┬───────────┘ + │ + register_external_agent.py │ trace view + │ ▼ + ▼ ┌─────────────────────┐ + ┌─────────────────────┐ │ Foundry Portal │ + │ Foundry Project │ ◀──── │ Agents → traces │ + │ agent kind=external│ │ Evaluations │ + └─────────────────────┘ └─────────────────────┘ +``` + +## Prerequisites + +1. **Azure resources** + - A Foundry project with an Application Insights connection. + - An Azure Container Apps environment + ACR in the same subscription. + - An Azure OpenAI deployment (e.g. `gpt-4o-mini`) for both the agent + LLM and the eval judge. (You can split them with the + `EVAL_MODEL_DEPLOYMENT` env var.) +2. **Permissions** — `Contributor` (or equivalent) on the RG, plus + permission to create agents in the Foundry project (e.g. + `Azure AI User`). +3. **Tooling** — Azure CLI, Docker is *not* required (we use + `az acr build`), Python 3.11+. + +## Step 1 — Configure environment + +Start from [.env.example](.env.example), create a local `.env`, and +load those values into your shell before running the scripts. The local +`.env` file is ignored by git. + +```bash +export AZURE_SUBSCRIPTION_ID="" +export FOUNDRY_PROJECT_ENDPOINT="https://.services.ai.azure.com/api/projects/" +export RESOURCE_GROUP="..." +export LOCATION="eastus2" +export ACA_ENV="..." +export ACR_NAME="..." +export APPLICATIONINSIGHTS_CONNECTION_STRING="InstrumentationKey=...;IngestionEndpoint=..." +export AZURE_OPENAI_ENDPOINT="https://.openai.azure.com" +export AZURE_OPENAI_DEPLOYMENT="gpt-4o-mini" +export AZURE_OPENAI_API_VERSION="2024-10-21" +export AZURE_OPENAI_API_KEY="..." +``` + +PowerShell users: set the same names with `$env:NAME = "..."`. + +## Step 2 — Deploy the external runtime to ACA and generate traffic + +```bash +cd samples/python/external-agents/observability +./deploy.sh # or .\deploy.ps1 on Windows +``` + +The script will: + +1. `az acr build` the image. +2. Create or update the Container App. +3. Wait for `/healthz`. +4. Run [generate_traffic.py](generate_traffic.py) to ask several weather + questions. +5. Wait for OTel export and ingestion. Override the default 90 seconds + with `TRACE_INGEST_WAIT_SECS` if needed. + +## Step 3 — Register the external agent in Foundry + +```bash +python register_external_agent.py +``` + +This calls `project_client.agents.create_version(...)` with an +`ExternalAgentDefinition`, which atomically creates the Foundry agent +record on first call (per the spec, external agents are versionless +from the user's perspective). After registration succeeds, open the +Foundry portal: + +> **Project → Agents → `weather-agent` → Traces** + +You should see the spans you just generated, attributed to the new +`external` agent. + +## Step 4 — Run a one-off trace evaluation + +```bash +python run_trace_eval.py +``` + +This: + +1. Resolves the registered agent's `otel_agent_id`. +2. Creates an OpenAI-compatible eval group with two built-in trace + evaluators (`intent_resolution`, `task_adherence`). +3. Creates an `azure_ai_traces_preview` run scoped to that + `agent_id` over the last 24 hours. +4. Polls until completion and prints per-criterion pass/fail counts. + +> The trace-eval surface is currently only on the OpenAI-compatible +> `evals` API (`project_client.get_openai_client().evals`). When the +> native `project_client.evaluations` surface adds trace-eval support, +> this sample will move there. + +## Cleanup + +```bash +az containerapp delete -n "$AGENT_NAME" -g "$RESOURCE_GROUP" --yes +# Optional: delete the Foundry registration (does not affect the runtime) +python -c "import os; from azure.identity import DefaultAzureCredential; \ +from azure.ai.projects import AIProjectClient; \ +AIProjectClient(endpoint=os.environ['FOUNDRY_PROJECT_ENDPOINT'], \ +credential=DefaultAzureCredential(), allow_preview=True) \ +.agents.delete(agent_name=os.environ.get('AGENT_NAME','weather-agent'))" +``` diff --git a/samples/python/external-agents/observability/deploy.ps1 b/samples/python/external-agents/observability/deploy.ps1 new file mode 100644 index 00000000..a20cb73d --- /dev/null +++ b/samples/python/external-agents/observability/deploy.ps1 @@ -0,0 +1,105 @@ +<# +.SYNOPSIS + Deploy the weather agent to Azure Container Apps and generate traffic. + +.DESCRIPTION + Mirrors deploy.sh for Windows / PowerShell users. Requires the + Azure CLI (`az`) and Python on PATH. + +.NOTES + Required env vars: + AZURE_SUBSCRIPTION_ID, + RESOURCE_GROUP, LOCATION, ACA_ENV, ACR_NAME, + APPLICATIONINSIGHTS_CONNECTION_STRING, + AZURE_OPENAI_ENDPOINT, AZURE_OPENAI_DEPLOYMENT + Optional: + AZURE_OPENAI_API_KEY, AGENT_NAME (default weather-agent), IMAGE_TAG (default current timestamp), + TRACE_INGEST_WAIT_SECS (default 90) +#> + +$ErrorActionPreference = "Stop" + +function Require-Env($name) { + if (-not (Get-Item "Env:$name" -ErrorAction SilentlyContinue)) { + throw "Missing required env var: $name" + } +} + +foreach ($v in @( + "AZURE_SUBSCRIPTION_ID", + "RESOURCE_GROUP","LOCATION","ACA_ENV","ACR_NAME", + "APPLICATIONINSIGHTS_CONNECTION_STRING", + "AZURE_OPENAI_ENDPOINT","AZURE_OPENAI_DEPLOYMENT" +)) { Require-Env $v } + +Write-Host "==> Selecting Azure subscription" +az account set --subscription $env:AZURE_SUBSCRIPTION_ID --only-show-errors + +$AgentName = if ($env:AGENT_NAME) { $env:AGENT_NAME } else { "weather-agent" } +$OtelAgentId = if ($env:OTEL_AGENT_ID) { $env:OTEL_AGENT_ID } else { $AgentName } +$ImageTag = if ($env:IMAGE_TAG) { $env:IMAGE_TAG } else { Get-Date -Format "yyyyMMddHHmmss" } +$TraceIngestWaitSecs = if ($env:TRACE_INGEST_WAIT_SECS) { [int]$env:TRACE_INGEST_WAIT_SECS } else { 90 } +$Image = "$($env:ACR_NAME).azurecr.io/${AgentName}:${ImageTag}" + +Write-Host "==> Building and pushing image to ACR" +az acr build --registry $env:ACR_NAME --image "${AgentName}:${ImageTag}" --no-logs --only-show-errors . | Out-Null + +Write-Host "==> Deploying to Azure Container Apps" +$envVars = @( + "AGENT_NAME=$AgentName", + "OTEL_AGENT_ID=$OtelAgentId", + "APPLICATIONINSIGHTS_CONNECTION_STRING=$($env:APPLICATIONINSIGHTS_CONNECTION_STRING)", + "AZURE_OPENAI_ENDPOINT=$($env:AZURE_OPENAI_ENDPOINT)", + "AZURE_OPENAI_DEPLOYMENT=$($env:AZURE_OPENAI_DEPLOYMENT)", + "AZURE_OPENAI_API_VERSION=$(if ($env:AZURE_OPENAI_API_VERSION) { $env:AZURE_OPENAI_API_VERSION } else { '2024-10-21' })", + "AZURE_OPENAI_API_KEY=$($env:AZURE_OPENAI_API_KEY)" +) + +$exists = az containerapp show -n $AgentName -g $env:RESOURCE_GROUP --only-show-errors 2>$null +if ($LASTEXITCODE -eq 0 -and $exists) { + az containerapp update ` + -n $AgentName ` + -g $env:RESOURCE_GROUP ` + --image $Image ` + --set-env-vars $envVars ` + --only-show-errors | Out-Null +} else { + az containerapp create ` + --name $AgentName ` + --resource-group $env:RESOURCE_GROUP ` + --environment $env:ACA_ENV ` + --image $Image ` + --target-port 8000 ` + --ingress external ` + --min-replicas 1 --max-replicas 2 ` + --registry-server "$($env:ACR_NAME).azurecr.io" ` + --system-assigned ` + --registry-identity system ` + --env-vars $envVars ` + --only-show-errors | Out-Null +} + +$Fqdn = az containerapp show -n $AgentName -g $env:RESOURCE_GROUP ` + --query properties.configuration.ingress.fqdn -o tsv --only-show-errors +$AgentUrl = "https://$Fqdn" +Write-Host "==> Agent URL: $AgentUrl" + +Write-Host "==> Waiting for /healthz" +for ($i = 0; $i -lt 30; $i++) { + try { Invoke-WebRequest -UseBasicParsing -Uri "$AgentUrl/healthz" -TimeoutSec 10 | Out-Null; break } + catch { Start-Sleep -Seconds 5 } +} + +Write-Host "==> Generating traffic" +$env:AGENT_URL = $AgentUrl +$env:AGENT_NAME = $AgentName +$env:OTEL_AGENT_ID = $OtelAgentId +python generate_traffic.py + +if ($TraceIngestWaitSecs -gt 0) { + Write-Host "==> Waiting ${TraceIngestWaitSecs}s for OTel export and ingestion" + Start-Sleep -Seconds $TraceIngestWaitSecs +} + +Write-Host "==> Done. Agent URL: $AgentUrl" +Write-Host "==> Then run: python register_external_agent.py; python run_trace_eval.py" diff --git a/samples/python/external-agents/observability/deploy.sh b/samples/python/external-agents/observability/deploy.sh new file mode 100644 index 00000000..bdb99e10 --- /dev/null +++ b/samples/python/external-agents/observability/deploy.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# Deploy the weather agent to Azure Container Apps and generate traffic. +# +# Required env vars: +# AZURE_SUBSCRIPTION_ID - Azure subscription to deploy into +# RESOURCE_GROUP - existing RG +# LOCATION - e.g. eastus2 +# ACA_ENV - existing Container Apps env name +# ACR_NAME - existing Azure Container Registry +# APPLICATIONINSIGHTS_CONNECTION_STRING +# AZURE_OPENAI_ENDPOINT +# AZURE_OPENAI_DEPLOYMENT +# AZURE_OPENAI_API_KEY - or rely on AAD via DefaultAzureCredential +# +# Optional: +# AGENT_NAME (default: weather-agent) +# IMAGE_TAG (default: current timestamp) +# TRACE_INGEST_WAIT_SECS (default: 90) + +set -euo pipefail + +AGENT_NAME="${AGENT_NAME:-weather-agent}" +IMAGE_TAG="${IMAGE_TAG:-$(date +%Y%m%d%H%M%S)}" +TRACE_INGEST_WAIT_SECS="${TRACE_INGEST_WAIT_SECS:-90}" +IMAGE="${ACR_NAME}.azurecr.io/${AGENT_NAME}:${IMAGE_TAG}" + +: "${AZURE_SUBSCRIPTION_ID:?}" +: "${RESOURCE_GROUP:?}"; : "${LOCATION:?}"; : "${ACA_ENV:?}"; : "${ACR_NAME:?}" +: "${APPLICATIONINSIGHTS_CONNECTION_STRING:?}" +: "${AZURE_OPENAI_ENDPOINT:?}"; : "${AZURE_OPENAI_DEPLOYMENT:?}" + +echo "==> Selecting Azure subscription" +az account set --subscription "$AZURE_SUBSCRIPTION_ID" --only-show-errors + +echo "==> Building and pushing image to ACR" +az acr build \ + --registry "$ACR_NAME" \ + --image "${AGENT_NAME}:${IMAGE_TAG}" \ + --no-logs \ + --only-show-errors \ + . + +echo "==> Deploying to Azure Container Apps" +az containerapp create \ + --name "$AGENT_NAME" \ + --resource-group "$RESOURCE_GROUP" \ + --environment "$ACA_ENV" \ + --image "$IMAGE" \ + --target-port 8000 \ + --ingress external \ + --min-replicas 1 \ + --max-replicas 2 \ + --registry-server "${ACR_NAME}.azurecr.io" \ + --system-assigned \ + --registry-identity system \ + --env-vars \ + AGENT_NAME="$AGENT_NAME" \ + OTEL_AGENT_ID="${OTEL_AGENT_ID:-$AGENT_NAME}" \ + APPLICATIONINSIGHTS_CONNECTION_STRING="$APPLICATIONINSIGHTS_CONNECTION_STRING" \ + AZURE_OPENAI_ENDPOINT="$AZURE_OPENAI_ENDPOINT" \ + AZURE_OPENAI_DEPLOYMENT="$AZURE_OPENAI_DEPLOYMENT" \ + AZURE_OPENAI_API_VERSION="${AZURE_OPENAI_API_VERSION:-2024-10-21}" \ + AZURE_OPENAI_API_KEY="${AZURE_OPENAI_API_KEY:-}" \ + --output none \ + --only-show-errors \ + || az containerapp update \ + --name "$AGENT_NAME" \ + --resource-group "$RESOURCE_GROUP" \ + --image "$IMAGE" \ + --set-env-vars \ + AGENT_NAME="$AGENT_NAME" \ + OTEL_AGENT_ID="${OTEL_AGENT_ID:-$AGENT_NAME}" \ + APPLICATIONINSIGHTS_CONNECTION_STRING="$APPLICATIONINSIGHTS_CONNECTION_STRING" \ + AZURE_OPENAI_ENDPOINT="$AZURE_OPENAI_ENDPOINT" \ + AZURE_OPENAI_DEPLOYMENT="$AZURE_OPENAI_DEPLOYMENT" \ + AZURE_OPENAI_API_VERSION="${AZURE_OPENAI_API_VERSION:-2024-10-21}" \ + AZURE_OPENAI_API_KEY="${AZURE_OPENAI_API_KEY:-}" \ + --output none \ + --only-show-errors + +FQDN=$(az containerapp show -n "$AGENT_NAME" -g "$RESOURCE_GROUP" \ + --query properties.configuration.ingress.fqdn -o tsv --only-show-errors) +AGENT_URL="https://${FQDN}" +echo "==> Agent URL: $AGENT_URL" + +echo "==> Waiting for /healthz" +for i in {1..30}; do + if curl -fsS --max-time 10 "${AGENT_URL}/healthz" >/dev/null; then break; fi + sleep 5 +done + +echo "==> Generating traffic" +AGENT_URL="$AGENT_URL" AGENT_NAME="$AGENT_NAME" OTEL_AGENT_ID="${OTEL_AGENT_ID:-$AGENT_NAME}" python generate_traffic.py + +if [ "$TRACE_INGEST_WAIT_SECS" -gt 0 ]; then + echo "==> Waiting ${TRACE_INGEST_WAIT_SECS}s for OTel export and ingestion" + sleep "$TRACE_INGEST_WAIT_SECS" +fi + +echo "==> Done. Agent URL: $AGENT_URL" +echo "==> Then run: python register_external_agent.py && python run_trace_eval.py" diff --git a/samples/python/external-agents/observability/generate_traffic.py b/samples/python/external-agents/observability/generate_traffic.py new file mode 100644 index 00000000..4a1d0002 --- /dev/null +++ b/samples/python/external-agents/observability/generate_traffic.py @@ -0,0 +1,36 @@ +"""Hit the deployed weather agent with varied questions so there are +spans in Application Insights for the eval run to score. +""" + +from __future__ import annotations + +import os +import sys + +import httpx + +QUESTIONS = [ + "What's the weather in Seattle right now?", + "Give me a 5-day forecast for Tokyo.", + "Compare today's weather in New York and London.", + "Should I bring an umbrella in Seattle today?", + "What's the warmest of these cities right now: Seattle, Tokyo, London?", +] + + +def main() -> None: + base_url = os.environ.get("AGENT_URL") or (sys.argv[1] if len(sys.argv) > 1 else None) + if not base_url: + raise SystemExit("Set AGENT_URL or pass the agent base URL as argv[1].") + + base_url = base_url.rstrip("/") + with httpx.Client(timeout=60.0) as client: + for question in QUESTIONS: + print(f"\n>>> {question}") + response = client.post(f"{base_url}/ask", json={"question": question}) + response.raise_for_status() + print(response.json().get("answer")) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/samples/python/external-agents/observability/register_external_agent.py b/samples/python/external-agents/observability/register_external_agent.py new file mode 100644 index 00000000..cd8fd6d8 --- /dev/null +++ b/samples/python/external-agents/observability/register_external_agent.py @@ -0,0 +1,62 @@ +"""Register the externally hosted weather agent in Foundry. + +After this runs successfully, open the Foundry portal -> your project -> +Agents -> ``weather-agent`` to see the trace view light up with spans +emitted by the running container. + +Prereqs: + * FOUNDRY_PROJECT_ENDPOINT env var + * AAD credentials with permission to create agents in the project + * The external runtime (weather_agent.py) is already emitting OTel + spans to the Application Insights connected to the Foundry project +""" + +from __future__ import annotations + +import os + +from azure.ai.projects import AIProjectClient +from azure.ai.projects.models import ExternalAgentDefinition +from azure.identity import DefaultAzureCredential + +AGENT_NAME = os.environ.get("AGENT_NAME", "weather-agent") +# The id the running agent emits as gen_ai.agent.id on its OTel spans. +# Defaults to AGENT_NAME but can differ -- e.g. "weather-agent-v1". +OTEL_AGENT_ID = os.environ.get("OTEL_AGENT_ID", AGENT_NAME) + + +def main() -> None: + project_client = AIProjectClient( + endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], + credential=DefaultAzureCredential(), + # External agents are gated behind a preview feature flag during + # public preview. See spec: Foundry-Features: ExternalAgents=V1Preview. + allow_preview=True, + ) + + # First-time create goes through create_version() in azure-ai-projects + # v2.1.0; called with a new agent_name it atomically creates the + # agent + first registration revision. External agents are + # versionless from the user's perspective. + agent = project_client.agents.create_version( + agent_name=AGENT_NAME, + description="Weather agent hosted externally on Azure Container Apps.", + definition=ExternalAgentDefinition( + # Optional: defaults to agent_name. Set explicitly here so + # the mapping to gen_ai.agent.id on the OTel spans is + # obvious -- and to show otel_agent_id can differ from the + # Foundry agent name (e.g. "weather-agent-v1"). + otel_agent_id=OTEL_AGENT_ID, + ), + ) + + print(f"Registered external agent: {agent.name}") + print(f"Resolved otel_agent_id : {agent.versions.latest.definition.otel_agent_id}") + print() + print("Open the Foundry portal and navigate to:") + print(f" Project -> Agents -> {agent.name} -> Traces") + print("to see traces emitted by the external runtime.") + + +if __name__ == "__main__": + main() diff --git a/samples/python/external-agents/observability/requirements.txt b/samples/python/external-agents/observability/requirements.txt new file mode 100644 index 00000000..c964c3e4 --- /dev/null +++ b/samples/python/external-agents/observability/requirements.txt @@ -0,0 +1,10 @@ +azure-ai-projects>=2.1.0 +azure-identity>=1.17.0 +microsoft-opentelemetry[langchain]>=1.2.0 +langchain>=0.3.0 +langchain-openai>=0.2.0 +langchain-core>=0.3.0 +fastapi>=0.115.0 +uvicorn[standard]>=0.30.0 +httpx>=0.27.0 +openai>=1.50.0 diff --git a/samples/python/external-agents/observability/run_trace_eval.py b/samples/python/external-agents/observability/run_trace_eval.py new file mode 100644 index 00000000..20a029ec --- /dev/null +++ b/samples/python/external-agents/observability/run_trace_eval.py @@ -0,0 +1,116 @@ +"""Run a one-off trace-based evaluation over the registered external +weather agent and print the per-criterion scores. + +The trace-eval surface is currently exposed only via the OpenAI-compatible +``evals`` API (``azure_ai_source`` / ``azure_ai_traces_preview`` data sources), +so we go through ``project_client.get_openai_client()`` per the spec. +""" + +from __future__ import annotations + +import os +import time + +from azure.ai.projects import AIProjectClient +from azure.identity import DefaultAzureCredential + +AGENT_NAME = os.environ.get("AGENT_NAME", "weather-agent") +EVAL_DEPLOYMENT = os.environ.get( + "EVAL_MODEL_DEPLOYMENT", + os.environ.get("AZURE_OPENAI_DEPLOYMENT", "gpt-4o-mini"), +) +LOOKBACK_HOURS = int(os.environ.get("LOOKBACK_HOURS", "24")) +POLL_TIMEOUT_SECS = int(os.environ.get("EVAL_POLL_TIMEOUT_SECS", "900")) + + +def main() -> None: + project_client = AIProjectClient( + endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], + credential=DefaultAzureCredential(), + allow_preview=True, + ) + + # Resolve otel_agent_id from the registration so we evaluate exactly + # the spans Foundry attributes to this agent. + agent = project_client.agents.get(agent_name=AGENT_NAME) + otel_agent_id = agent.versions.latest.definition.otel_agent_id + print(f"Evaluating traces for agent_id={otel_agent_id} (last {LOOKBACK_HOURS}h)") + + openai_client = project_client.get_openai_client() + + # 1. Eval group -- defines what we measure. + eval_group = openai_client.evals.create( + name=f"{AGENT_NAME}-trace-eval", + data_source_config={"type": "azure_ai_source", "scenario": "traces_preview"}, + testing_criteria=[ + { + "type": "azure_ai_evaluator", + "name": "intent_resolution", + "evaluator_name": "builtin.intent_resolution", + "data_mapping": { + "query": "{{query}}", + "response": "{{response}}", + "tool_definitions": "{{tool_definitions}}", + }, + "initialization_parameters": {"deployment_name": EVAL_DEPLOYMENT}, + }, + { + "type": "azure_ai_evaluator", + "name": "task_adherence", + "evaluator_name": "builtin.task_adherence", + "data_mapping": { + "query": "{{query}}", + "response": "{{response}}", + "tool_definitions": "{{tool_definitions}}", + }, + "initialization_parameters": {"deployment_name": EVAL_DEPLOYMENT}, + }, + ], + ) + + # 2. One-off run -- scoped to this agent's traces over the lookback. + run = openai_client.evals.runs.create( + eval_id=eval_group.id, + name=f"{AGENT_NAME}-trace-run", + data_source={ + "type": "azure_ai_traces_preview", + "agent_id": otel_agent_id, + "lookback_hours": LOOKBACK_HOURS, + }, + ) + print(f"Created eval run {run.id}; polling for completion...") + + # 3. Poll until terminal. + deadline = time.time() + POLL_TIMEOUT_SECS + terminal = {"completed", "failed", "canceled"} + while time.time() < deadline: + run = openai_client.evals.runs.retrieve(run_id=run.id, eval_id=eval_group.id) + if run.status in terminal: + break + print(f" status={run.status} ...") + time.sleep(15) + else: + raise TimeoutError(f"Eval run did not finish within {POLL_TIMEOUT_SECS}s") + + print(f"\nEval run finished: status={run.status}") + if run.status != "completed": + print(run) + return + + # 4. Print per-criterion aggregate scores. + print("\nResult counts:") + print(f" passed : {getattr(run.result_counts, 'passed', 'n/a')}") + print(f" failed : {getattr(run.result_counts, 'failed', 'n/a')}") + print(f" errored: {getattr(run.result_counts, 'errored', 'n/a')}") + print(f" total : {getattr(run.result_counts, 'total', 'n/a')}") + + print("\nPer-criterion scores:") + for tc in getattr(run, "per_testing_criteria_results", []) or []: + name = getattr(tc, "testing_criteria", None) or getattr(tc, "name", "?") + passed = getattr(tc, "passed", "?") + failed = getattr(tc, "failed", "?") + print(f" - {name}: passed={passed} failed={failed}") + + +if __name__ == "__main__": + main() diff --git a/samples/python/external-agents/observability/weather_agent.py b/samples/python/external-agents/observability/weather_agent.py new file mode 100644 index 00000000..dcdc32a6 --- /dev/null +++ b/samples/python/external-agents/observability/weather_agent.py @@ -0,0 +1,130 @@ +"""Weather agent built with LangChain, instrumented with the Microsoft +OpenTelemetry distro so its spans flow into Application Insights. + +This file is the runtime that lives *outside* Foundry (e.g. on Azure +Container Apps, GCP Cloud Run, AWS, on-prem...). Foundry only stores a +registration record (see ``register_external_agent.py``) and reads the +spans this process emits. + +Reference for the LangChain + distro setup: +https://github.com/microsoft/opentelemetry-distro-python/blob/main/samples/langchain/sample_langchain_instrumentation.py + +The Microsoft distro is configured with an explicit LangChain +``agent_id`` so emitted spans line up with the Foundry external-agent +registration. +""" + +from __future__ import annotations + +import os +from contextlib import asynccontextmanager + +# --- OTel: configure the Microsoft distro BEFORE importing anything that +# should be instrumented at runtime. ---------------------------------------- +from microsoft.opentelemetry import use_microsoft_opentelemetry # type: ignore + +AGENT_NAME = os.environ.get("AGENT_NAME", "weather-agent") +# Emitted on every OTel span as gen_ai.agent.id. +OTEL_AGENT_ID = os.environ.get("OTEL_AGENT_ID", AGENT_NAME) + +use_microsoft_opentelemetry( + enable_azure_monitor=True, + sampling_ratio=1.0, + instrumentation_options={ + "fastapi": {"enabled": False}, + "langchain": { + "enabled": True, + "agent_id": OTEL_AGENT_ID, + "agent_name": AGENT_NAME, + }, + }, +) + +from fastapi import FastAPI +from langchain_core.messages import HumanMessage, SystemMessage +from langchain_core.tools import tool +from langchain_openai import AzureChatOpenAI +from langgraph.prebuilt import create_react_agent +from pydantic import BaseModel + + +# --- Tools ----------------------------------------------------------------- +@tool +def get_current_weather(city: str) -> str: + """Return the current weather for the given city. + + This is a stub that returns deterministic fake data so the sample is + runnable without a third-party weather API key. + """ + fake = { + "seattle": "59F and raining", + "new york": "72F and partly cloudy", + "tokyo": "68F and clear", + "london": "55F and overcast", + } + return fake.get(city.lower(), f"70F and sunny in {city}") + + +@tool +def get_forecast(city: str, days: int = 3) -> str: + """Return a short multi-day forecast for the given city.""" + return f"{days}-day forecast for {city}: mild temperatures, occasional showers." + + +# --- Agent ----------------------------------------------------------------- +def build_agent(): + llm = AzureChatOpenAI( + azure_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"], + azure_deployment=os.environ["AZURE_OPENAI_DEPLOYMENT"], + api_version=os.environ.get("AZURE_OPENAI_API_VERSION", "2024-10-21"), + api_key=os.environ.get("AZURE_OPENAI_API_KEY"), + temperature=0, + ) + return create_react_agent( + model=llm, + tools=[get_current_weather, get_forecast], + prompt=SystemMessage( + content=( + "You are a helpful weather assistant. Use the provided " + "tools to answer questions about current weather and " + "short-term forecasts. Be concise." + ) + ), + ) + + +# --- HTTP surface ---------------------------------------------------------- +class AskRequest(BaseModel): + question: str + + +@asynccontextmanager +async def lifespan(app: FastAPI): + app.state.agent = build_agent() + yield + + +app = FastAPI(title=AGENT_NAME, lifespan=lifespan) + + +@app.get("/healthz") +def healthz(): + return {"status": "ok", "agent": AGENT_NAME} + + +@app.post("/ask") +def ask(req: AskRequest): + agent = app.state.agent + result = agent.invoke({"messages": [HumanMessage(content=req.question)]}) + final = result["messages"][-1].content + return {"agent": AGENT_NAME, "answer": final} + + +if __name__ == "__main__": + import uvicorn + + uvicorn.run( + "weather_agent:app", + host="0.0.0.0", + port=int(os.environ.get("PORT", "8000")), + )