diff --git a/.github/workflows/check-models.yml b/.github/workflows/check-models.yml index 4bc50f8..47fdd88 100644 --- a/.github/workflows/check-models.yml +++ b/.github/workflows/check-models.yml @@ -28,7 +28,7 @@ jobs: - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e with: - node-version: 20 + node-version: 24 - name: Scan touched files for outdated models id: scan @@ -40,7 +40,7 @@ jobs: cat model-scan.json - name: Comment and gate - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 (Node 24) with: script: | const fs = require('fs'); diff --git a/python/llm/evaluation/building-a-custom-evaluator.ipynb b/python/llm/evaluation/building-a-custom-evaluator.ipynb deleted file mode 100644 index 4a1aa32..0000000 --- a/python/llm/evaluation/building-a-custom-evaluator.ipynb +++ /dev/null @@ -1,602 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "hdr-logo", - "metadata": {}, - "source": [ - "
\n", - "

\n", - " \"arize\n", - "
\n", - " Docs\n", - " |\n", - " GitHub\n", - " |\n", - " Slack Community\n", - "

\n", - "
" - ] - }, - { - "cell_type": "markdown", - "id": "title", - "metadata": {}, - "source": [ - "# Using a Benchmark Dataset to Build a Custom LLM as a Judge Evaluator" - ] - }, - { - "cell_type": "markdown", - "id": "intro", - "metadata": {}, - "source": [ - "In this tutorial, you'll learn how to build a custom LLM-as-a-Judge Evaluator tailored to your specific use case. While Arize AX provides several [pre-built evaluators](https://arize.com/docs/ax/evaluate/llm-as-a-judge/arize-evaluators-llm-as-a-judge) that have been tested against benchmark datasets, these may not always cover the nuances of your application.\n", - "\n", - "So how can you achieve the same level of rigor when your use case falls outside the scope of standard evaluators?\n", - "\n", - "The discipline is simple: **don't trust a judge until you've measured it against ground truth.** We'll build a small benchmark dataset from a handful of human-annotated examples, then use it to build and refine a custom evaluator β€” measuring how often the judge agrees with the human labels, and iterating the judge prompt until that agreement is high enough to trust. The use case we'll explore is data extraction from an image of a receipt.\n", - "\n", - "To follow along, you'll need:\n", - "\n", - "* A free [Arize AX](https://app.arize.com/auth/join) account\n", - "* An OpenAI API Key" - ] - }, - { - "cell_type": "markdown", - "id": "setup-hdr", - "metadata": {}, - "source": [ - "## Set up Keys and Dependencies" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "pip", - "metadata": {}, - "outputs": [], - "source": [ - "%pip install -qqqq arize arize-otel openinference-instrumentation-openai openai nest_asyncio pandas" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "keys", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import pandas as pd\n", - "import nest_asyncio\n", - "from getpass import getpass\n", - "\n", - "nest_asyncio.apply()\n", - "\n", - "\n", - "if \"SPACE_ID\" not in os.environ:\n", - " os.environ[\"SPACE_ID\"] = getpass(\"πŸ”‘ Enter your Arize Space ID: \")\n", - "\n", - "if \"API_KEY\" not in os.environ:\n", - " os.environ[\"API_KEY\"] = getpass(\"πŸ”‘ Enter your Arize API Key: \")\n", - "\n", - "if \"OPENAI_API_KEY\" not in os.environ:\n", - " os.environ[\"OPENAI_API_KEY\"] = getpass(\"πŸ”‘ Enter your OpenAI API Key: \")" - ] - }, - { - "cell_type": "markdown", - "id": "tracing-hdr", - "metadata": {}, - "source": [ - "# Configure Tracing" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "register", - "metadata": {}, - "outputs": [], - "source": [ - "from arize.otel import register\n", - "from openinference.instrumentation.openai import OpenAIInstrumentor\n", - "\n", - "MODEL = \"gpt-5.4-mini\"\n", - "JUDGE_MODEL = \"gpt-4.1\"\n", - "\n", - "tracer_provider = register(\n", - " space_id=os.environ[\"SPACE_ID\"],\n", - " api_key=os.environ[\"API_KEY\"],\n", - " project_name=\"receipt-classifications\",\n", - ")\n", - "OpenAIInstrumentor().instrument(tracer_provider=tracer_provider)" - ] - }, - { - "cell_type": "markdown", - "id": "traces-hdr", - "metadata": {}, - "source": [ - "# Generate Image Classification Traces" - ] - }, - { - "cell_type": "markdown", - "id": "traces-desc", - "metadata": {}, - "source": [ - "In this tutorial, we'll ask an LLM to generate expense reports from receipt images provided as public URLs. Running the cells below will generate traces, which you can explore directly in Arize AX for annotation. We'll use `gpt-5.4-mini`, which supports image inputs.\n", - "\n", - "\n", - "Dataset Information:\n", - "Jakob (2024). Receipt or Invoice Dataset. Roboflow Universe. CC BY 4.0. Available at: https://universe.roboflow.com/jakob-awn1e/receipt-or-invoice (accessed on 2025-07-29)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "urls", - "metadata": {}, - "outputs": [], - "source": [ - "import uuid\n", - "import pandas as pd\n", - "urls = [\n", - "\"https://source.roboflow.com/Zf1kEIcRTrhHBZ7wgJleS4E92P23/8M5px2yLoNtZ6gOQ2r1D/original.jpg\",\n", - "\"https://source.roboflow.com/Zf1kEIcRTrhHBZ7wgJleS4E92P23/8EVgYMNObyV6kLqBNeFG/original.jpg\",\n", - "\"https://source.roboflow.com/Zf1kEIcRTrhHBZ7wgJleS4E92P23/86aohWmcEfO0XkflO8AB/original.jpg\",\n", - "\"https://source.roboflow.com/HahhKcbQqdf8YAudM4kU3PuVCS72/1eGPBChz7wvovQROk2l8/original.jpg\",\n", - "\"https://source.roboflow.com/HahhKcbQqdf8YAudM4kU3PuVCS72/0WqR2GSfGmxWB7ozo3Pj/original.jpg\",\n", - "\"https://source.roboflow.com/HahhKcbQqdf8YAudM4kU3PuVCS72/FAEJRtviIboCYSKFZcEZ/original.jpg\",\n", - "\"https://source.roboflow.com/HahhKcbQqdf8YAudM4kU3PuVCS72/0AoEaFy8FAw6DVieWCa8/original.jpg\",\n", - "\"https://source.roboflow.com/HahhKcbQqdf8YAudM4kU3PuVCS72/0Q3hAyNwXNpHTeoWU7fz/original.jpg\",\n", - "\"https://source.roboflow.com/HahhKcbQqdf8YAudM4kU3PuVCS72/2r876u4WpaCYFdMPwieK/original.jpg\",\n", - "\"https://source.roboflow.com/HahhKcbQqdf8YAudM4kU3PuVCS72/2ZWeE0yO0oJUDtpgEAPY/original.jpg\",\n", - "\"https://source.roboflow.com/HahhKcbQqdf8YAudM4kU3PuVCS72/37PF6xfHyuqzIBdO7Kgw/original.jpg\",\n", - "\"https://source.roboflow.com/Zf1kEIcRTrhHBZ7wgJleS4E92P23/6mo4M0nJeKZEsdKrRfsR/original.jpg\",\n", - "\"https://source.roboflow.com/Zf1kEIcRTrhHBZ7wgJleS4E92P23/5ezJ8tUBGbNnt0jZi2JU/original.jpg\",\n", - "\"https://source.roboflow.com/Zf1kEIcRTrhHBZ7wgJleS4E92P23/4BCIWGazhCj03oTMWboO/original.jpg\",\n", - "\"https://source.roboflow.com/Zf1kEIcRTrhHBZ7wgJleS4E92P23/4B8vXJNwJ7ZuHEWyjgAv/original.jpg\",\n", - "\"https://source.roboflow.com/Zf1kEIcRTrhHBZ7wgJleS4E92P23/2EpeKbAqsSwciH2IHGyV/original.jpg\",\n", - "\"https://source.roboflow.com/Zf1kEIcRTrhHBZ7wgJleS4E92P23/2LP3g9rKZrYDkNB3I78c/original.jpg\",\n", - "\"https://source.roboflow.com/Zf1kEIcRTrhHBZ7wgJleS4E92P23/1hT6iLEIAFBw8W70u2FY/original.jpg\",\n", - "\"https://source.roboflow.com/Zf1kEIcRTrhHBZ7wgJleS4E92P23/1zaKpaDhRPxkiIDTvMuc/original.jpg\",\n", - "\"https://source.roboflow.com/Zf1kEIcRTrhHBZ7wgJleS4E92P23/1hF1R2Pt41hnlqhlXLDD/original.jpg\"\n", - "]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "extract", - "metadata": {}, - "outputs": [], - "source": [ - "from openai import OpenAI\n", - "client = OpenAI()\n", - "\n", - "def extract_receipt_data(input):\n", - " response = client.chat.completions.create(\n", - " model=MODEL,\n", - " messages=[\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": [\n", - " {\"type\": \"text\", \"text\": \"Analyze this receipt and return a brief summary for an expense report. Only include category of expense, total cost, and summary of items\"},\n", - " {\n", - " \"type\": \"image_url\",\n", - " \"image_url\": {\n", - " \"url\": input,\n", - " },\n", - " },\n", - " ],\n", - " }\n", - " ],\n", - " )\n", - " return response" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "run-extract", - "metadata": {}, - "outputs": [], - "source": [ - "import time\n", - "\n", - "for url in urls:\n", - " extract_receipt_data(url)\n", - "\n", - "# Flush the traces so they're queryable when you annotate / export them.\n", - "tracer_provider.force_flush()\n", - "time.sleep(5)" - ] - }, - { - "cell_type": "markdown", - "id": "dataset-hdr", - "metadata": {}, - "source": [ - "# Create Benchmarked Dataset" - ] - }, - { - "cell_type": "markdown", - "id": "annotate-desc", - "metadata": {}, - "source": [ - "After generating traces, open Arize AX to begin annotating your dataset. In this example, we'll annotate based on \"accuracy\", but you can choose any evaluation criterion that fits your use case. Just be sure to update the query below to match the annotation key you're usingβ€”this ensures the annotated examples are included in your benchmark dataset.\n", - "\n", - "Run the cell below to see annotations in action:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "video", - "metadata": {}, - "outputs": [], - "source": [ - "from IPython.display import HTML\n", - "\n", - "video_url = \"https://storage.googleapis.com/arize-phoenix-assets/assets/videos/arize-annotation.mp4\"\n", - "\n", - "HTML(f\"\"\"\n", - "\n", - "\"\"\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "export", - "metadata": {}, - "outputs": [], - "source": [ - "from datetime import datetime, timedelta, timezone\n", - "\n", - "from arize.client import ArizeClient\n", - "\n", - "ax_client = ArizeClient(api_key=os.environ[\"API_KEY\"])\n", - "\n", - "print(\"#### Exporting your traces into a dataframe.\")\n", - "\n", - "primary_df = ax_client.spans.export_to_df(\n", - " space_id=os.environ[\"SPACE_ID\"],\n", - " project_name=\"receipt-classifications\",\n", - " start_time=datetime.now(timezone.utc) - timedelta(days=50),\n", - " end_time=datetime.now(timezone.utc),\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "filter", - "metadata": {}, - "outputs": [], - "source": [ - "filtered_df = primary_df[\n", - " (primary_df[\"annotation.accuracy.label\"].notna())\n", - "][[\n", - " \"attributes.input.value\",\n", - " \"attributes.output.value\",\n", - " \"annotation.accuracy.label\",\n", - "]].rename(columns={\n", - " \"attributes.input.value\": \"image\",\n", - " \"attributes.output.value\": \"response\",\n", - " \"annotation.accuracy.label\": \"accuracy\"\n", - "})\n", - "\n", - "filtered_df" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "flatten", - "metadata": {}, - "outputs": [], - "source": [ - "import json\n", - "\n", - "def extract_url(input_value):\n", - " data = json.loads(input_value)\n", - " return data[\"messages\"][0][\"content\"][1][\"image_url\"][\"url\"]\n", - "\n", - "def extract_content(input_value):\n", - " data = json.loads(input_value)\n", - " return data[\"choices\"][0][\"message\"][\"content\"]\n", - "\n", - "filtered_df[\"image\"] = filtered_df[\"image\"].apply(extract_url)\n", - "filtered_df[\"response\"] = filtered_df[\"response\"].apply(extract_content)\n", - "\n", - "\n", - "filtered_df" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "create-dataset", - "metadata": {}, - "outputs": [], - "source": [ - "from datetime import datetime, timezone\n", - "\n", - "DATASET_NAME = f\"annotated-receipts-{datetime.now(timezone.utc):%Y%m%d-%H%M%S}\"\n", - "dataset = ax_client.datasets.create(\n", - " space=os.environ[\"SPACE_ID\"],\n", - " name=DATASET_NAME,\n", - " examples=filtered_df,\n", - ")\n", - "DATASET_NAME" - ] - }, - { - "cell_type": "markdown", - "id": "dataset-img", - "metadata": {}, - "source": [ - "![Dataset](https://storage.googleapis.com/arize-phoenix-assets/assets/images/eval-tutorial-annotated-dataset.png)" - ] - }, - { - "cell_type": "markdown", - "id": "template-hdr", - "metadata": {}, - "source": [ - "# Create the custom judge" - ] - }, - { - "cell_type": "markdown", - "id": "template-desc", - "metadata": {}, - "source": [ - "Next we'll build a baseline judge and benchmark it against the annotations. The judge is an LLM-as-a-Judge that *reads the receipt image* and the model's expense report, and classifies the report as `accurate`, `almost accurate`, or `inaccurate` β€” the same labels a human annotator used. `make_judge(prompt)` binds one judge prompt into a task we can run as an experiment; the experiment's evaluator then checks whether the judge's label matches the human annotation." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "judge-v1", - "metadata": {}, - "outputs": [], - "source": [ - "RAILS = [\"accurate\", \"almost accurate\", \"inaccurate\"]\n", - "\n", - "JUDGE_PROMPT_V1 = \"\"\"You are an evaluator tasked with assessing the quality of a model-generated expense report based on a receipt.\n", - "\n", - "MODEL OUTPUT (Expense Report):\n", - "{output}\n", - "\n", - "The input receipt image is attached. Evaluate the report and assign exactly one label:\n", - "- \"accurate\" - Fully correct\n", - "- \"almost accurate\" - Mostly correct\n", - "- \"inaccurate\" - Substantially wrong\n", - "\n", - "Respond with only the label.\"\"\"\n", - "\n", - "\n", - "def make_judge(prompt):\n", - " \"\"\"Bind one judge prompt into an experiment task. The task reads the receipt\n", - " image and the model's report from the dataset row and returns one rail label.\"\"\"\n", - "\n", - " def task_function(dataset_row):\n", - " response = client.chat.completions.create(\n", - " model=JUDGE_MODEL,\n", - " temperature=0,\n", - " messages=[\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": [\n", - " {\"type\": \"text\", \"text\": prompt.format(output=dataset_row[\"response\"])},\n", - " {\"type\": \"image_url\", \"image_url\": {\"url\": dataset_row[\"image\"]}},\n", - " ],\n", - " }\n", - " ],\n", - " )\n", - " verdict = response.choices[0].message.content.strip().lower()\n", - " # Map the reply onto one of the rails (check the longer labels first, since\n", - " # \"accurate\" is a substring of \"almost accurate\").\n", - " for rail in [\"almost accurate\", \"inaccurate\", \"accurate\"]:\n", - " if rail in verdict:\n", - " return rail\n", - " return verdict\n", - "\n", - " return task_function" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "evaluator", - "metadata": {}, - "outputs": [], - "source": [ - "from arize.experiments import EvaluationResult\n", - "\n", - "\n", - "def matches_annotation(output, dataset_row) -> EvaluationResult:\n", - " \"\"\"Score the judge: does its label match the human annotation for this example?\"\"\"\n", - " expected = dataset_row[\"accuracy\"]\n", - " if output == expected:\n", - " return EvaluationResult(\n", - " score=1.0, label=\"correct\", explanation=\"Judge label matches the human annotation\"\n", - " )\n", - " return EvaluationResult(\n", - " score=0.0,\n", - " label=\"incorrect\",\n", - " explanation=f\"Judge said '{output}', annotation was '{expected}'\",\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "run-v1", - "metadata": {}, - "outputs": [], - "source": [ - "experiment_v1, results_v1 = ax_client.experiments.run(\n", - " space=os.environ[\"SPACE_ID\"],\n", - " dataset=DATASET_NAME,\n", - " task=make_judge(JUDGE_PROMPT_V1),\n", - " evaluators=[matches_annotation],\n", - " name=\"Initial Experiment\",\n", - ")\n", - "results_v1[\"eval.matches_annotation.score\"].mean()" - ] - }, - { - "cell_type": "markdown", - "id": "iter1-hdr", - "metadata": {}, - "source": [ - "# Iteration 1 to improve the evaluator prompt" - ] - }, - { - "cell_type": "markdown", - "id": "iter1-desc", - "metadata": {}, - "source": [ - "Next, we'll refine our judge prompt by adding more specific classification rules. We can add these rules based on gaps we saw in the previous iteration. This additional guidance helps improve accuracy and ensures the evaluator's judgments better align with human expectations." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "judge-v2", - "metadata": {}, - "outputs": [], - "source": [ - "JUDGE_PROMPT_V2 = \"\"\"You are an evaluator tasked with assessing the quality of a model-generated expense report based on a receipt.\n", - "\n", - "MODEL OUTPUT (Expense Report):\n", - "{output}\n", - "\n", - "The input receipt image is attached. Evaluate the report and assign exactly one label:\n", - "- \"accurate\" - Total price, itemized list, and expense category are all accurate. All three must be correct to get this label.\n", - "- \"almost accurate\" - Mostly correct but with small issues. For example, the expense category is too vague.\n", - "- \"inaccurate\" - Substantially wrong or missing information. For example, an incorrect total price.\n", - "\n", - "Respond with only the label.\"\"\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "run-v2", - "metadata": {}, - "outputs": [], - "source": [ - "experiment_v2, results_v2 = ax_client.experiments.run(\n", - " space=os.environ[\"SPACE_ID\"],\n", - " dataset=DATASET_NAME,\n", - " task=make_judge(JUDGE_PROMPT_V2),\n", - " evaluators=[matches_annotation],\n", - " name=\"Stronger Prompt Experiment\",\n", - ")\n", - "results_v2[\"eval.matches_annotation.score\"].mean()" - ] - }, - { - "cell_type": "markdown", - "id": "iter2-hdr", - "metadata": {}, - "source": [ - "# Iteration 2 to improve the evaluator prompt" - ] - }, - { - "cell_type": "markdown", - "id": "iter2-desc", - "metadata": {}, - "source": [ - "To further improve our evaluator, we'll add few-shot guidance to the prompt. These examples help highlight common failure cases and guide the evaluator toward more consistent and generalized judgments." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "judge-v3", - "metadata": {}, - "outputs": [], - "source": [ - "JUDGE_PROMPT_V3 = \"\"\"You are an evaluator tasked with assessing the quality of a model-generated expense report based on a receipt.\n", - "\n", - "MODEL OUTPUT (Expense Report):\n", - "{output}\n", - "\n", - "The input receipt image is attached. Evaluate the report and assign exactly one label:\n", - "- \"accurate\" - Total price, itemized list, and expense category are accurate. All three must be correct to get this label.\n", - " An incorrect category is one that is overly vague (e.g., \"Miscellaneous\", \"Supplies\") or does not accurately reflect the itemized list.\n", - " For example, \"Dining and Entertainment\" should not be grouped together if the itemized list only includes food.\n", - " Reasonable general categories like \"Office Supplies\" or \"Groceries\" are acceptable if they align with the listed items.\n", - "- \"almost accurate\" - Mostly correct but with small issues. For example, the expense category is too vague.\n", - " If a category includes extra fields (e.g., \"Dining and Entertainment\" but the receipt only includes food) mark this as almost accurate.\n", - "- \"inaccurate\" - Substantially wrong or missing. For example, an incorrect total price, or one or more missing items, makes the result inaccurate.\n", - "\n", - "Respond with only the label.\"\"\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "run-v3", - "metadata": {}, - "outputs": [], - "source": [ - "experiment_v3, results_v3 = ax_client.experiments.run(\n", - " space=os.environ[\"SPACE_ID\"],\n", - " dataset=DATASET_NAME,\n", - " task=make_judge(JUDGE_PROMPT_V3),\n", - " evaluators=[matches_annotation],\n", - " name=\"Few Shot Experiment\",\n", - ")\n", - "results_v3[\"eval.matches_annotation.score\"].mean()" - ] - }, - { - "cell_type": "markdown", - "id": "final-hdr", - "metadata": {}, - "source": [ - "# Final Results" - ] - }, - { - "cell_type": "markdown", - "id": "final-desc", - "metadata": {}, - "source": [ - "Each experiment reports the share of examples where the judge's label matched the human annotation β€” that agreement rate is how you know whether to trust the judge. Compare the three runs in the **Experiments** tab of your dataset (or from the `results_*` dataframes above) and watch the agreement climb as the prompt is refined.\n", - "\n", - "Once your evaluator reaches a performance level you're satisfied with, it's ready for use. The target score will depend on your benchmark dataset and specific use case. That said, you can continue applying the techniques from this tutorial to refine and iterate until the evaluator meets your desired level of quality." - ] - }, - { - "cell_type": "markdown", - "id": "final-img", - "metadata": {}, - "source": [ - "![Final Results](https://storage.googleapis.com/arize-phoenix-assets/assets/images/eval-tutorial-compare-experiment.png)" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/.gitignore b/typescript/cookbooks/mastra-arize-ax-tracing/.gitignore index 03bb9ce..3192fdb 100644 --- a/typescript/cookbooks/mastra-arize-ax-tracing/.gitignore +++ b/typescript/cookbooks/mastra-arize-ax-tracing/.gitignore @@ -1,3 +1,6 @@ node_modules .mastra/ package-lock.json +*.db +*.db-shm +*.db-wal diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/README.md b/typescript/cookbooks/mastra-arize-ax-tracing/README.md index 56a2abe..477da0c 100644 --- a/typescript/cookbooks/mastra-arize-ax-tracing/README.md +++ b/typescript/cookbooks/mastra-arize-ax-tracing/README.md @@ -1,10 +1,10 @@ # Mastra Agent with Arize AX Tracing -Companion repo for the [Arize AX tutorial](https://arize.com/docs/ax/). A Mastra orchestrator agent instrumented with OpenInference tracing that sends spans to Arize AX. +Companion repo for the Arize AX guide [Align LLM Evals with Human Judgment](https://arize.com/docs/ax/cookbooks/evaluate/align-llm-evals-with-human-judgment). A Mastra orchestrator agent instrumented with OpenInference tracing that sends spans to Arize AX. ## Prerequisites -- [Node.js](https://nodejs.org) v20.9.0+ +- [Node.js](https://nodejs.org) v22.13.0+ - An [Arize AX](https://app.arize.com) account (Space ID and API key) - An [OpenAI](https://platform.openai.com) API key @@ -28,27 +28,42 @@ export ARIZE_SPACE_ID="your-arize-space-id" npm start ``` -This starts the Mastra dev server and opens the agent UI. Send requests to the weather orchestrator agent to generate traces β€” they will appear in your Arize AX space within a few seconds. +This starts the Mastra dev server. Open [Mastra Studio](https://mastra.ai/docs/studio/overview) at [http://localhost:4111](http://localhost:4111) in your browser (Mastra picks the next free port if 4111 is taken), select the **WeatherOrchestratorAgent**, and send it a request such as "What should I do in London today?" to generate traces β€” they will appear in your Arize AX space within a few seconds. + +### Generate a batch of example traces + +To populate Arize AX with a diverse set of traces at once β€” covering each tool path (weather lookup, activity planning, and the full analysis-and-planning chain) β€” run: + +```bash +npm run generate-traces +``` + +This sends a curated set of ~50 prompts to the orchestrator agent and flushes the resulting spans to Arize AX. The prompts live in [src/scripts/example-prompts.json](src/scripts/example-prompts.json), grouped by the tool path each one exercises β€” edit that file to add or change prompts. ## How it works A single orchestrator agent coordinates three tools in sequence: -``` +```text User request ↓ Weather Orchestrator Agent ↓ -weatherTool β†’ fetches current weather data +weatherTool β†’ fetches current weather data (Open-Meteo) ↓ -weatherAnalysisTool β†’ analyzes weather patterns +weatherAnalysisTool β†’ delegates to Weather Analysis Agent (LLM) ↓ -activityPlanningTool β†’ recommends activities +activityPlanningTool β†’ delegates to Activity Planning Agent (LLM) ↓ Final response ``` -Tracing is configured in [src/mastra/index.ts](src/mastra/index.ts) using `@arizeai/openinference-mastra`. Every agent invocation and tool call is exported as an OpenInference span to `https://otlp.arize.com/v1/traces`. +The analysis and planning tools don't call the model directly β€” they resolve a +dedicated worker agent from the Mastra instance (`mastra.getAgent(...)`) and call +`agent.generate(...)`. Because those calls run inside Mastra, their LLM spans are +captured as nested children of the tool span in the trace. + +Tracing is configured in [src/mastra/index.ts](src/mastra/index.ts) using `@mastra/arize`, wired into Mastra's AI Tracing (`observability`) system. Every agent invocation and tool call is exported as an OpenInference span to `https://otlp.arize.com/v1/traces`. ## Project structure @@ -56,7 +71,9 @@ Tracing is configured in [src/mastra/index.ts](src/mastra/index.ts) using `@ariz src/mastra/ β”œβ”€β”€ index.ts # Mastra setup + Arize AX exporter β”œβ”€β”€ agents/ -β”‚ └── weather-orchestrator-agent.ts +β”‚ β”œβ”€β”€ weather-orchestrator-agent.ts # coordinates the tools +β”‚ β”œβ”€β”€ weather-analysis-agent.ts # LLM worker, called by weatherAnalysisTool +β”‚ └── activity-planning-agent.ts # LLM worker, called by activityPlanningTool └── tools/ β”œβ”€β”€ weather-tool.ts β”œβ”€β”€ weather-analysis-tool.ts @@ -66,5 +83,5 @@ src/mastra/ ## Resources - [Arize AX Documentation](https://arize.com/docs/ax/) -- [OpenInference for Mastra](https://github.com/Arize-ai/openinference/tree/main/js/packages/openinference-mastra) +- [Mastra Arize exporter (`@mastra/arize`)](https://mastra.ai/en/docs/observability/ai-tracing/exporters/arize) - [Mastra Documentation](https://mastra.ai/docs) diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/package.json b/typescript/cookbooks/mastra-arize-ax-tracing/package.json index ef9f1c5..1860aa9 100644 --- a/typescript/cookbooks/mastra-arize-ax-tracing/package.json +++ b/typescript/cookbooks/mastra-arize-ax-tracing/package.json @@ -6,6 +6,7 @@ "start": "mastra dev", "dev": "mastra dev", "build": "mastra build", + "generate-traces": "tsx src/scripts/generate-traces.ts", "test": "echo \"Error: no test specified\" && exit 1" }, "keywords": [], @@ -14,21 +15,22 @@ "description": "Companion repo for Arize AX tutorial: Mastra orchestrator agent with Arize AX tracing", "type": "module", "engines": { - "node": ">=20.9.0" + "node": ">=22.13.0" }, "dependencies": { - "@ai-sdk/openai": "^1.3.24", - "@arizeai/openinference-mastra": "^2.2.0", - "@mastra/core": "^0.15.3", - "@mastra/libsql": "^0.13.7", - "@mastra/loggers": "^0.10.9", - "@mastra/memory": "^0.14.2", - "zod": "^3.25.76" + "@ai-sdk/openai": "^3.0.74", + "@mastra/arize": "^1.3.1", + "@mastra/core": "^1.46.0", + "@mastra/libsql": "^1.14.1", + "@mastra/loggers": "^1.2.0", + "@mastra/memory": "^1.21.1", + "@mastra/observability": "^1.15.1", + "zod": "^4.4.3" }, "devDependencies": { - "@types/node": "^24.3.0", - "mastra": "^0.11.3", - "tsx": "^4.0.0", - "typescript": "^5.9.2" + "@types/node": "^26.0.0", + "mastra": "^1.15.1", + "tsx": "^4.22.4", + "typescript": "^6.0.3" } } diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/activity-planning-agent.ts b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/activity-planning-agent.ts new file mode 100644 index 0000000..f0b1a38 --- /dev/null +++ b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/activity-planning-agent.ts @@ -0,0 +1,20 @@ +import { openai } from '@ai-sdk/openai'; +import { Agent } from '@mastra/core/agent'; + +// LLM-backed planning step. Invoked by activityPlanningTool via the Mastra +// instance so the call is captured as a traced child span in Arize AX. +export const activityPlanningAgent = new Agent({ + id: 'activityPlanningAgent', + name: 'ActivityPlanningAgent', + instructions: ` + You create detailed, location-specific activity plans from weather data + (and an optional weather analysis). Recommend concrete activities with + specific venues, parks, trails, or locations, sensible timing, and the + weather considerations behind each choice. Always include indoor + alternatives and any special considerations (gear, warnings). + + Follow the exact output format requested in the message. Respond with the + plan only. +`, + model: openai('gpt-5.4-mini'), +}); diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/weather-analysis-agent.ts b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/weather-analysis-agent.ts new file mode 100644 index 0000000..2864ca2 --- /dev/null +++ b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/weather-analysis-agent.ts @@ -0,0 +1,20 @@ +import { openai } from '@ai-sdk/openai'; +import { Agent } from '@mastra/core/agent'; + +// LLM-backed analysis step. Invoked by weatherAnalysisTool via the Mastra +// instance so the call is captured as a traced child span in Arize AX. +export const weatherAnalysisAgent = new Agent({ + id: 'weatherAnalysisAgent', + name: 'WeatherAnalysisAgent', + instructions: ` + You are a meteorological analyst. Given raw weather data for a location, + provide a concise, specific analysis that covers: + + - Notable current conditions and what they mean in practical terms + - Comfort and safety considerations (heat, cold, wind, precipitation, UV) + - How the conditions are likely to affect outdoor plans for the day + + Respond with the analysis only. Do not ask follow-up questions. +`, + model: openai('gpt-5.4-mini'), +}); diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/weather-orchestrator-agent.ts b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/weather-orchestrator-agent.ts index 475e2f7..6f2c6b4 100644 --- a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/weather-orchestrator-agent.ts +++ b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/weather-orchestrator-agent.ts @@ -7,6 +7,7 @@ import { weatherAnalysisTool } from '../tools/weather-analysis-tool'; import { activityPlanningTool } from '../tools/activity-planning-tool'; export const weatherOrchestratorAgent = new Agent({ + id: 'weatherOrchestratorAgent', name: 'WeatherOrchestratorAgent', instructions: ` You are the Weather Orchestrator Agent. You MUST use the available tools to provide weather information and recommendations. @@ -38,7 +39,7 @@ export const weatherOrchestratorAgent = new Agent({ Remember: You are a tool orchestrator, not a direct weather information provider. `, - model: openai('gpt-4o-mini'), + model: openai('gpt-5.4-mini'), tools: { weatherTool, weatherAnalysisTool, @@ -46,6 +47,7 @@ export const weatherOrchestratorAgent = new Agent({ }, memory: new Memory({ storage: new LibSQLStore({ + id: 'agent-memory', url: 'file:../mastra.db', }), }), diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/index.ts b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/index.ts index 5a31dd0..2a9b315 100644 --- a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/index.ts +++ b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/index.ts @@ -1,13 +1,12 @@ import { Mastra } from '@mastra/core/mastra'; import { PinoLogger } from '@mastra/loggers'; import { LibSQLStore } from '@mastra/libsql'; -// Import orchestrator/worker agents - this is the only workflow pattern now +import { Observability } from '@mastra/observability'; +import { ArizeExporter } from '@mastra/arize'; +// Orchestrator agent plus the LLM-backed worker agents its tools delegate to. import { weatherOrchestratorAgent } from './agents/weather-orchestrator-agent'; - -import { - isOpenInferenceSpan, - OpenInferenceOTLPTraceExporter, -} from "@arizeai/openinference-mastra"; +import { weatherAnalysisAgent } from './agents/weather-analysis-agent'; +import { activityPlanningAgent } from './agents/activity-planning-agent'; const ARIZE_SPACE_ID = process.env.ARIZE_SPACE_ID; const ARIZE_API_KEY = process.env.ARIZE_API_KEY; @@ -22,29 +21,36 @@ export const mastra = new Mastra({ // No workflows - using pure orchestrator/worker agent pattern agents: { // Orchestrator agent that coordinates the entire workflow - weatherOrchestratorAgent + weatherOrchestratorAgent, + // Worker agents the orchestrator's tools delegate to (traced child spans) + weatherAnalysisAgent, + activityPlanningAgent }, storage: new LibSQLStore({ - // stores telemetry, evals, ... into memory storage, if it needs to persist, change to file:../mastra.db - url: ":memory:", + // File-backed so Mastra's internal workflow/scheduler tables (e.g. + // mastra_workflow_snapshot) persist and are shared across connections. + // ":memory:" gives each libsql connection its own DB, which breaks them. + id: "mastra-storage", + url: "file:../mastra.db", }), logger: new PinoLogger({ name: 'Mastra', level: 'info', }), - telemetry: { - enabled: true, - serviceName: "mastra-orchestrator-workflow", - export: { - type: "custom", - exporter: new OpenInferenceOTLPTraceExporter({ - url: "https://otlp.arize.com/v1/traces", - headers: { - "arize-space-id": ARIZE_SPACE_ID, - "arize-api-key": ARIZE_API_KEY, - }, - spanFilter: isOpenInferenceSpan, - }), + // Mastra AI Tracing: export every agent and tool span to Arize AX via the + // native @mastra/arize exporter (replaces the deprecated legacy telemetry path). + observability: new Observability({ + configs: { + arize: { + serviceName: "mastra-orchestrator-workflow", + exporters: [ + new ArizeExporter({ + spaceId: ARIZE_SPACE_ID, + apiKey: ARIZE_API_KEY, + projectName: "mastra-orchestrator-workflow", + }), + ], + }, }, - }, + }), }); diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/tools/activity-planning-tool.ts b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/tools/activity-planning-tool.ts index 0ff10d6..5bea653 100644 --- a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/tools/activity-planning-tool.ts +++ b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/tools/activity-planning-tool.ts @@ -1,7 +1,5 @@ import { createTool } from '@mastra/core/tools'; import { z } from 'zod'; -import { generateText } from 'ai'; -import { openai } from '@ai-sdk/openai'; export const activityPlanningTool = createTool({ id: 'activity-planning-tool', @@ -14,12 +12,16 @@ export const activityPlanningTool = createTool({ outputSchema: z.object({ activityPlan: z.string().describe('Detailed activity recommendations and schedule') }), - execute: async ({ context }) => { - const { weatherData, weatherAnalysis, location } = context; + // Delegate to the activityPlanningAgent via the Mastra instance so the LLM + // call is traced as a child span of this tool in Arize AX. + execute: async ({ weatherData, weatherAnalysis, location }, { mastra }) => { + if (!mastra) { + throw new Error('Mastra instance is not available in the tool execution context'); + } - const result = await generateText({ - model: openai('gpt-4o-mini'), - prompt: `Based on the weather data${weatherAnalysis ? ' and analysis' : ''} for ${location}, create a detailed activity plan: + const agent = mastra.getAgent('activityPlanningAgent'); + const result = await agent.generate( + `Based on the weather data${weatherAnalysis ? ' and analysis' : ''} for ${location}, create a detailed activity plan: WEATHER DATA: ${weatherData} @@ -64,7 +66,7 @@ Guidelines followed: - Consider weather conditions for activity selection - Provide backup indoor options - Include timing recommendations` - }); + ); return { activityPlan: result.text diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/tools/weather-analysis-tool.ts b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/tools/weather-analysis-tool.ts index 64fdb38..433592c 100644 --- a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/tools/weather-analysis-tool.ts +++ b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/tools/weather-analysis-tool.ts @@ -1,7 +1,5 @@ import { createTool } from '@mastra/core/tools'; import { z } from 'zod'; -import { generateText } from 'ai'; -import { openai } from '@ai-sdk/openai'; export const weatherAnalysisTool = createTool({ id: 'weather-analysis-tool', @@ -13,12 +11,16 @@ export const weatherAnalysisTool = createTool({ outputSchema: z.object({ analysis: z.string().describe('Detailed weather analysis and forecast interpretation') }), - execute: async ({ context }) => { - const { weatherData, location } = context; + // Delegate to the weatherAnalysisAgent via the Mastra instance so the LLM + // call is traced as a child span of this tool in Arize AX. + execute: async ({ weatherData, location }, { mastra }) => { + if (!mastra) { + throw new Error('Mastra instance is not available in the tool execution context'); + } - const result = await generateText({ - model: openai('gpt-4o-mini'), - prompt: `Analyze the following weather data for ${location} and provide detailed insights: + const agent = mastra.getAgent('weatherAnalysisAgent'); + const result = await agent.generate( + `Analyze the following weather data for ${location} and provide detailed insights: ${weatherData} @@ -28,7 +30,7 @@ Please provide: 3. Optimal timing windows throughout the day 4. Any weather risks or special considerations 5. Overall assessment of weather suitability for outdoor vs indoor activities` - }); + ); return { analysis: result.text diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/tools/weather-tool.ts b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/tools/weather-tool.ts index b58e256..e40fb02 100644 --- a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/tools/weather-tool.ts +++ b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/tools/weather-tool.ts @@ -35,8 +35,8 @@ export const weatherTool = createTool({ conditions: z.string(), location: z.string(), }), - execute: async ({ context }) => { - return await getWeather(context.location); + execute: async ({ location }) => { + return await getWeather(location); }, }); diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/src/scripts/example-prompts.json b/typescript/cookbooks/mastra-arize-ax-tracing/src/scripts/example-prompts.json new file mode 100644 index 0000000..550b163 --- /dev/null +++ b/typescript/cookbooks/mastra-arize-ax-tracing/src/scripts/example-prompts.json @@ -0,0 +1,64 @@ +{ + "weather-only": [ + "What's the weather in Tokyo right now?", + "Is it raining in Seattle at the moment?", + "How hot does it feel in Dubai today?", + "Give me the current conditions in Reykjavik.", + "What's the temperature in Nairobi right now?", + "Is it windy in Wellington today?", + "How humid is it in Bangkok at the moment?", + "What are the current conditions in Buenos Aires?", + "Is it cold in Helsinki today?", + "Tell me the current weather in Vancouver.", + "How's the weather in Mexico City right now?", + "What's it like outside in Cairo?" + ], + "weather+planning": [ + "What should I do in Barcelona today?", + "Give me some activity ideas for a day out in Amsterdam.", + "I'm spending the day in Cape Town β€” what's good to do given the weather?", + "Recommend things to do in Lisbon today.", + "What's a good way to spend the afternoon in Berlin?", + "Plan a fun day for me in Sydney.", + "I have a free day in Kyoto β€” what should I do?", + "Suggest some outdoor activities in Vienna today.", + "What can I do with the kids in Copenhagen today?", + "I'm visiting Edinburgh β€” recommend a day's itinerary.", + "What's good to do in Marrakech given today's weather?", + "Help me plan a relaxed day in Porto." + ], + "weather+analysis+planning": [ + "Give me a detailed weather analysis and a full activity plan for London today.", + "Analyze the conditions in Reykjavik and plan my whole day around them.", + "I want an in-depth look at the weather in Singapore plus a complete itinerary for today.", + "Break down today's weather in Toronto and build me a full day plan.", + "Give me a thorough weather assessment for Rome and a detailed schedule.", + "Analyze the forecast for Oslo and create a complete day itinerary.", + "I need a deep dive on Chicago's weather and a full plan around it.", + "Assess the conditions in Auckland and map out my entire day.", + "Detailed weather breakdown and full-day plan for Seoul, please.", + "Examine today's weather in Athens and plan everything for me.", + "Give me a complete analysis and itinerary for a day in Prague.", + "Walk me through the weather in Cape Town and a full activity schedule." + ], + "multi-location": [ + "Is it better weather for a long walk in Oslo or in Lisbon right now, and what should I do there?", + "Compare the weather in Madrid and Rome today and tell me which is better for sightseeing.", + "Which has nicer weather right now β€” San Francisco or Los Angeles β€” and what should I do?", + "I'm choosing between Berlin and Munich for a day trip; which has better weather and what should I do?", + "Should I go to the beach in Nice or Barcelona today based on the weather?", + "Compare conditions in New York and Boston and suggest activities for the better one." + ], + "vague-intent": [ + "I might head outside in Chicago later... or maybe not. What do you reckon?", + "Not sure what to do today in Dublin, help me out?", + "Thinking about going out in Melbourne, but I'm undecided β€” thoughts?", + "I was wondering if you could help me figure out my day in Glasgow." + ], + "activity-constrained": [ + "Plan an indoor-friendly day in Mumbai in case the weather is rough.", + "I want only outdoor activities in Lisbon today β€” is that doable given the weather?", + "Suggest a rainy-day plan for Manchester.", + "Plan a day of mostly walking and parks in Munich if the weather allows." + ] +} diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/src/scripts/generate-traces.ts b/typescript/cookbooks/mastra-arize-ax-tracing/src/scripts/generate-traces.ts new file mode 100644 index 0000000..ec0f6a2 --- /dev/null +++ b/typescript/cookbooks/mastra-arize-ax-tracing/src/scripts/generate-traces.ts @@ -0,0 +1,69 @@ +/** + * Sends a diverse set of prompts to the WeatherOrchestratorAgent to generate + * traces in Arize AX. The prompts live in ./example-prompts.json, grouped by + * the tool path each one is meant to exercise: + * + * - weather-only β†’ weatherTool + * - weather+planning β†’ weatherTool β†’ activityPlanningTool + * - weather+analysis+plan β†’ weatherTool β†’ weatherAnalysisTool β†’ activityPlanningTool + * - multi-location / vague-intent / activity-constrained β†’ varied phrasing + * + * Run with: npm run generate-traces + * Requires the same env vars as the agent: ARIZE_SPACE_ID, ARIZE_API_KEY, OPENAI_API_KEY. + */ +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; + +// The Mastra config points its storage at `file:../mastra.db`, resolved relative +// to the process working directory. Run from `src/` so that local db lands in +// the cookbook root (where *.db is gitignored) instead of the parent directory. +process.chdir(fileURLToPath(new URL('..', import.meta.url))); + +const { mastra } = await import('../mastra/index'); + +interface Example { + category: string; + prompt: string; +} + +// Prompts grouped by category in the JSON file; flattened to a run list here. +const promptsByCategory = JSON.parse( + readFileSync(new URL('./example-prompts.json', import.meta.url), 'utf8'), +) as Record; + +const examples: Example[] = Object.entries(promptsByCategory).flatMap( + ([category, prompts]) => prompts.map((prompt) => ({ category, prompt })), +); + +async function main(): Promise { + const agent = mastra.getAgent('weatherOrchestratorAgent'); + + console.log(`Sending ${examples.length} prompts to WeatherOrchestratorAgent…\n`); + + let ok = 0; + for (const [index, example] of examples.entries()) { + const label = `[${index + 1}/${examples.length}] (${example.category})`; + console.log(`${label} β†’ ${example.prompt}`); + try { + const result = await agent.generate(example.prompt); + const text = (result.text ?? '').replace(/\s+/g, ' ').trim(); + console.log(` βœ“ ${text.slice(0, 140)}${text.length > 140 ? '…' : ''}\n`); + ok += 1; + } catch (error) { + console.error(` βœ— failed: ${(error as Error).message}\n`); + } + } + + console.log(`Completed ${ok}/${examples.length} prompts. Flushing traces to Arize AX…`); + // Cleanly shut down so the tracing exporter flushes all buffered spans before exit. + await mastra.shutdown(); + console.log('Done. The traces should appear in your Arize AX space within a few seconds.'); +} + +try { + await main(); + process.exit(0); +} catch (error) { + console.error(error); + process.exit(1); +}