diff --git a/examples/README.md b/examples/README.md index c23aff1..a713efd 100644 --- a/examples/README.md +++ b/examples/README.md @@ -30,6 +30,7 @@ agentevals accepts OTLP/HTTP on port 4318 (`http/protobuf` and `http/json`) and | [zero-code-examples/strands/](./zero-code-examples/strands/) | Strands | OpenAI | | [zero-code-examples/adk/](./zero-code-examples/adk/) | Google ADK | Gemini | | [zero-code-examples/pydantic-ai/](./zero-code-examples/pydantic-ai/) | Pydantic AI | OpenAI | +| [zero-code-examples/llama-index/](./zero-code-examples/llama-index/) | LlamaIndex | OpenAI | This approach works with any framework that has OTel instrumentation: LangChain, Strands, Google ADK, etc. If your framework already emits OTel spans, you only need to add `OTLPSpanExporter` (and `OTLPLogExporter` if it uses GenAI log-based content delivery). @@ -105,6 +106,7 @@ Detection checks for `gen_ai.request.model` / `gen_ai.input.messages` (GenAI sem | [zero-code-examples/strands/](./zero-code-examples/strands/) | Strands | OpenAI | GenAI semconv (events*) | Standard OTLP export | | [zero-code-examples/adk/](./zero-code-examples/adk/) | Google ADK | Gemini | ADK built-in | Standard OTLP export | | [zero-code-examples/pydantic-ai/](./zero-code-examples/pydantic-ai/) | Pydantic AI | OpenAI | GenAI semconv (span attrs) | Standard OTLP export | +| [zero-code-examples/llama-index/](./zero-code-examples/llama-index/) | LlamaIndex | OpenAI | GenAI semconv (logs) | Standard OTLP export | | [langchain_agent](./langchain_agent/) | LangChain | OpenAI | GenAI semconv (logs) | SDK WebSocket | | [strands_agent](./strands_agent/) | Strands | OpenAI | GenAI semconv (events*) | SDK WebSocket | | [dice_agent](./dice_agent/) | Google ADK | Gemini | ADK built-in | SDK WebSocket | @@ -226,6 +228,7 @@ python examples/zero-code-examples/ollama/run.py python examples/zero-code-examples/strands/run.py python examples/zero-code-examples/adk/run.py python examples/zero-code-examples/pydantic-ai/run.py +python examples/zero-code-examples/llama-index/run.py # SDK examples: python examples/sdk_example/context_manager_example.py diff --git a/examples/zero-code-examples/llama-index/requirements.txt b/examples/zero-code-examples/llama-index/requirements.txt new file mode 100644 index 0000000..b43f9e9 --- /dev/null +++ b/examples/zero-code-examples/llama-index/requirements.txt @@ -0,0 +1,7 @@ +llama-index-core>=0.14.0 +llama-index-llms-openai-like>=0.3.0 +llama-index-observability-otel>=0.1.0 + +opentelemetry-sdk>=1.36.0 +opentelemetry-exporter-otlp-proto-http>=1.36.0 +python-dotenv>=1.0.0 diff --git a/examples/zero-code-examples/llama-index/run.py b/examples/zero-code-examples/llama-index/run.py new file mode 100644 index 0000000..369a04b --- /dev/null +++ b/examples/zero-code-examples/llama-index/run.py @@ -0,0 +1,98 @@ +"""Run a LlamaIndex dice agent with standard OTLP export. + +Uses the official LlamaIndexOpenTelemetry integration to set up OTel tracing. +It handles the tracer provider setup and span export internally. +Traces stream to agentevals via OTLPSpanExporter with no agentevals SDK needed. + +Note: LlamaIndexOpenTelemetry exports spans only. Log-based content delivery +is not part of this integration. Message content is captured via span attributes. + +Prerequisites: + 1. pip install -r examples/zero-code-examples/llama-index/requirements.txt + 2. agentevals serve --dev + 3. export OPENAI_API_KEY="your-key-here" + +Usage: + python examples/zero-code-examples/llama-index/run.py +""" + +import asyncio +import os +import random + +from dotenv import load_dotenv +from llama_index.core.agent.workflow import FunctionAgent +from llama_index.core.tools import FunctionTool +from llama_index.llms.openai_like import OpenAILike +from llama_index.observability.otel import LlamaIndexOpenTelemetry +from opentelemetry import trace +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.resources import Resource + +load_dotenv(override=True) + + +def roll_die(sides: int) -> int: + """Roll a die with the given number of sides and return the result.""" + return random.randint(1, sides) + + +def check_prime(number: int) -> bool: + """Return True if the number is prime, False otherwise.""" + if number < 2: + return False + return all(number % i for i in range(2, int(number**0.5) + 1)) + + +async def main(): + if not os.getenv("OPENAI_API_KEY"): + print("OPENAI_API_KEY not set.") + return + + endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4318") + print(f"OTLP endpoint: {endpoint}") + + os.environ.setdefault( + "OTEL_RESOURCE_ATTRIBUTES", + "agentevals.eval_set_id=llama_index_eval,agentevals.session_name=llama-index-zero-code", + ) + + resource = Resource.create() + + LlamaIndexOpenTelemetry( + span_exporter=OTLPSpanExporter(), + span_processor="batch", + service_name_or_resource=resource, + ).start_registering() + + llm = OpenAILike( + model=os.environ.get("OPENAI_MODEL", "gpt-4o-mini"), + api_base=os.environ.get("OPENAI_BASE_URL", "https://api.openai.com/v1"), + is_chat_model=True, + is_function_calling_model=True, + ) + agent = FunctionAgent( + tools=[FunctionTool.from_defaults(fn=roll_die), FunctionTool.from_defaults(fn=check_prime)], + llm=llm, + system_prompt="You are a helpful assistant. You can roll dice and check if numbers are prime.", + ) + + test_queries = [ + "Hi! Can you help me?", + "Roll a 20-sided die for me", + "Is the number you rolled prime?", + ] + + try: + for i, query in enumerate(test_queries, 1): + print(f"\n[{i}/{len(test_queries)}] User: {query}") + result = await agent.run(query) + print(f" Agent: {result.response.content}") + finally: + print() + trace.get_tracer_provider().force_flush() + print("All traces flushed to OTLP receiver.") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/integration/test_live_agents.py b/tests/integration/test_live_agents.py index fff9a96..dd4fa5b 100644 --- a/tests/integration/test_live_agents.py +++ b/tests/integration/test_live_agents.py @@ -365,6 +365,72 @@ def test_session_visible_via_api(self, live_servers): assert session_name in session_ids +@_skip_no_openai +class TestLlamaIndexZeroCode: + """Run the LlamaIndex zero-code OTLP example and verify session grouping.""" + + def test_session_created_spans_only(self, live_servers): + main_port, otlp_http_port, mgr = live_servers + session_name = "e2e-llama-index" + + result = _run_agent( + "examples/zero-code-examples/llama-index/run.py", + otlp_http_port, + session_name, + extra_env={ + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": "true", + }, + ) + assert result.returncode == 0, f"Agent failed:\nstdout: {result.stdout}\nstderr: {result.stderr}" + + wait_for_session_complete_sync(mgr, session_name, timeout=30) + session = mgr.sessions[session_name] + + assert session.is_complete + assert session.source == "otlp" + assert len(session.spans) > 0, "Expected spans from LlamaIndex agent" + + def test_invocations_extracted(self, live_servers): + main_port, otlp_http_port, mgr = live_servers + session_name = "e2e-llama-index-inv" + + result = _run_agent( + "examples/zero-code-examples/llama-index/run.py", + otlp_http_port, + session_name, + extra_env={ + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": "true", + }, + ) + assert result.returncode == 0, f"Agent failed:\nstdout: {result.stdout}\nstderr: {result.stderr}" + + wait_for_session_complete_sync(mgr, session_name, timeout=30) + session = mgr.sessions[session_name] + + assert len(session.invocations) > 0, "Expected extracted invocations" + + def test_session_visible_via_api(self, live_servers): + main_port, otlp_http_port, mgr = live_servers + session_name = "e2e-llama-index-api" + + result = _run_agent( + "examples/zero-code-examples/llama-index/run.py", + otlp_http_port, + session_name, + extra_env={ + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": "true", + }, + ) + assert result.returncode == 0 + + wait_for_session_complete_sync(mgr, session_name, timeout=30) + + resp = httpx.get(f"http://127.0.0.1:{main_port}/api/streaming/sessions") + assert resp.status_code == 200 + session_ids = [s["sessionId"] for s in resp.json()["data"]] + assert session_name in session_ids + + @_skip_no_openai class TestAgentRerun: """Verify that re-running an agent with the same session_name creates