Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 41 additions & 7 deletions lib/trajectory.ex
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,11 @@ defmodule LangChain.Trajectory do

trajectory = Trajectory.from_chain(chain)

# Or build from a bare message list when no live chain is available
# (e.g. a persisted agent state). The llm is optional and only feeds
# metadata.
trajectory = Trajectory.from_messages(messages, llm)

# Serialize for logging or golden-file comparison
map = Trajectory.to_map(trajectory)

Expand Down Expand Up @@ -170,16 +175,45 @@ defmodule LangChain.Trajectory do
trajectory = Trajectory.from_chain(chain)
"""
@spec from_chain(LLMChain.t()) :: t()
def from_chain(%LLMChain{exchanged_messages: messages, llm: llm}) do
tool_calls = extract_tool_calls(messages)
token_usage = aggregate_token_usage(messages)
metadata = extract_metadata(llm)
def from_chain(%LLMChain{exchanged_messages: exchanged_messages, llm: llm}) do
from_messages(exchanged_messages, llm)
end

@doc """
Build a `Trajectory` from a list of exchanged messages.

Use this when you have the messages from an assistant operation, workflow,
or set of tool-call iterations but do **not** have a live `LLMChain` — for
example, a persisted agent state or a sliced "last turn" of a conversation.

The `messages` argument should be the **exchanged messages for the operation
you want to analyze**, not necessarily an entire conversation. Including the
leading system/user preamble is harmless for tool-call extraction and
matching (those messages carry no tool calls), but for token aggregation and
`calls_by_turn/1` semantics you generally want just the turns produced during
the operation under analysis.

`llm` is optional and is only used to populate `metadata` (`:model` and
`:llm_module`). Pass `nil` (the default) to produce an empty metadata map.
Note that metadata does not survive a JSON roundtrip anyway (see the
`from_map/1` note), so it is safe to omit when you only have a stored
message list.

## Example

# From a persisted agent state, analyzing the whole conversation:
trajectory = Trajectory.from_messages(state.messages, agent.model)

# From a pre-sliced "last turn" with no llm handy:
trajectory = Trajectory.from_messages(last_turn_messages)
"""
@spec from_messages([Message.t()], struct() | nil) :: t()
def from_messages(messages, llm \\ nil) when is_list(messages) do
%Trajectory{
messages: messages,
tool_calls: tool_calls,
token_usage: token_usage,
metadata: metadata
tool_calls: extract_tool_calls(messages),
token_usage: aggregate_token_usage(messages),
metadata: extract_metadata(llm)
}
end

Expand Down
89 changes: 89 additions & 0 deletions test/trajectory_test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,95 @@ defmodule LangChain.TrajectoryTest do
end
end

describe "from_messages/2" do
test "builds a trajectory from a bare message list without an llm" do
tc = make_tool_call("search", %{"query" => "weather"})

messages = [
user_msg("What's the weather?"),
assistant_msg(nil, tool_calls: [tc])
]

trajectory = Trajectory.from_messages(messages)

assert trajectory.messages == messages
assert [%{name: "search", arguments: %{"query" => "weather"}}] = trajectory.tool_calls
# No llm provided, so metadata is empty
assert trajectory.metadata == %{}
end

test "populates metadata when an llm is provided" do
{:ok, llm} = ChatOpenAI.new(%{temperature: 0})

messages = [user_msg("Hello"), assistant_msg("Hi")]
trajectory = Trajectory.from_messages(messages, llm)

assert trajectory.metadata.model == "gpt-3.5-turbo"
assert trajectory.metadata.llm_module == ChatOpenAI
end

test "aggregates token usage over a hand-built message list" do
usage1 = make_usage(10, 20)
usage2 = make_usage(5, 15)

messages = [
user_msg("Hello"),
assistant_msg("Hi", usage: usage1),
user_msg("More"),
assistant_msg("Sure", usage: usage2)
]

trajectory = Trajectory.from_messages(messages)

assert trajectory.token_usage.input == 15
assert trajectory.token_usage.output == 35
end

test "is equivalent to from_chain/1 on the chain's exchanged_messages" do
tc1 = make_tool_call("search", %{"query" => "weather"})
tc2 = make_tool_call("get_forecast", %{"city" => "Paris"})
tr1 = make_tool_result("search", "Sunny")

messages = [
user_msg("What's the weather in Paris?"),
assistant_msg(nil, tool_calls: [tc1], usage: make_usage(10, 20)),
tool_msg([tr1]),
assistant_msg(nil, tool_calls: [tc2], usage: make_usage(5, 15))
]

chain = chain_with_messages(messages)

from_chain = Trajectory.from_chain(chain)
from_messages = Trajectory.from_messages(chain.exchanged_messages, chain.llm)

assert from_chain == from_messages
end

test "a bare-list-without-llm trajectory still matches?/3 correctly" do
tc1 = make_tool_call("search", %{"query" => "weather"})
tc2 = make_tool_call("get_forecast", %{"city" => "Paris"})

messages = [
user_msg("What's the weather in Paris?"),
assistant_msg(nil, tool_calls: [tc1]),
assistant_msg(nil, tool_calls: [tc2])
]

trajectory = Trajectory.from_messages(messages)

assert Trajectory.matches?(trajectory, [
%{name: "search", arguments: %{"query" => "weather"}},
%{name: "get_forecast", arguments: %{"city" => "Paris"}}
])

assert Trajectory.matches?(trajectory, [%{name: "search", arguments: nil}], mode: :superset)

refute Trajectory.matches?(trajectory, [%{name: "delete_all", arguments: nil}],
mode: :superset
)
end
end

describe "to_map/1" do
test "serializes empty trajectory" do
trajectory = %Trajectory{messages: [], tool_calls: [], token_usage: nil}
Expand Down