diff --git a/lib/trajectory.ex b/lib/trajectory.ex index b3e93837..680e1556 100644 --- a/lib/trajectory.ex +++ b/lib/trajectory.ex @@ -28,6 +28,11 @@ defmodule LangChain.Trajectory do trajectory = Trajectory.from_chain(chain) + # Or build from a bare message list when no live chain is available + # (e.g. a persisted agent state). The llm is optional and only feeds + # metadata. + trajectory = Trajectory.from_messages(messages, llm) + # Serialize for logging or golden-file comparison map = Trajectory.to_map(trajectory) @@ -170,16 +175,45 @@ defmodule LangChain.Trajectory do trajectory = Trajectory.from_chain(chain) """ @spec from_chain(LLMChain.t()) :: t() - def from_chain(%LLMChain{exchanged_messages: messages, llm: llm}) do - tool_calls = extract_tool_calls(messages) - token_usage = aggregate_token_usage(messages) - metadata = extract_metadata(llm) + def from_chain(%LLMChain{exchanged_messages: exchanged_messages, llm: llm}) do + from_messages(exchanged_messages, llm) + end + + @doc """ + Build a `Trajectory` from a list of exchanged messages. + + Use this when you have the messages from an assistant operation, workflow, + or set of tool-call iterations but do **not** have a live `LLMChain` — for + example, a persisted agent state or a sliced "last turn" of a conversation. + + The `messages` argument should be the **exchanged messages for the operation + you want to analyze**, not necessarily an entire conversation. Including the + leading system/user preamble is harmless for tool-call extraction and + matching (those messages carry no tool calls), but for token aggregation and + `calls_by_turn/1` semantics you generally want just the turns produced during + the operation under analysis. + `llm` is optional and is only used to populate `metadata` (`:model` and + `:llm_module`). Pass `nil` (the default) to produce an empty metadata map. + Note that metadata does not survive a JSON roundtrip anyway (see the + `from_map/1` note), so it is safe to omit when you only have a stored + message list. + + ## Example + + # From a persisted agent state, analyzing the whole conversation: + trajectory = Trajectory.from_messages(state.messages, agent.model) + + # From a pre-sliced "last turn" with no llm handy: + trajectory = Trajectory.from_messages(last_turn_messages) + """ + @spec from_messages([Message.t()], struct() | nil) :: t() + def from_messages(messages, llm \\ nil) when is_list(messages) do %Trajectory{ messages: messages, - tool_calls: tool_calls, - token_usage: token_usage, - metadata: metadata + tool_calls: extract_tool_calls(messages), + token_usage: aggregate_token_usage(messages), + metadata: extract_metadata(llm) } end diff --git a/test/trajectory_test.exs b/test/trajectory_test.exs index ad7d23e6..1620ac09 100644 --- a/test/trajectory_test.exs +++ b/test/trajectory_test.exs @@ -173,6 +173,95 @@ defmodule LangChain.TrajectoryTest do end end + describe "from_messages/2" do + test "builds a trajectory from a bare message list without an llm" do + tc = make_tool_call("search", %{"query" => "weather"}) + + messages = [ + user_msg("What's the weather?"), + assistant_msg(nil, tool_calls: [tc]) + ] + + trajectory = Trajectory.from_messages(messages) + + assert trajectory.messages == messages + assert [%{name: "search", arguments: %{"query" => "weather"}}] = trajectory.tool_calls + # No llm provided, so metadata is empty + assert trajectory.metadata == %{} + end + + test "populates metadata when an llm is provided" do + {:ok, llm} = ChatOpenAI.new(%{temperature: 0}) + + messages = [user_msg("Hello"), assistant_msg("Hi")] + trajectory = Trajectory.from_messages(messages, llm) + + assert trajectory.metadata.model == "gpt-3.5-turbo" + assert trajectory.metadata.llm_module == ChatOpenAI + end + + test "aggregates token usage over a hand-built message list" do + usage1 = make_usage(10, 20) + usage2 = make_usage(5, 15) + + messages = [ + user_msg("Hello"), + assistant_msg("Hi", usage: usage1), + user_msg("More"), + assistant_msg("Sure", usage: usage2) + ] + + trajectory = Trajectory.from_messages(messages) + + assert trajectory.token_usage.input == 15 + assert trajectory.token_usage.output == 35 + end + + test "is equivalent to from_chain/1 on the chain's exchanged_messages" do + tc1 = make_tool_call("search", %{"query" => "weather"}) + tc2 = make_tool_call("get_forecast", %{"city" => "Paris"}) + tr1 = make_tool_result("search", "Sunny") + + messages = [ + user_msg("What's the weather in Paris?"), + assistant_msg(nil, tool_calls: [tc1], usage: make_usage(10, 20)), + tool_msg([tr1]), + assistant_msg(nil, tool_calls: [tc2], usage: make_usage(5, 15)) + ] + + chain = chain_with_messages(messages) + + from_chain = Trajectory.from_chain(chain) + from_messages = Trajectory.from_messages(chain.exchanged_messages, chain.llm) + + assert from_chain == from_messages + end + + test "a bare-list-without-llm trajectory still matches?/3 correctly" do + tc1 = make_tool_call("search", %{"query" => "weather"}) + tc2 = make_tool_call("get_forecast", %{"city" => "Paris"}) + + messages = [ + user_msg("What's the weather in Paris?"), + assistant_msg(nil, tool_calls: [tc1]), + assistant_msg(nil, tool_calls: [tc2]) + ] + + trajectory = Trajectory.from_messages(messages) + + assert Trajectory.matches?(trajectory, [ + %{name: "search", arguments: %{"query" => "weather"}}, + %{name: "get_forecast", arguments: %{"city" => "Paris"}} + ]) + + assert Trajectory.matches?(trajectory, [%{name: "search", arguments: nil}], mode: :superset) + + refute Trajectory.matches?(trajectory, [%{name: "delete_all", arguments: nil}], + mode: :superset + ) + end + end + describe "to_map/1" do test "serializes empty trajectory" do trajectory = %Trajectory{messages: [], tool_calls: [], token_usage: nil}