feat(benchmark): add the Toolathlon adapter

.agents/TASKS.md

@@ -30,6 +30,42 @@
   official Git LFS installation still leaves the pinned Parquet as an LFS
   pointer, so the dataset integration remains an explicit environment gap.
 - Final Docker inspection found no ARIES containers or networks.
+## R23 — Toolathlon benchmark adapter
+
+1. [x] Pin the Toolathlon checkout, install it atomically, and materialize the
+   two gitignored site configs its scripts import unconditionally.
+2. [x] Map task directories to benchmark-neutral tasks, and refuse at load every
+   task whose MCP servers need a third-party account or the k8s host runtime.
+3. [x] Before bridge access, install the pinned project tree, start a loopback
+   forwarder for Toolathlon's fixed application ports, run Toolathlon's own
+   preprocess, validate the task bundle it writes, stash the grader and ground
+   truth to the private run directory and prove them absent, then start the
+   MCP gateway and wait for its health endpoint.
+4. [x] Render Hermes's `mcp_servers` block from a new `harness.mcp.servers`
+   profile block, so the harness reaches the gateway at the sandbox's fixed
+   network alias.
+5. [x] After harness stop and bridge revocation, discard agent-planted grader
+   paths, restore the stash, re-inject the trusted bundle, run Toolathlon's
+   evaluator, and score from its verdict file.
+6. [x] Add strict configuration/version decoding, explicit command wiring, a
+   one-task profile, package regressions with a scripted-sandbox flow test,
+   and public documentation.
+
+Completion evidence:
+
+- `make lint`, `make build`, and `go test ./...` pass; `aries setup` installs
+  the pinned checkout and verifies it clean.
+- `canvas-list-test` ran end to end against DeepSeek: preprocess seeded Canvas
+  through the forwarder, Hermes made 55 tool calls (30 through the gateway, 25
+  through the SSH bridge), both isolation gates were confirmed, the grader ran,
+  and cleanup left no container or network behind. The verdict was a fail by
+  one row — the grader's judgement of the agent, not a pipeline fault.
+- Running the first profile exposed a stale DeepSeek model list in preflight
+  (the API now serves `deepseek-flash`); fixed in a separate commit.
+- `make test-race` passes. `make integration` passes once the Terminal-Bench 2
+  checkout the OpenClaw integration test expects is installed (`aries setup`
+  with a TB2 profile); it is an environment prerequisite, not a code change.
+  Final Docker inspection found no ARIES containers or networks.
 
 ## R22 — Public SWE-bench Pro benchmark adapter
 

.github/configs/wordlist.txt

@@ -145,3 +145,16 @@ workdirs
 worktree
 xfs
 yaml
+toolathlon
+toolathlon's
+woocommerce
+poste
+hkust
+nlp
+forwarder
+preprocess
+mcp
+sse
+arxiv
+youtube
+wandb

cmd/aries/wiring.go

@@ -13,6 +13,7 @@ import (
 	"github.com/hyscale-lab/aries/pkg/benchmark/sweatlas"
 	"github.com/hyscale-lab/aries/pkg/benchmark/swebenchpro"
 	"github.com/hyscale-lab/aries/pkg/benchmark/terminalbench"
+	"github.com/hyscale-lab/aries/pkg/benchmark/toolathlon"
 	"github.com/hyscale-lab/aries/pkg/bridge/hermesssh"
 	"github.com/hyscale-lab/aries/pkg/bridge/openclawssh"
 	"github.com/hyscale-lab/aries/pkg/config"
@@ -57,6 +58,19 @@ func validateComponents(cfg config.Config) error {
 	case "deepresearchbench":
 	case "sweatlasqa":
 	case "swebenchpro":
+	case "toolathlon":
+		// Toolathlon's tools reach the harness only as an MCP server; both
+		// harnesses have an MCP client. The gateway itself is added to that
+		// client by mcpServers, so a profile's own harness.mcp entries are
+		// extra servers and may not take its name.
+		if cfg.Harness.Type != "hermes" && cfg.Harness.Type != "openclaw" {
+			return fmt.Errorf("benchmark type \"toolathlon\" requires a harness with an MCP client (hermes or openclaw), not %q", cfg.Harness.Type)
+		}
+		for _, server := range cfg.Harness.MCP.Servers {
+			if server.Name == toolathlon.GatewayServerName {
+				return fmt.Errorf("harness.mcp.servers may not name %q: the adapter adds Toolathlon's gateway to the harness itself", toolathlon.GatewayServerName)
+			}
+		}
 	default:
 		return fmt.Errorf("unsupported benchmark type %q", cfg.Benchmark.Type)
 	}
@@ -85,6 +99,15 @@ func validateComponents(cfg config.Config) error {
 	return nil
 }
 
+// toolathlonGatewayPort is the port the adapter will start the gateway on
+// for this profile.
+func toolathlonGatewayPort(cfg config.Config) int {
+	if settings := cfg.Benchmark.Toolathlon; settings != nil && settings.GatewayPort != 0 {
+		return settings.GatewayPort
+	}
+	return toolathlon.DefaultGatewayPort
+}
+
 // prepareBackend turns the profile's runtime block into the model the harness
 // receives and, for managed SGLang only, the host process ARIES owns.
 // runtime.mode is the ownership distinction: every external endpoint is
@@ -193,11 +216,41 @@ func newBenchmark(cfg config.Config, outputRoot, logicalID, occurrenceID string,
 			return nil, fmt.Errorf("construct sweatlasqa benchmark: %w", err)
 		}
 		return benchmark, nil
+	case "toolathlon":
+		var executionIDs []string
+		if occurrenceID != logicalID {
+			executionIDs = []string{occurrenceID}
+		}
+		benchmark, err := toolathlon.New(toolathlonOptions(cfg, []string{logicalID}, executionIDs, outputRoot))
+		if err != nil {
+			return nil, fmt.Errorf("construct toolathlon benchmark: %w", err)
+		}
+		return benchmark, nil
 	default:
 		return nil, fmt.Errorf("unsupported benchmark type %q", cfg.Benchmark.Type)
 	}
 }
 
+// toolathlonOptions maps the profile onto the adapter. The model ID is
+// bookkeeping for Toolathlon's task bundle; the harness owns the model.
+func toolathlonOptions(cfg config.Config, taskIDs, executionIDs []string, outputDir string) toolathlon.Options {
+	options := toolathlon.Options{
+		Root: cfg.Benchmark.Root, TaskIDs: taskIDs, ExecutionTaskIDs: executionIDs, OutputDir: outputDir,
+		Revision:         cfg.Versions.Toolathlon.Revision,
+		Environment:      environmentFromConfig(cfg.Benchmark.Environment),
+		ModelName:        cfg.Model.ID,
+		HarnessWebSearch: cfg.Harness.WebSearch.Enabled,
+		Concurrency:      cfg.Execution.Concurrency,
+	}
+	if settings := cfg.Benchmark.Toolathlon; settings != nil {
+		options.GatewayPort = settings.GatewayPort
+		options.AppHost = settings.AppHost
+		options.MaxSteps = settings.MaxSteps
+		options.CredentialsDir = settings.CredentialsDir
+	}
+	return options
+}
+
 // deepresearchbenchModels resolves the RACE judge and FACT judge model
 // configs for a deepresearchbench profile. A nil benchmark.judge, or a
 // benchmark.fact whose model fields are all empty, default to the profile's
@@ -234,6 +287,37 @@ func sweatlasModels(cfg config.Config) (judge core.ModelConfig, judgeDisabled bo
 	return judgeCfg.CoreModel(), false
 }
 
+// mcpServers is the harness's MCP client configuration, whichever harness
+// renders it: the benchmark's own server first, when the benchmark exposes
+// its tools that way (Toolathlon's gateway, at the sandbox's alias on the
+// gateway port), then the profile's harness.mcp entries.
+func mcpServers(cfg config.Config) []config.HarnessMCPServerConfig {
+	out := make([]config.HarnessMCPServerConfig, 0, len(cfg.Harness.MCP.Servers)+1)
+	if cfg.Benchmark.Type == "toolathlon" {
+		gateway := toolathlon.Gateway(dockersandbox.NetworkAlias, toolathlonGatewayPort(cfg))
+		out = append(out, config.HarnessMCPServerConfig{Name: gateway.Name, URL: gateway.URL, Transport: gateway.Transport, TimeoutSeconds: gateway.TimeoutSeconds})
+	}
+	return append(out, cfg.Harness.MCP.Servers...)
+}
+
+func hermesMCPServers(cfg config.Config) []hermesharness.MCPServer {
+	servers := mcpServers(cfg)
+	out := make([]hermesharness.MCPServer, 0, len(servers))
+	for _, server := range servers {
+		out = append(out, hermesharness.MCPServer{Name: server.Name, URL: server.URL, Transport: server.Transport, TimeoutSeconds: server.TimeoutSeconds})
+	}
+	return out
+}
+
+func openclawMCPServers(cfg config.Config) []openclawharness.MCPServer {
+	servers := mcpServers(cfg)
+	out := make([]openclawharness.MCPServer, 0, len(servers))
+	for _, server := range servers {
+		out = append(out, openclawharness.MCPServer{Name: server.Name, URL: server.URL, Transport: server.Transport, TimeoutSeconds: server.TimeoutSeconds})
+	}
+	return out
+}
+
 // environmentFromConfig converts a profile's benchmark.environment block into
 // the runner-neutral core.Environment. cfg is nil only when Config.validate
 // hasn't run (e.g. ad-hoc construction); callers of newBenchmark and
@@ -261,6 +345,7 @@ func newHarness(cfg config.Config, outputRoot string, lookup func(string) ([]byt
 			ExtractAPIKeyEnv:       cfg.Harness.WebSearch.ExtractAPIKeyEnv,
 			SubagentsEnabled:       cfg.Harness.Subagents.Enabled != nil && *cfg.Harness.Subagents.Enabled,
 			MaxConcurrentSubagents: cfg.Harness.Subagents.MaxConcurrent,
+			MCPServers:             openclawMCPServers(cfg),
 		}
 		if cfg.Harness.Mode == openclawharness.ModeRealtime || cfg.Harness.Mode == openclawharness.ModeVoiceTranscribe {
 			options.Realtime = openClawVoiceOptions(cfg.Harness)
@@ -279,6 +364,7 @@ func newHarness(cfg config.Config, outputRoot string, lookup func(string) ([]byt
 			MaxConcurrentSubagents: cfg.Harness.Subagents.MaxConcurrent,
 			Compaction:             hermesCompaction(cfg.Harness.Compaction),
 			ExtraBody:              hermesExtraBody(cfg.Harness.Hermes),
+			MCPServers:             hermesMCPServers(cfg),
 		}
 
 		if cfg.Harness.Mode == hermesharness.ModeVoiceTranscribe {
@@ -417,6 +503,8 @@ func setupBenchmark(ctx context.Context, cfg config.Config) error {
 		return sweatlas.Setup(ctx, cfg.Benchmark.Root, cfg.Versions.SWEAtlas.RepositoryURL, cfg.Versions.SWEAtlas.Revision)
 	case "swebenchpro":
 		return swebenchpro.Setup(ctx, cfg.Benchmark.Root, cfg.Versions.SWEbenchPro.DatasetRepositoryURL, cfg.Versions.SWEbenchPro.DatasetRevision, cfg.Versions.SWEbenchPro.EvaluatorRepositoryURL, cfg.Versions.SWEbenchPro.EvaluatorRevision)
+	case "toolathlon":
+		return toolathlon.Setup(ctx, cfg.Benchmark.Root, cfg.Versions.Toolathlon.RepositoryURL, cfg.Versions.Toolathlon.Revision)
 	default:
 		return fmt.Errorf("unsupported benchmark type %q", cfg.Benchmark.Type)
 	}
@@ -490,6 +578,16 @@ func loadPreparationTasks(ctx context.Context, cfg config.Config, taskIDs []stri
 			return nil, fmt.Errorf("load terminalbench2 tasks: %w", err)
 		}
 		return tasks, nil
+	case "toolathlon":
+		benchmark, err := toolathlon.New(toolathlonOptions(cfg, taskIDs, nil, cfg.OutputDir))
+		if err != nil {
+			return nil, fmt.Errorf("validate toolathlon profile: %w", err)
+		}
+		tasks, err := benchmark.Tasks(ctx)
+		if err != nil {
+			return nil, fmt.Errorf("load toolathlon tasks: %w", err)
+		}
+		return tasks, nil
 	default:
 		return nil, fmt.Errorf("unsupported benchmark type %q", cfg.Benchmark.Type)
 	}

cmd/aries/wiring_test.go

@@ -10,14 +10,17 @@ import (
 	"os"
 	"path/filepath"
 	"reflect"
+	"slices"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/hyscale-lab/aries/internal/app"
 	runtimesglang "github.com/hyscale-lab/aries/internal/modelruntime/sglang"
+	"github.com/hyscale-lab/aries/pkg/benchmark/toolathlon"
 	"github.com/hyscale-lab/aries/pkg/config"
 	"github.com/hyscale-lab/aries/pkg/core"
+	hermesharness "github.com/hyscale-lab/aries/pkg/harness/hermes"
 	openclawharness "github.com/hyscale-lab/aries/pkg/harness/openclaw"
 )
 
@@ -177,6 +180,84 @@ func TestValidateComponentsRequiresPairedHarnessAndBridge(t *testing.T) {
 	}
 }
 
+// The adapter starts Toolathlon's gateway at the sandbox's alias on the
+// gateway port and adds it to the harness's MCP client itself, ahead of any
+// server the profile names; the profile may not name one after it, and a
+// harness without an MCP client is refused.
+func TestToolathlonGatewayIsAddedToTheHarness(t *testing.T) {
+	base := func(servers ...config.HarnessMCPServerConfig) config.Config {
+		return config.Config{
+			Benchmark: config.BenchmarkConfig{Type: "toolathlon"},
+			Harness:   config.HarnessConfig{Type: "hermes", MCP: config.HarnessMCPConfig{Servers: servers}},
+			Sandbox:   config.SandboxConfig{Type: "docker"},
+			Bridge:    config.BridgeConfig{Type: "hermes-ssh"},
+		}
+	}
+	docs := config.HarnessMCPServerConfig{Name: "docs", URL: "https://docs.example/mcp", Transport: "streamable-http", TimeoutSeconds: 30}
+	for _, tc := range []struct {
+		name string
+		cfg  config.Config
+		want string
+	}{
+		{name: "no profile servers", cfg: base()},
+		{name: "another server beside the gateway", cfg: base(docs)},
+		{name: "the gateway's name taken", cfg: base(config.HarnessMCPServerConfig{Name: "toolathlon", URL: "http://task-sandbox:10086/sse", Transport: "sse"}), want: `harness.mcp.servers may not name "toolathlon"`},
+		{name: "openclaw harness", cfg: func() config.Config {
+			cfg := base()
+			cfg.Harness.Type = "openclaw"
+			cfg.Bridge.Type = "openclaw-ssh"
+			return cfg
+		}()},
+		{name: "a harness without an MCP client", cfg: func() config.Config {
+			cfg := base()
+			cfg.Harness.Type = "other"
+			return cfg
+		}(), want: "requires a harness with an MCP client"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			err := validateComponents(tc.cfg)
+			if tc.want == "" {
+				if err != nil {
+					t.Fatalf("err=%v", err)
+				}
+				return
+			}
+			if err == nil || !strings.Contains(err.Error(), tc.want) {
+				t.Fatalf("err=%v, want %q", err, tc.want)
+			}
+		})
+	}
+
+	gateway := hermesharness.MCPServer{Name: "toolathlon", URL: "http://task-sandbox:10086/sse", Transport: "sse", TimeoutSeconds: toolathlon.GatewayCallTimeoutSeconds}
+	if got := hermesMCPServers(base()); !slices.Equal(got, []hermesharness.MCPServer{gateway}) {
+		t.Fatalf("servers = %+v, want the gateway alone", got)
+	}
+	want := []hermesharness.MCPServer{gateway, {Name: "docs", URL: "https://docs.example/mcp", Transport: "streamable-http", TimeoutSeconds: 30}}
+	if got := hermesMCPServers(base(docs)); !slices.Equal(got, want) {
+		t.Fatalf("servers = %+v, want the gateway then the profile's", got)
+	}
+	// A profile that moves the gateway port moves the entry with it.
+	moved := base()
+	moved.Benchmark.Toolathlon = &config.ToolathlonConfig{GatewayPort: 20086}
+	if got := hermesMCPServers(moved); len(got) != 1 || got[0].URL != "http://task-sandbox:20086/sse" {
+		t.Fatalf("moved port: servers = %+v", got)
+	}
+	// Another benchmark gets only what its profile names.
+	other := base(docs)
+	other.Benchmark.Type = "terminalbench2"
+	if got := hermesMCPServers(other); !slices.Equal(got, want[1:]) {
+		t.Fatalf("terminalbench2 servers = %+v, want the profile's alone", got)
+	}
+	// OpenClaw receives the same list in its own type.
+	openclaw := base(docs)
+	openclaw.Harness.Type = "openclaw"
+	openclaw.Bridge.Type = "openclaw-ssh"
+	wantOpenclaw := []openclawharness.MCPServer{{Name: "toolathlon", URL: "http://task-sandbox:10086/sse", Transport: "sse", TimeoutSeconds: toolathlon.GatewayCallTimeoutSeconds}, {Name: "docs", URL: "https://docs.example/mcp", Transport: "streamable-http", TimeoutSeconds: 30}}
+	if got := openclawMCPServers(openclaw); !slices.Equal(got, wantOpenclaw) {
+		t.Fatalf("openclaw servers = %+v", got)
+	}
+}
+
 func TestMakeLintIncludesInternalPackages(t *testing.T) {
 	makefile, err := os.ReadFile(filepath.Join("..", "..", "Makefile"))
 	if err != nil {

configs/versions.json

@@ -17,6 +17,10 @@
     "evaluator_repository_url": "https://github.com/scaleapi/SWE-bench_Pro-os.git",
     "evaluator_revision": "ca10a60a5fcae51e6948ffe1485d4153d421e6c5"
   },
+  "toolathlon": {
+    "repository_url": "https://github.com/hkust-nlp/Toolathlon.git",
+    "revision": "9be8d8fe07a497b18ee61e3f2ae694e9797f39eb"
+  },
   "openclaw": {
     "image": "ghcr.io/openclaw/openclaw:2026.7.1"
   },