feat(mcp): retry unhealthy servers and restore tools on recovery (#1655)

.changeset/mcp-retry-unhealthy-servers.md

@@ -0,0 +1,5 @@
+---
+'@nanocollective/nanocoder': minor
+---
+
+Retry unhealthy MCP servers on periodic health checks with exponential backoff and restore tools automatically when servers recover

docs/configuration/mcp-configuration.md

@@ -200,7 +200,7 @@ These fields work with all transport types:
 | `alwaysAllow` | Array of tool names that skip confirmation prompts |
 | `enabled` | Whether the server is active (default: `true`). `false` skips it entirely — no connection, no tools registered |
 | `timeout` | Connection timeout in milliseconds. Bounds the connection handshake and the initial tool listing; individual tool calls keep the MCP SDK's default timeout (60 seconds) |
-| `healthCheckInterval` | Optional MCP ping interval in milliseconds (default: `30000`). Set to `0` to disable periodic health checks. Transport errors trigger a liveness ping even when periodic checks are disabled; recoverable errors preserve tools if the server responds. Health pings use `timeout` when configured, otherwise 10 seconds. A failed ping or transport close marks the server unhealthy and removes its tools until MCP servers are reinitialized or the session is restarted. |
+| `healthCheckInterval` | Optional MCP ping interval in milliseconds (default: `30000`). Set to `0` to disable periodic health checks. Transport errors trigger a liveness ping even when periodic checks are disabled; recoverable errors preserve tools if the server responds. Health pings use `timeout` when configured, otherwise 10 seconds. A failed ping or transport close marks the server unhealthy and removes its tools; unhealthy servers continue to be checked with exponential backoff and automatically restore their tools when they respond again. |
 | `tags` | Array of tags, shown as `#tag` labels in `/mcp` output |
 
 ## Auto-Approve Tools

source/mcp/mcp-client.spec.ts

@@ -1441,7 +1441,229 @@ test('MCPClient health check marks a server unhealthy and removes its tools', as
 	t.false(client.isServerConnected(serverName));
 	t.deepEqual(client.getServerTools(serverName), []);
 	t.is(client.getServerInfo(serverName)?.health, 'unhealthy');
-	clearInterval((client as any).healthTimers.get(serverName));
+	clearTimeout((client as any).healthTimers.get(serverName));
+});
+
+test('MCPClient health check retries unhealthy server and restores its tools when ping succeeds', async t => {
+	let pingCalls = 0;
+	const transport: {onclose?: () => void; onerror?: (error: Error) => void} = {};
+	const client = new SeamMCPClient({
+		async ping() {
+			pingCalls++;
+			if (pingCalls === 1) {
+				throw new Error('temporary network hiccup');
+			}
+			return {};
+		},
+		async listTools() {
+			return {
+				tools: [
+					{
+						name: 'restored_tool',
+						description: 'restored tool description',
+						inputSchema: {type: 'object'},
+					},
+				],
+			};
+		},
+		getServerCapabilities() {
+			return {tools: {}};
+		},
+	});
+	const serverName = 'recovering-server';
+	(client as any).clients.set(serverName, (client as any).injected);
+	(client as any).serverConfigs.set(serverName, {
+		name: serverName,
+		transport: 'http',
+		url: 'http://localhost:1/mcp',
+	});
+	(client as any).serverTools.set(serverName, [
+		{name: 'restored_tool', serverName},
+	]);
+	(client as any).health.set(serverName, 'connected');
+	(client as any).startHealthChecks(
+		serverName,
+		(client as any).injected,
+		transport,
+		10,
+	);
+
+	// Wait for first ping to fail and second ping to recover
+	await new Promise<void>(resolve => {
+		const deadline = Date.now() + 1000;
+		const waitForPings = () => {
+			if (pingCalls >= 2 || Date.now() >= deadline) {
+				resolve();
+				return;
+			}
+			setTimeout(waitForPings, 5);
+		};
+		waitForPings();
+	});
+
+	t.true(pingCalls >= 2);
+	t.true(client.isServerConnected(serverName));
+	t.is(client.getServerInfo(serverName)?.health, 'connected');
+	const tools = client.getServerTools(serverName);
+	t.is(tools.length, 1);
+	t.is(tools[0].name, 'restored_tool');
+	clearTimeout((client as any).healthTimers.get(serverName));
+});
+
+test('MCPClient health check treats JSON-RPC MethodNotFound as alive and recovers tools', async t => {
+	let pingCalls = 0;
+	const transport: {onclose?: () => void; onerror?: (error: Error) => void} = {};
+	const client = new SeamMCPClient({
+		async ping() {
+			pingCalls++;
+			const error = new Error('Method not found');
+			(error as any).code = -32601;
+			throw error;
+		},
+		async listTools() {
+			return {
+				tools: [
+					{
+						name: 'alive_tool',
+						description: 'alive tool description',
+						inputSchema: {type: 'object'},
+					},
+				],
+			};
+		},
+		getServerCapabilities() {
+			return {tools: {}};
+		},
+	});
+	const serverName = 'method-not-found-server';
+	(client as any).clients.set(serverName, (client as any).injected);
+	(client as any).serverConfigs.set(serverName, {
+		name: serverName,
+		transport: 'stdio',
+	});
+	(client as any).serverTools.set(serverName, []);
+	(client as any).health.set(serverName, 'unhealthy');
+	(client as any).startHealthChecks(
+		serverName,
+		(client as any).injected,
+		transport,
+		10,
+	);
+
+	await new Promise<void>(resolve => {
+		const deadline = Date.now() + 1000;
+		const waitForPing = () => {
+			if (pingCalls > 0 || Date.now() >= deadline) {
+				resolve();
+				return;
+			}
+			setTimeout(waitForPing, 5);
+		};
+		waitForPing();
+	});
+
+	t.true(pingCalls > 0);
+	t.true(client.isServerConnected(serverName));
+	t.is(client.getServerInfo(serverName)?.health, 'connected');
+	const tools = client.getServerTools(serverName);
+	t.is(tools.length, 1);
+	t.is(tools[0].name, 'alive_tool');
+	clearTimeout((client as any).healthTimers.get(serverName));
+});
+
+test('MCPClient health check applies exponential backoff on consecutive failures and resets on recovery', async t => {
+	let pingCalls = 0;
+	let pingShouldFail = true;
+
+	const client = new SeamMCPClient({
+		async ping() {
+			pingCalls++;
+			if (pingShouldFail) {
+				throw new Error('connection refused');
+			}
+			return {};
+		},
+		async listTools() {
+			return {
+				tools: [
+					{
+						name: 'backoff_tool',
+						description: 'backoff tool desc',
+						inputSchema: {type: 'object'},
+					},
+				],
+			};
+		},
+		getServerCapabilities() {
+			return {tools: {}};
+		},
+	});
+
+	const serverName = 'backoff-server';
+	(client as any).clients.set(serverName, (client as any).injected);
+	(client as any).serverConfigs.set(serverName, {
+		name: serverName,
+		transport: 'http',
+		url: 'http://localhost:1/mcp',
+	});
+	(client as any).serverTools.set(serverName, [
+		{name: 'backoff_tool', serverName},
+	]);
+	(client as any).health.set(serverName, 'connected');
+
+	const transport: {onclose?: () => void; onerror?: (error: Error) => void} = {};
+	(client as any).startHealthChecks(
+		serverName,
+		(client as any).injected,
+		transport,
+		10,
+	);
+
+	// First failure happens at initial 10ms timer
+	await new Promise<void>(resolve => {
+		const check = () => {
+			if ((client as any).healthFailureCounts.get(serverName) === 1) {
+				resolve();
+				return;
+			}
+			setTimeout(check, 5);
+		};
+		check();
+	});
+
+	t.is(client.getServerInfo(serverName)?.health, 'unhealthy');
+	t.is((client as any).healthFailureCounts.get(serverName), 1);
+
+	// Wait for second failure -> failure count = 2
+	await new Promise<void>(resolve => {
+		const check = () => {
+			if ((client as any).healthFailureCounts.get(serverName) >= 2) {
+				resolve();
+				return;
+			}
+			setTimeout(check, 5);
+		};
+		check();
+	});
+	t.is((client as any).healthFailureCounts.get(serverName), 2);
+
+	// Now allow ping to succeed -> should recover and reset failure count
+	pingShouldFail = false;
+	await new Promise<void>(resolve => {
+		const check = () => {
+			if (client.getServerInfo(serverName)?.health === 'connected') {
+				resolve();
+				return;
+			}
+			setTimeout(check, 5);
+		};
+		check();
+	});
+
+	t.is(client.getServerInfo(serverName)?.health, 'connected');
+	t.is((client as any).healthFailureCounts.get(serverName), undefined);
+	t.is(client.getServerTools(serverName).length, 1);
+	clearTimeout((client as any).healthTimers.get(serverName));
 });
 
 test('MCPClient transport close marks a server unhealthy immediately', t => {
@@ -1475,7 +1697,7 @@ test('MCPClient transport close marks a server unhealthy immediately', t => {
 	t.false(client.isServerConnected(serverName));
 	t.is(client.getServerInfo(serverName)?.health, 'unhealthy');
 	t.deepEqual(client.getServerTools(serverName), []);
-	clearInterval((client as any).healthTimers.get(serverName));
+	clearTimeout((client as any).healthTimers.get(serverName));
 });
 
 for (const pingFails of [false, true]) {