diff --git a/client/public/incidentio.svg b/client/public/incidentio.svg new file mode 100644 index 000000000..caebc3fe6 --- /dev/null +++ b/client/public/incidentio.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/client/src/app/api/connected-accounts/[provider]/route.ts b/client/src/app/api/connected-accounts/[provider]/route.ts index 2f14eac0a..5bd4868f1 100644 --- a/client/src/app/api/connected-accounts/[provider]/route.ts +++ b/client/src/app/api/connected-accounts/[provider]/route.ts @@ -24,7 +24,7 @@ export async function DELETE( const { provider } = await context.params // Validate provider - if (!['gcp', 'azure', 'aws', 'github', 'grafana', 'datadog', 'netdata', 'ovh', 'scaleway', 'tailscale', 'slack', 'google_chat', 'splunk', 'dynatrace', 'confluence', 'jira', 'sharepoint', 'coroot', 'thousandeyes', 'jenkins', 'cloudbees', 'bigpanda', 'spinnaker', 'newrelic', 'opsgenie'].includes(provider)) { + if (!['gcp', 'azure', 'aws', 'github', 'grafana', 'datadog', 'netdata', 'ovh', 'scaleway', 'tailscale', 'slack', 'google_chat', 'splunk', 'dynatrace', 'confluence', 'jira', 'sharepoint', 'coroot', 'thousandeyes', 'jenkins', 'cloudbees', 'bigpanda', 'spinnaker', 'newrelic', 'opsgenie', 'incidentio'].includes(provider)) { return NextResponse.json( { error: 'Invalid provider' }, { status: 400 } @@ -442,6 +442,26 @@ export async function DELETE( return NextResponse.json({ success: true }) } + // Special handling for incident.io + if (provider === 'incidentio') { + const response = await fetch(`${API_BASE_URL}/incidentio/disconnect`, { + method: 'DELETE', + headers: authHeaders, + }) + + if (!response.ok) { + const errorText = await response.text() + console.error('Backend error disconnecting incident.io:', errorText) + return NextResponse.json( + { error: 'Failed to disconnect incident.io' }, + { status: response.status } + ) + } + + const data = await response.json() + return NextResponse.json(data) + } + // Special handling for BigPanda if (provider === 'bigpanda') { const response = await fetch(`${API_BASE_URL}/bigpanda/disconnect`, { diff --git a/client/src/app/api/incident-io/alerts/route.ts b/client/src/app/api/incident-io/alerts/route.ts new file mode 100644 index 000000000..39a3c1150 --- /dev/null +++ b/client/src/app/api/incident-io/alerts/route.ts @@ -0,0 +1,6 @@ +import { NextRequest } from 'next/server'; +import { forwardRequest } from '@/lib/backend-proxy'; + +export async function GET(request: NextRequest) { + return forwardRequest(request, 'GET', '/incidentio/alerts', 'incident-io/alerts'); +} diff --git a/client/src/app/api/incident-io/alerts/webhook-url/route.ts b/client/src/app/api/incident-io/alerts/webhook-url/route.ts new file mode 100644 index 000000000..10805c674 --- /dev/null +++ b/client/src/app/api/incident-io/alerts/webhook-url/route.ts @@ -0,0 +1,6 @@ +import { NextRequest } from 'next/server'; +import { forwardRequest } from '@/lib/backend-proxy'; + +export async function GET(request: NextRequest) { + return forwardRequest(request, 'GET', '/incidentio/alerts/webhook-url', 'incident-io/webhook-url'); +} diff --git a/client/src/app/api/incident-io/connect/route.ts b/client/src/app/api/incident-io/connect/route.ts new file mode 100644 index 000000000..b9c97dde7 --- /dev/null +++ b/client/src/app/api/incident-io/connect/route.ts @@ -0,0 +1,6 @@ +import { NextRequest } from 'next/server'; +import { forwardRequest } from '@/lib/backend-proxy'; + +export async function POST(request: NextRequest) { + return forwardRequest(request, 'POST', '/incidentio/connect', 'incident-io/connect'); +} diff --git a/client/src/app/api/incident-io/rca-settings/route.ts b/client/src/app/api/incident-io/rca-settings/route.ts new file mode 100644 index 000000000..0fa0a1edd --- /dev/null +++ b/client/src/app/api/incident-io/rca-settings/route.ts @@ -0,0 +1,10 @@ +import { NextRequest } from 'next/server'; +import { forwardRequest } from '@/lib/backend-proxy'; + +export async function GET(request: NextRequest) { + return forwardRequest(request, 'GET', '/incidentio/rca-settings', 'incident-io/rca-settings'); +} + +export async function PUT(request: NextRequest) { + return forwardRequest(request, 'PUT', '/incidentio/rca-settings', 'incident-io/rca-settings'); +} diff --git a/client/src/app/api/incident-io/status/route.ts b/client/src/app/api/incident-io/status/route.ts new file mode 100644 index 000000000..4fa3a5bcf --- /dev/null +++ b/client/src/app/api/incident-io/status/route.ts @@ -0,0 +1,6 @@ +import { NextRequest } from 'next/server'; +import { forwardRequest } from '@/lib/backend-proxy'; + +export async function GET(request: NextRequest) { + return forwardRequest(request, 'GET', '/incidentio/status', 'incident-io/status'); +} diff --git a/client/src/app/api/incident-io/webhook-secret/route.ts b/client/src/app/api/incident-io/webhook-secret/route.ts new file mode 100644 index 000000000..8d7dc5678 --- /dev/null +++ b/client/src/app/api/incident-io/webhook-secret/route.ts @@ -0,0 +1,6 @@ +import { NextRequest } from 'next/server'; +import { forwardRequest } from '@/lib/backend-proxy'; + +export async function PUT(request: NextRequest) { + return forwardRequest(request, 'PUT', '/incidentio/webhook-secret', 'incident-io/webhook-secret'); +} diff --git a/client/src/app/incident-io/auth/page.tsx b/client/src/app/incident-io/auth/page.tsx new file mode 100644 index 000000000..ff6a2cbda --- /dev/null +++ b/client/src/app/incident-io/auth/page.tsx @@ -0,0 +1,170 @@ +"use client"; + +import { useState } from "react"; +import { useToast } from "@/hooks/use-toast"; +import { useConnectorAuth } from "@/hooks/use-connector-auth"; +import { incidentIoService } from "@/lib/services/incident-io"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { Loader2 } from "lucide-react"; +import { getUserFriendlyError } from "@/lib/utils"; +import { IncidentIoWebhookStep } from "@/components/incident-io/IncidentIoWebhookStep"; +import ConnectorAuthGuard from "@/components/connectors/ConnectorAuthGuard"; +import Image from "next/image"; + +export default function IncidentIoAuthPage() { + const { toast } = useToast(); + const [apiKey, setApiKey] = useState(""); + const [loading, setLoading] = useState(false); + + const { + isConnected, + isCheckingStatus, + updateLocalState, + disconnect, + } = useConnectorAuth({ + cacheKey: "incident_io_connection_status", + storageKey: "isIncidentIoConnected", + fetchStatus: () => incidentIoService.getStatus(), + disconnectPath: "/api/connected-accounts/incidentio", + }); + + const handleConnect = async (event: React.FormEvent) => { + event.preventDefault(); + setLoading(true); + + try { + const result = await incidentIoService.connect({ apiKey }); + updateLocalState(result); + toast({ title: "Success", description: "incident.io connected successfully!" }); + } catch (err: any) { + console.error("incident.io connection failed", err); + toast({ + title: "Failed to connect to incident.io", + description: getUserFriendlyError(err), + variant: "destructive", + }); + } finally { + setLoading(false); + setApiKey(""); + } + }; + + const handleDisconnect = async () => { + setLoading(true); + try { + await disconnect(); + toast({ title: "Success", description: "incident.io disconnected successfully" }); + } catch (err: any) { + console.error("incident.io disconnect failed", err); + toast({ + title: "Failed to disconnect incident.io", + description: getUserFriendlyError(err), + variant: "destructive", + }); + } finally { + setLoading(false); + } + }; + + if (isCheckingStatus) { + return ( + +
+ + + + + +
+ + ); + } + + return ( + +
+
+
+ incident.io +
+
+

incident.io

+

+ Incident lifecycle tracking and automated RCA +

+
+
+ +
+
+
+ 1 +
+
+
+ 2 +
+
+
+ +
+ + Connect + + → + + Configure Webhook + +
+ + {isConnected ? ( + + ) : ( + + + Connect to incident.io + + Create an API key at Settings → API keys in incident.io, then paste it below. + + + +
+
+ + setApiKey(e.target.value)} + required + /> +

+ The API key needs these permissions: View all incident data (including private incidents) and Create incidents. Keys are stored securely in Vault. +

+
+ + +
+ + + )} +
+ + ); +} diff --git a/client/src/app/incident-io/incidents/page.tsx b/client/src/app/incident-io/incidents/page.tsx new file mode 100644 index 000000000..b63a211c7 --- /dev/null +++ b/client/src/app/incident-io/incidents/page.tsx @@ -0,0 +1,206 @@ +"use client"; + +import { useEffect, useState } from "react"; +import { useRouter } from "next/navigation"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { incidentIoService, IncidentIoAlert } from "@/lib/services/incident-io"; + +export default function IncidentIoIncidentsPage() { + const router = useRouter(); + const [incidents, setIncidents] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + const [total, setTotal] = useState(0); + const [offset, setOffset] = useState(0); + const [limit] = useState(20); + + const loadIncidents = async (newOffset = 0) => { + try { + setLoading(true); + setError(null); + const response = await incidentIoService.getAlerts(limit, newOffset); + setIncidents(response.alerts); + setTotal(response.total); + setOffset(newOffset); + } catch (err: unknown) { + console.error("Failed to load incidents", err); + const errorMessage = err instanceof Error ? err.message : "Failed to load incidents"; + setError(errorMessage); + } finally { + setLoading(false); + } + }; + + useEffect(() => { + loadIncidents(); + }, []); + + const handleNextPage = () => { + if (offset + limit < total) { + loadIncidents(offset + limit); + } + }; + + const handlePrevPage = () => { + if (offset > 0) { + loadIncidents(Math.max(0, offset - limit)); + } + }; + + const formatDate = (dateStr?: string) => { + if (!dateStr) return "N/A"; + try { + return new Date(dateStr).toLocaleString(); + } catch { + return dateStr; + } + }; + + const getSeverityBadgeColor = (severity?: string) => { + const s = severity?.toLowerCase(); + if (s === "critical") return "bg-red-100 text-red-800 dark:bg-red-900 dark:text-red-200"; + if (s === "high") return "bg-orange-100 text-orange-800 dark:bg-orange-900 dark:text-orange-200"; + if (s === "medium") return "bg-yellow-100 text-yellow-800 dark:bg-yellow-900 dark:text-yellow-200"; + if (s === "low") return "bg-green-100 text-green-800 dark:bg-green-900 dark:text-green-200"; + return "bg-gray-100 text-gray-800 dark:bg-gray-800 dark:text-gray-200"; + }; + + const getStatusBadgeColor = (status?: string) => { + const s = status?.toLowerCase(); + if (s === "live" || s === "active" || s === "investigating") return "bg-red-100 text-red-800 dark:bg-red-900 dark:text-red-200"; + if (s === "fixing" || s === "monitoring") return "bg-yellow-100 text-yellow-800 dark:bg-yellow-900 dark:text-yellow-200"; + if (s === "closed" || s === "resolved") return "bg-green-100 text-green-800 dark:bg-green-900 dark:text-green-200"; + if (s === "declined") return "bg-gray-100 text-gray-800 dark:bg-gray-800 dark:text-gray-200"; + return "bg-blue-100 text-blue-800 dark:bg-blue-900 dark:text-blue-200"; + }; + + return ( +
+
+
+

incident.io Incidents

+

Incidents received via webhook

+
+
+ + +
+
+ + {error && ( + + +

+ + {error} +

+ + + )} + + {loading && ( + + +

Loading incidents...

+ + + )} + + {!loading && incidents.length === 0 && ( + + + +

No incidents received yet

+

+ Configure webhooks in incident.io to start receiving events +

+ + + + )} + + {!loading && incidents.length > 0 && ( + <> +
+ {incidents.map((incident) => ( + + +
+
+
+ {incident.name || "Untitled Incident"} + + {incident.severity || "unknown"} + + + {incident.status || "unknown"} + +
+ {incident.incidentType && ( + Type: {incident.incidentType} + )} +
+
+ + +
+
+ Received: + {formatDate(incident.receivedAt)} +
+ {incident.incidentId && ( +
+ Incident ID: + + {incident.incidentId} + +
+ )} +
+ {incident.payload && Object.keys(incident.payload).length > 0 && ( +
+ + View full payload + + 
+                        {JSON.stringify(incident.payload, null, 2)}
+
+
+ )} + + + ))} +
+ + {/* Pagination */} + {total > limit && ( +
+

+ Showing {offset + 1}-{Math.min(offset + limit, total)} of {total} +

+
+ + +
+
+ )} + + )} +
+ ); +} diff --git a/client/src/app/incidents/components/IncidentCard.tsx b/client/src/app/incidents/components/IncidentCard.tsx index 13b0a905a..85147fabb 100644 --- a/client/src/app/incidents/components/IncidentCard.tsx +++ b/client/src/app/incidents/components/IncidentCard.tsx @@ -37,6 +37,13 @@ import { Suggestion } from '@/lib/services/incidents'; import InfrastructureVisualization from '@/components/incidents/InfrastructureVisualization'; import ExecutionWaterfall from './ExecutionWaterfall'; import { ReactFlowProvider } from '@xyflow/react'; +import { connectorRegistry } from '@/components/connectors/ConnectorRegistry'; + +function sourceDisplayName(source: string): string { + const connector = connectorRegistry.get(source); + if (connector) return connector.name; + return source.charAt(0).toUpperCase() + source.slice(1); +} interface IncidentCardProps { incident: Incident; @@ -356,12 +363,12 @@ export default function IncidentCard({ incident, duration, showThoughts, onToggl rel="noopener noreferrer" className="inline-flex items-center gap-1.5 text-zinc-400 hover:text-white transition-colors" > - {alert.source.charAt(0).toUpperCase() + alert.source.slice(1)} Alert + {sourceDisplayName(alert.source)} Alert ) : ( - {alert.source.charAt(0).toUpperCase() + alert.source.slice(1)} Alert + {sourceDisplayName(alert.source)} Alert )} diff --git a/client/src/components/connectors/ConnectorRegistry.ts b/client/src/components/connectors/ConnectorRegistry.ts index f86ec5cd0..e45afa5d7 100644 --- a/client/src/components/connectors/ConnectorRegistry.ts +++ b/client/src/components/connectors/ConnectorRegistry.ts @@ -147,6 +147,19 @@ class ConnectorRegistry { storageKey: "isOpsGenieConnected", }); + this.register({ + id: "incidentio", + name: "incident.io", + description: "Connect incident.io for real-time incident lifecycle tracking. Receive webhook events, investigate incidents with timeline data, and post RCA results back automatically.", + iconPath: "/incidentio.svg", + iconBgColor: "bg-white dark:bg-white", + category: "Incident Management", + path: "/incident-io/auth", + storageKey: "isIncidentIoConnected", + alertsPath: "/incident-io/incidents", + alertsLabel: "View Incidents", + }); + this.register({ id: "bigpanda", name: "BigPanda", diff --git a/client/src/components/incident-io/IncidentIoWebhookStep.tsx b/client/src/components/incident-io/IncidentIoWebhookStep.tsx new file mode 100644 index 000000000..cf1e81bb5 --- /dev/null +++ b/client/src/components/incident-io/IncidentIoWebhookStep.tsx @@ -0,0 +1,380 @@ +"use client"; + +import { useEffect, useState } from "react"; +import { useRouter } from "next/navigation"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { Switch } from "@/components/ui/switch"; +import { Label } from "@/components/ui/label"; +import { Input } from "@/components/ui/input"; +import { CheckCircle2, Copy, ExternalLink, Loader2 } from "lucide-react"; +import { useToast } from "@/hooks/use-toast"; +import { incidentIoService, IncidentIoWebhookUrlResponse } from "@/lib/services/incident-io"; +import { copyToClipboard } from "@/lib/utils"; + +interface IncidentIoWebhookStepProps { + readonly onDisconnect: () => Promise; + readonly loading: boolean; +} + +function WebhookSecretField({ + hasSecret, + value, + onChange, + onSave, + saving, +}: { + readonly hasSecret: boolean; + readonly value: string; + readonly onChange: (v: string) => void; + readonly onSave: () => void; + readonly saving: boolean; +}) { + return ( +
+ + {hasSecret && ( +
+
+ + Signing secret configured — webhook signatures are being verified. +
+

+ To update, paste a new secret below. +

+
+ )} + {!hasSecret && ( +

+ After creating the webhook endpoint in incident.io, copy the signing secret + (starts with whsec_) and paste it below. + When set, Aurora will cryptographically verify that incoming webhooks are genuine. + Without it, any request to your webhook URL will be accepted. +

+ )} +
+ onChange(e.target.value)} + /> + +
+
+ ); +} + +function WebhookConfig({ + webhookData, + loadingWebhook, + hasWebhookSecret, + webhookSecret, + setWebhookSecret, + savingSecret, + onSaveSecret, + onCopyUrl, +}: { + readonly webhookData: IncidentIoWebhookUrlResponse | null; + readonly loadingWebhook: boolean; + readonly hasWebhookSecret: boolean; + readonly webhookSecret: string; + readonly setWebhookSecret: (v: string) => void; + readonly savingSecret: boolean; + readonly onSaveSecret: () => void; + readonly onCopyUrl: () => void; +}) { + if (loadingWebhook) { + return ( +
+ + Loading webhook URL... +
+ ); + } + + if (!webhookData) { + return ( +

+ Unable to load webhook URL. Please try refreshing. +

+ ); + } + + return ( +
+
+ +
+ + {webhookData.webhookUrl} + + +
+
+ + + +
+

Setup Instructions:

+
    + {webhookData.instructions.map((instruction) => ( +
  1. {instruction.replace(/^\d+\.\s*/, '')}
    2. + ))} +
+
+ + + View incident.io Webhook Documentation + +
+ ); +} + +export function IncidentIoWebhookStep({ onDisconnect, loading }: IncidentIoWebhookStepProps) { + const router = useRouter(); + const { toast } = useToast(); + const [webhookData, setWebhookData] = useState(null); + const [loadingWebhook, setLoadingWebhook] = useState(true); + const [rcaEnabled, setRcaEnabled] = useState(true); + const [postbackEnabled, setPostbackEnabled] = useState(false); + const [loadingSettings, setLoadingSettings] = useState(true); + const [updatingRca, setUpdatingRca] = useState(false); + const [updatingPostback, setUpdatingPostback] = useState(false); + const [webhookSecret, setWebhookSecret] = useState(""); + const [savingSecret, setSavingSecret] = useState(false); + const [hasWebhookSecret, setHasWebhookSecret] = useState(false); + + useEffect(() => { + let isMounted = true; + + const loadData = async () => { + setLoadingWebhook(true); + setLoadingSettings(true); + + try { + const [webhookResponse, rcaSettings] = await Promise.all([ + incidentIoService.getWebhookUrl(), + incidentIoService.getRcaSettings(), + ]); + + if (isMounted) { + setWebhookData(webhookResponse); + if (webhookResponse) { + setHasWebhookSecret(webhookResponse.hasWebhookSecret); + } + if (rcaSettings) { + setRcaEnabled(rcaSettings.rcaEnabled); + setPostbackEnabled(rcaSettings.postbackEnabled); + } + } + } catch (_error) { + console.error("Failed to load incident.io settings:", _error); + } finally { + if (isMounted) { + setLoadingWebhook(false); + setLoadingSettings(false); + } + } + }; + + loadData(); + return () => { isMounted = false; }; + }, []); + + const handleRcaToggle = async (enabled: boolean) => { + setUpdatingRca(true); + try { + const result = await incidentIoService.updateRcaSettings({ rcaEnabled: enabled }); + if (result) { + setRcaEnabled(result.rcaEnabled); + setPostbackEnabled(result.postbackEnabled); + toast({ + title: enabled ? "Automatic RCA Enabled" : "Automatic RCA Disabled", + description: enabled + ? "Aurora will automatically investigate new incidents from incident.io" + : "New incidents will be stored but not automatically investigated", + }); + } else { + toast({ title: "Failed to update settings", description: "Could not update RCA settings. Please try again.", variant: "destructive" }); + } + } catch (_error) { + toast({ title: "Failed to update settings", description: "Could not update RCA settings. Please try again.", variant: "destructive" }); + } finally { + setUpdatingRca(false); + } + }; + + const handlePostbackToggle = async (enabled: boolean) => { + setUpdatingPostback(true); + try { + const result = await incidentIoService.updateRcaSettings({ postbackEnabled: enabled }); + if (result) { + setPostbackEnabled(result.postbackEnabled); + toast({ + title: enabled ? "Post-back Enabled" : "Post-back Disabled", + description: enabled + ? "RCA results will be posted to the incident.io timeline" + : "RCA results will only be available in Aurora", + }); + } else { + toast({ title: "Failed to update settings", description: "Could not update post-back setting. Please try again.", variant: "destructive" }); + } + } catch (_error) { + toast({ title: "Failed to update settings", description: "Could not update post-back setting. Please try again.", variant: "destructive" }); + } finally { + setUpdatingPostback(false); + } + }; + + const copyWebhookUrl = async () => { + if (!webhookData?.webhookUrl) return; + try { + await copyToClipboard(webhookData.webhookUrl); + toast({ title: "Copied", description: "Webhook URL copied to clipboard" }); + } catch (_error) { + toast({ title: "Copy failed", description: "Could not copy to clipboard.", variant: "destructive" }); + } + }; + + const handleSaveWebhookSecret = async () => { + if (!webhookSecret.trim()) return; + setSavingSecret(true); + try { + const success = await incidentIoService.saveWebhookSecret(webhookSecret.trim()); + if (success) { + toast({ title: "Webhook secret saved", description: "Webhook signatures will now be verified." }); + setWebhookSecret(""); + setHasWebhookSecret(true); + } else { + toast({ title: "Failed to save", description: "Could not save webhook secret. Please try again.", variant: "destructive" }); + } + } catch (_error) { + toast({ title: "Failed to save", description: "Could not save webhook secret.", variant: "destructive" }); + } finally { + setSavingSecret(false); + } + }; + + return ( + + + + + Connected to incident.io + + + Your incident.io account is connected. Configure webhooks to receive incident events. + + + +
+ +
+ +
+
+
+ +

+ Automatically investigate new incidents with Aurora +

+
+ {loadingSettings ? ( + + ) : ( + + )} +
+
+ + {rcaEnabled && ( +
+
+
+ +

+ Automatically post RCA results back to the incident timeline +

+
+ {loadingSettings ? ( + + ) : ( + + )} +
+
+ )} + +
+

Webhook Configuration

+ +
+ +
+ +
+ + + ); +} diff --git a/client/src/components/tool-calls/CommandLogo.tsx b/client/src/components/tool-calls/CommandLogo.tsx index 2e9b04d8e..177c16f40 100644 --- a/client/src/components/tool-calls/CommandLogo.tsx +++ b/client/src/components/tool-calls/CommandLogo.tsx @@ -244,6 +244,14 @@ const logos = { onError={(e) => console.error('Failed to load OpsGenie logo:', e)} /> ), + incidentio: ( + incident.io console.error('Failed to load incident.io logo:', e)} + /> + ), web: ( { + cacheKey: string; + storageKey: string; + fetchStatus: () => Promise; + disconnectPath: string; +} + +export function useConnectorAuth({ + cacheKey, + storageKey, + fetchStatus, + disconnectPath, +}: UseConnectorAuthOptions) { + const [status, setStatus] = useState(null); + const [isCheckingStatus, setIsCheckingStatus] = useState(true); + const fetchStatusRef = useRef(fetchStatus); + fetchStatusRef.current = fetchStatus; + + const updateLocalState = useCallback( + (result: T, { fireEvent = true } = {}) => { + const prev = localStorage.getItem(cacheKey); + const wasConnected = prev ? JSON.parse(prev)?.connected : false; + + setStatus(result); + localStorage.setItem(cacheKey, JSON.stringify(result)); + if (result.connected) { + localStorage.setItem(storageKey, "true"); + } else { + localStorage.removeItem(storageKey); + } + if (fireEvent && wasConnected !== result.connected) { + globalThis.dispatchEvent(new CustomEvent("providerStateChanged")); + } + }, + [cacheKey, storageKey], + ); + + const refresh = useCallback(async () => { + try { + const result = await fetchStatusRef.current(); + if (result !== null) updateLocalState(result); + } catch { + // leave current status as-is + } finally { + setIsCheckingStatus(false); + } + }, [updateLocalState]); + + useEffect(() => { + const cached = localStorage.getItem(cacheKey); + if (cached) { + const parsed = JSON.parse(cached); + setStatus(parsed); + setIsCheckingStatus(false); + } + refresh(); + }, [cacheKey, refresh]); + + const disconnect = useCallback(async (): Promise => { + const response = await fetch(disconnectPath, { + method: "DELETE", + credentials: "include", + }); + if (response.ok || response.status === 204) { + setStatus({ connected: false } as T); + localStorage.removeItem(cacheKey); + localStorage.removeItem(storageKey); + window.dispatchEvent(new CustomEvent("providerStateChanged")); + return true; + } + const text = await response.text(); + throw new Error(text || "Failed to disconnect"); + }, [cacheKey, storageKey, disconnectPath]); + + return { + status, + setStatus, + isCheckingStatus, + isConnected: Boolean(status?.connected), + updateLocalState, + disconnect, + refresh, + }; +} diff --git a/client/src/lib/backend-proxy.ts b/client/src/lib/backend-proxy.ts index d890b8efe..3771c843f 100644 --- a/client/src/lib/backend-proxy.ts +++ b/client/src/lib/backend-proxy.ts @@ -140,3 +140,15 @@ export async function forwardRequest( return NextResponse.json({ error: `Failed to load ${errorLabel}` }, { status: 500 }); } } + +/** + * Forward an authenticated GET request to a backend API path, + * passing through query-string parameters and auth headers. + */ +export async function forwardAuthenticatedGet( + request: NextRequest, + backendPath: string, + errorLabel: string, +): Promise { + return forwardRequest(request, 'GET', backendPath, errorLabel); +} diff --git a/client/src/lib/services/incident-io.ts b/client/src/lib/services/incident-io.ts new file mode 100644 index 000000000..3535f008a --- /dev/null +++ b/client/src/lib/services/incident-io.ts @@ -0,0 +1,131 @@ +import { apiRequest } from '@/lib/services/api-client'; + +export interface IncidentIoStatus { + connected: boolean; + error?: string; +} + +export interface IncidentIoConnectPayload { + apiKey: string; +} + +export interface IncidentIoAlert { + id: number; + incidentId?: string; + name?: string; + status?: string; + severity?: string; + incidentType?: string; + payload?: Record; + receivedAt?: string; + createdAt?: string; +} + +export interface IncidentIoAlertsResponse { + alerts: IncidentIoAlert[]; + total: number; + limit: number; + offset: number; +} + +export interface IncidentIoWebhookUrlResponse { + webhookUrl: string; + hasWebhookSecret: boolean; + instructions: string[]; +} + +export interface IncidentIoRcaSettings { + rcaEnabled: boolean; + postbackEnabled: boolean; +} + +const API_BASE = '/api/incident-io'; + +export const incidentIoService = { + async getStatus(): Promise { + try { + const raw = await apiRequest>(`${API_BASE}/status`, { + cache: 'no-store', + }); + if (!raw) return null; + return { + connected: Boolean(raw.connected), + error: raw.error as string | undefined, + }; + } catch (error) { + console.error('[incidentIoService] Failed to fetch status:', error); + return null; + } + }, + + async connect(payload: IncidentIoConnectPayload): Promise { + const raw = await apiRequest>(`${API_BASE}/connect`, { + method: 'POST', + body: JSON.stringify(payload), + cache: 'no-store', + }); + return { + connected: Boolean(raw?.success), + error: raw?.error as string | undefined, + }; + }, + + async getAlerts(limit = 50, offset = 0, severity?: string): Promise { + const params = new URLSearchParams({ limit: String(limit), offset: String(offset) }); + if (severity) params.append('severity', severity); + + const raw = await apiRequest(`${API_BASE}/alerts?${params}`, { + cache: 'no-store', + }); + return raw ?? { alerts: [], total: 0, limit, offset }; + }, + + async getWebhookUrl(): Promise { + try { + return await apiRequest(`${API_BASE}/alerts/webhook-url`, { + cache: 'no-store', + }); + } catch (error) { + console.error('[incidentIoService] Failed to fetch webhook URL:', error); + return null; + } + }, + + async saveWebhookSecret(webhookSecret: string): Promise { + try { + await apiRequest(`${API_BASE}/webhook-secret`, { + method: 'PUT', + body: JSON.stringify({ webhookSecret }), + cache: 'no-store', + }); + return true; + } catch (error) { + console.error('[incidentIoService] Failed to save webhook secret:', error); + return false; + } + }, + + async getRcaSettings(): Promise { + try { + return await apiRequest(`${API_BASE}/rca-settings`, { + cache: 'no-store', + }); + } catch (error) { + console.error('[incidentIoService] Failed to fetch RCA settings:', error); + return null; + } + }, + + async updateRcaSettings(settings: Partial): Promise { + try { + return await apiRequest(`${API_BASE}/rca-settings`, { + method: 'PUT', + body: JSON.stringify(settings), + cache: 'no-store', + }); + } catch (error) { + console.error('[incidentIoService] Failed to update RCA settings:', error); + return null; + } + }, +}; diff --git a/client/src/middleware.ts b/client/src/middleware.ts index c4d898cbb..d63b07a9a 100644 --- a/client/src/middleware.ts +++ b/client/src/middleware.ts @@ -27,8 +27,8 @@ export default auth((req) => { const { nextUrl } = req const isLoggedIn = !!req.auth?.user?.id - const isPublicRoute = publicRoutes.some(route => - nextUrl.pathname.startsWith(route) + const isPublicRoute = publicRoutes.some(route => + nextUrl.pathname === route || nextUrl.pathname.startsWith(`${route}/`) ) const isAuthRoute = authRoutes.some(route => nextUrl.pathname.startsWith(route) diff --git a/server/celery_config.py b/server/celery_config.py index 2dca461de..bc80285cb 100644 --- a/server/celery_config.py +++ b/server/celery_config.py @@ -83,6 +83,7 @@ 'routes.newrelic.tasks', 'routes.jenkins.tasks', 'routes.spinnaker.tasks', + 'routes.incidentio.tasks', 'utils.terminal.terminal_pod_cleanup', 'chat.background.task', 'chat.background.summarization', diff --git a/server/chat/backend/agent/skills/integrations/incidentio/SKILL.md b/server/chat/backend/agent/skills/integrations/incidentio/SKILL.md new file mode 100644 index 000000000..40df39997 --- /dev/null +++ b/server/chat/backend/agent/skills/integrations/incidentio/SKILL.md @@ -0,0 +1,61 @@ +--- +name: incidentio +id: incidentio +description: "incident.io integration for listing incidents, investigating details, and reviewing timelines during RCA" +category: incident_management +connection_check: + method: is_connected_function + module: chat.backend.agent.tools.incidentio_tool + function: is_incidentio_connected +tools: + - list_incidentio_incidents + - get_incidentio_incident + - get_incidentio_timeline +index: "Incident management -- list incidents, get details, review timeline" +rca_priority: 2 +allowed-tools: list_incidentio_incidents, get_incidentio_incident, get_incidentio_timeline +metadata: + author: aurora + version: "1.0" +--- + +# incident.io Integration + +## Overview +incident.io integration for investigating incidents during Root Cause Analysis. Provides access to incident details, severity, roles, custom fields, and timeline events. + +## Instructions + +### Tool Usage (use in this order) +1. `list_incidentio_incidents()` -- Find recent or related incidents. Filter by status or severity. +2. `get_incidentio_incident(incident_id='X')` -- Get full details including roles, custom fields, and duration. +3. `get_incidentio_timeline(incident_id='X')` -- See the sequence of events, status changes, and human updates. + +### Investigation Patterns +- Find similar incidents: `list_incidentio_incidents(status='closed', page_size=10)` to check for recurring issues. +- Deep-dive current incident: `get_incidentio_incident(incident_id='...')` for severity, responders, and custom fields. +- Understand timeline: `get_incidentio_timeline(incident_id='...')` to see what actions were taken and when. + +## RCA Investigation Workflow + +**Step 1 -- Context gathering:** +`list_incidentio_incidents(status='live')` -- See what's currently happening. + +**Step 2 -- Incident details:** +`get_incidentio_incident(incident_id='...')` -- Understand severity, who's responding, and what services are affected. + +**Step 3 -- Timeline analysis:** +`get_incidentio_timeline(incident_id='...')` -- Reconstruct the sequence of events to find the trigger point. + +**Step 4 -- Pattern matching:** +`list_incidentio_incidents(status='closed', page_size=25)` -- Check if similar incidents happened before. + +**Step 5 -- Cross-correlate:** +After incident.io analysis, correlate with infrastructure data from other connected tools (logs, metrics, deployments). + +## Important Rules +- incident.io is a REMOTE service. Use only the API tools listed above. +- Start with `list_incidentio_incidents` to understand the landscape before diving into specifics. +- Timeline data is essential for understanding causality -- always check it during RCA. +- Incident.io roles tell you who was involved, which helps validate findings. +- Custom fields often contain service/component info useful for correlation. diff --git a/server/chat/backend/agent/skills/load_skill_tool.py b/server/chat/backend/agent/skills/load_skill_tool.py index 6296ff2dd..8661c0df6 100644 --- a/server/chat/backend/agent/skills/load_skill_tool.py +++ b/server/chat/backend/agent/skills/load_skill_tool.py @@ -52,25 +52,41 @@ def load_skill(skill_id: str, **kwargs) -> str: logger.warning("load_skill called without user_id — with_user_context wrapper may have failed") return json.dumps({"error": "No user context available — this indicates a system configuration issue."}) - # Dedup: if this skill was already loaded in this session, return short ack key = _session_key(user_id, session_id) - with _loaded_skills_lock: - already_loaded = _loaded_skills.get(key, set()) - if skill_id in already_loaded: - return f"Skill '{skill_id}' is already loaded in this conversation — no need to reload." try: from .registry import SkillRegistry registry = SkillRegistry.get_instance() + + # Normalize: strip dots/dashes/underscores/spaces for typo tolerance + # (e.g. "incident.io" or "incident-io" → "incidentio") + # No prefix matching — avoids false positives like "google" → "google_chat" + if skill_id not in registry.get_all_skill_ids(): + normalized = skill_id.lower().replace("dot", ".").replace(".", "").replace("-", "").replace("_", "").replace(" ", "") + for candidate in sorted(registry.get_all_skill_ids()): + candidate_norm = candidate.lower().replace(".", "").replace("-", "").replace("_", "") + if normalized == candidate_norm: + logger.info("load_skill normalized '%s' -> '%s'", skill_id, candidate) + skill_id = candidate + break + + # Dedup after canonicalization so aliases don't bypass cache + with _loaded_skills_lock: + already_loaded = _loaded_skills.get(key, set()) + if skill_id in already_loaded: + return f"Skill '{skill_id}' is already loaded in this conversation — no need to reload." + result = registry.load_skill(skill_id, user_id) if not result.is_connected: all_ids = registry.get_all_skill_ids() if skill_id not in all_ids: + connected = registry.get_connected_skill_ids(user_id) + hint = f"Only these integrations have skills: {', '.join(connected)}. Cloud providers (aws, gcp, azure) use cloud_exec directly — no load_skill needed." return json.dumps({ - "error": f"Unknown skill '{skill_id}'. Check the CONNECTED INTEGRATIONS index for valid skill IDs.", - "available": registry.get_connected_skill_ids(user_id), + "error": f"No skill '{skill_id}'. {hint}", + "available_skills": connected, }) return json.dumps({ "error": f"Integration '{skill_id}' is not connected for this user.", @@ -86,6 +102,6 @@ def load_skill(skill_id: str, **kwargs) -> str: return result.content - except Exception as e: - logger.error(f"Error loading skill '{skill_id}': {e}", exc_info=True) - return json.dumps({"error": f"Failed to load skill: {e}"}) + except Exception: + logger.exception("Error loading skill '%s'", skill_id) + return json.dumps({"error": "Failed to load skill. Please retry or contact support if the issue persists."}) diff --git a/server/chat/backend/agent/skills/registry.py b/server/chat/backend/agent/skills/registry.py index d700603ac..622290e01 100644 --- a/server/chat/backend/agent/skills/registry.py +++ b/server/chat/backend/agent/skills/registry.py @@ -313,18 +313,18 @@ def build_index(self, user_id: str) -> str: return "" lines = [ - "CONNECTED INTEGRATIONS (MANDATORY: call load_skill('id') BEFORE using ANY integration tool below):", - "You MUST call load_skill first to get the workflow, syntax, and constraints. Using tools without loading the skill first will produce wrong results.", + "CONNECTED INTEGRATIONS — call load_skill with the exact skill_id before using that integration's tools.", "", ] for meta in sorted(connected, key=lambda m: m.name): + display_name = meta.name or meta.id if meta.tools: tools_str = ", ".join(meta.tools[:4]) if len(meta.tools) > 4: tools_str += ", ..." - lines.append(f"- {meta.id}: {meta.index} [tools: {tools_str}]") + lines.append(f"- load_skill('{meta.id}') # {display_name}: {meta.index} [tools: {tools_str}]") else: - lines.append(f"- {meta.id}: {meta.index}") + lines.append(f"- load_skill('{meta.id}') # {display_name}: {meta.index}") lines.append("") return "\n".join(lines) diff --git a/server/chat/backend/agent/tools/cloud_tools.py b/server/chat/backend/agent/tools/cloud_tools.py index b1c24b054..74e4b271e 100644 --- a/server/chat/backend/agent/tools/cloud_tools.py +++ b/server/chat/backend/agent/tools/cloud_tools.py @@ -80,6 +80,15 @@ SplunkListIndexesArgs, SplunkListSourcetypesArgs, ) +from .incidentio_tool import ( + list_incidentio_incidents, + get_incidentio_incident, + get_incidentio_timeline, + is_incidentio_connected, + ListIncidentsArgs, + GetIncidentArgs, + GetTimelineArgs, +) from .coroot_tool import ( coroot_get_incidents, coroot_get_incident_detail, @@ -1506,6 +1515,58 @@ def cloud_exec_wrapper(provider: str, command: str, output_file: Optional[str] = else: logging.debug(f"Splunk tools not added - user {user_id} not connected to Splunk") + # Add incident.io tools if connected + if user_id and is_incidentio_connected(user_id): + context_wrapped_list = with_user_context(list_incidentio_incidents) + notification_wrapped_list = with_completion_notification(context_wrapped_list) + final_list_func = wrap_func_with_capture(notification_wrapped_list, "list_incidentio_incidents") if tool_capture else notification_wrapped_list + + tools.append(StructuredTool.from_function( + func=final_list_func, + name="list_incidentio_incidents", + description=( + "List incidents from incident.io. Use this to find related incidents, " + "identify patterns, and understand the scope of an ongoing issue. " + "Filter by status (live/closed/declined) or severity. " + "Supports pagination via 'after' cursor for large result sets." + ), + args_schema=ListIncidentsArgs, + )) + + context_wrapped_get = with_user_context(get_incidentio_incident) + notification_wrapped_get = with_completion_notification(context_wrapped_get) + final_get_func = wrap_func_with_capture(notification_wrapped_get, "get_incidentio_incident") if tool_capture else notification_wrapped_get + + tools.append(StructuredTool.from_function( + func=final_get_func, + name="get_incidentio_incident", + description=( + "Get full details of a specific incident.io incident including severity, " + "roles, custom fields, timestamps, and duration. Use this for deep-dive " + "investigation of a particular incident." + ), + args_schema=GetIncidentArgs, + )) + + context_wrapped_timeline = with_user_context(get_incidentio_timeline) + notification_wrapped_timeline = with_completion_notification(context_wrapped_timeline) + final_timeline_func = wrap_func_with_capture(notification_wrapped_timeline, "get_incidentio_timeline") if tool_capture else notification_wrapped_timeline + + tools.append(StructuredTool.from_function( + func=final_timeline_func, + name="get_incidentio_timeline", + description=( + "Get the timeline/updates for an incident.io incident. Shows the sequence " + "of events, status changes, severity changes, and human updates — essential " + "for understanding what happened and when during an incident." + ), + args_schema=GetTimelineArgs, + )) + + logging.info(f"Added 3 incident.io tools for user {user_id}") + else: + logging.debug(f"incident.io tools not added - user {user_id} not connected") + # Add Dynatrace tool if connected if user_id and is_dynatrace_connected(user_id): context_wrapped_dt = with_user_context(query_dynatrace) diff --git a/server/chat/backend/agent/tools/incidentio_tool.py b/server/chat/backend/agent/tools/incidentio_tool.py new file mode 100644 index 000000000..94c35ffa9 --- /dev/null +++ b/server/chat/backend/agent/tools/incidentio_tool.py @@ -0,0 +1,295 @@ +"""incident.io investigation tools for RCA agent. + +Provides three tools for RCA: +- list_incidentio_incidents: Find recent/related incidents for pattern analysis +- get_incidentio_incident: Deep-dive a specific incident (status, roles, custom fields, timestamps) +- get_incidentio_timeline: Fetch timeline updates for an incident to understand sequence of events +""" + +import json +import logging +from typing import Any, Dict, Optional + +from pydantic import BaseModel, Field + +from utils.auth.token_management import get_token_data + +_ERR_NO_USER = json.dumps({"error": "No user context available"}) +_ERR_NOT_CONNECTED = json.dumps({"error": "incident.io not connected"}) +_ERR_INVALID_ID = json.dumps({"error": "Invalid incident_id"}) + +logger = logging.getLogger(__name__) + +INCIDENTIO_API_BASE = "https://api.incident.io/v2" +INCIDENTIO_TIMEOUT = 20 +MAX_OUTPUT_SIZE = 500_000 + + +class ListIncidentsArgs(BaseModel): + """Arguments for list_incidentio_incidents.""" + status: Optional[str] = Field( + default=None, + description="Filter by status category: 'live', 'closed', 'declined'. Leave empty for all.", + ) + severity: Optional[str] = Field( + default=None, + description="Filter by severity ID or name (e.g., 'critical', 'major').", + ) + page_size: int = Field( + default=25, + description="Number of incidents to return (max 100).", + ) + after: Optional[str] = Field( + default=None, + description="Pagination cursor from a previous response's 'next_cursor'. Use to fetch the next page.", + ) + + +class GetIncidentArgs(BaseModel): + """Arguments for get_incidentio_incident.""" + incident_id: str = Field(description="The incident.io incident ID to retrieve.") + + +class GetTimelineArgs(BaseModel): + """Arguments for get_incidentio_timeline.""" + incident_id: str = Field(description="The incident.io incident ID to get timeline for.") + + +def _get_incidentio_credentials(user_id: str) -> Optional[str]: + try: + creds = get_token_data(user_id, "incidentio") + if not creds: + return None + api_key = creds.get("api_key") or creds.get("token") or creds.get("access_token") + return api_key if api_key else None + except Exception as exc: + logger.error("[INCIDENTIO-TOOL] Failed to get credentials: %s", exc) + return None + + +def is_incidentio_connected(user_id: str) -> bool: + return _get_incidentio_credentials(user_id) is not None + + +def _api_request(api_key: str, method: str, path: str, params: Optional[Dict] = None) -> Dict[str, Any]: + import requests + + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + "Accept": "application/json", + } + url = f"{INCIDENTIO_API_BASE}{path}" + try: + resp = requests.request(method, url, headers=headers, params=params, timeout=INCIDENTIO_TIMEOUT) + resp.raise_for_status() + return resp.json() + except requests.exceptions.Timeout: + return {"error": "Request to incident.io timed out"} + except requests.exceptions.ConnectionError: + return {"error": "Unable to reach incident.io API"} + except requests.HTTPError as exc: + status = exc.response.status_code if exc.response else None + if status == 401: + return {"error": "Authentication failed — API key may be invalid or expired"} + if status == 403: + return {"error": "API key lacks required permissions"} + if status == 404: + return {"error": "Resource not found"} + return {"error": f"incident.io API error (HTTP {status})"} + + +def _format_incident_summary(incident: Dict[str, Any]) -> Dict[str, Any]: + """Distill an incident object into the fields useful for RCA.""" + severity = incident.get("severity") or {} + inc_type = incident.get("incident_type") or {} + roles = incident.get("incident_role_assignments") or [] + + formatted_roles = [] + for r in roles[:10]: + role_def = r.get("role", {}) + assignee = r.get("assignee", {}) + formatted_roles.append({ + "role": role_def.get("name", "unknown"), + "assignee": assignee.get("name", "unassigned"), + }) + + custom_fields = [] + for cf in (incident.get("custom_field_entries") or [])[:20]: + field_def = cf.get("custom_field", {}) + values = cf.get("values") or [] + val_labels = [v.get("label") or v.get("value_text") or str(v) for v in values[:3]] + custom_fields.append({ + "field": field_def.get("name", "unknown"), + "values": val_labels, + }) + + return { + "id": incident.get("id"), + "name": incident.get("name"), + "status": incident.get("status"), + "severity": severity.get("name") if isinstance(severity, dict) else str(severity), + "type": inc_type.get("name") if isinstance(inc_type, dict) else str(inc_type), + "summary": incident.get("summary") or "", + "created_at": incident.get("created_at"), + "updated_at": incident.get("updated_at"), + "permalink": incident.get("permalink"), + "roles": formatted_roles, + "custom_fields": custom_fields, + "duration_seconds": _calculate_duration(incident), + } + + +def _calculate_duration(incident: Dict[str, Any]) -> Optional[int]: + from datetime import datetime + created = incident.get("created_at") + closed = incident.get("closed_at") + if not created: + return None + try: + start = datetime.fromisoformat(created.replace("Z", "+00:00")) + if closed: + end = datetime.fromisoformat(closed.replace("Z", "+00:00")) + else: + end = datetime.now(start.tzinfo) + return int((end - start).total_seconds()) + except Exception: + return None + + +def _truncate_output(data: Any) -> str: + output = json.dumps(data, default=str) + if len(output) <= MAX_OUTPUT_SIZE: + return output + if isinstance(data, dict) and "incidents" in data and isinstance(data["incidents"], list): + items = data["incidents"] + while items and len(json.dumps(data, default=str)) > MAX_OUTPUT_SIZE: + items.pop() + data["truncated"] = True + data["total_returned"] = len(items) + return json.dumps(data, default=str) + return output[:MAX_OUTPUT_SIZE] + + +def list_incidentio_incidents( + status: Optional[str] = None, + severity: Optional[str] = None, + page_size: int = 25, + after: Optional[str] = None, + user_id: Optional[str] = None, + **kwargs, +) -> str: + """List incidents from incident.io. Use 'after' cursor to paginate through large result sets.""" + if not user_id: + return _ERR_NO_USER + + api_key = _get_incidentio_credentials(user_id) + if not api_key: + return _ERR_NOT_CONNECTED + + page_size = min(max(page_size, 1), 100) + params: Dict[str, Any] = {"page_size": page_size} + if status: + params["status_category[one_of]"] = status + if severity: + params["severity[one_of]"] = severity + if after: + params["after"] = after + + result = _api_request(api_key, "GET", "/incidents", params=params) + if "error" in result: + return json.dumps(result) + + incidents = result.get("incidents") or [] + summaries = [_format_incident_summary(inc) for inc in incidents] + + pagination = result.get("pagination_meta") or {} + output: Dict[str, Any] = { + "incidents": summaries, + "total_returned": len(summaries), + } + if pagination.get("after"): + output["next_cursor"] = pagination["after"] + output["has_more"] = True + else: + output["has_more"] = False + if "total_record_count" in pagination: + output["total_count"] = pagination["total_record_count"] + + return _truncate_output(output) + + +def get_incidentio_incident( + incident_id: str, + user_id: Optional[str] = None, + **kwargs, +) -> str: + """Get full details of a specific incident.io incident for deep-dive investigation.""" + if not user_id: + return _ERR_NO_USER + + api_key = _get_incidentio_credentials(user_id) + if not api_key: + return _ERR_NOT_CONNECTED + + if not incident_id or len(incident_id) > 100: + return _ERR_INVALID_ID + + result = _api_request(api_key, "GET", f"/incidents/{incident_id}") + if "error" in result: + return json.dumps(result) + + incident = result.get("incident") or result + summary = _format_incident_summary(incident) + + # Include additional detail not in the list view + summary["timestamps"] = { + "created_at": incident.get("created_at"), + "updated_at": incident.get("updated_at"), + "closed_at": incident.get("closed_at"), + "last_activity_at": incident.get("last_activity_at"), + } + summary["workload_minutes"] = incident.get("workload_minutes_total") + summary["slack_channel"] = (incident.get("slack_channel_name") or + (incident.get("incident_channel") or {}).get("name")) + + return _truncate_output(summary) + + +def get_incidentio_timeline( + incident_id: str, + user_id: Optional[str] = None, + **kwargs, +) -> str: + """Get timeline/updates for an incident to understand the sequence of events and decisions.""" + if not user_id: + return _ERR_NO_USER + + api_key = _get_incidentio_credentials(user_id) + if not api_key: + return _ERR_NOT_CONNECTED + + if not incident_id or len(incident_id) > 100: + return _ERR_INVALID_ID + + result = _api_request(api_key, "GET", "/incident_updates", params={"incident_id": incident_id}) + if "error" in result: + return json.dumps(result) + + updates = result.get("incident_updates") or [] + formatted = [] + for u in updates[:50]: + formatted.append({ + "id": u.get("id"), + "message": u.get("message") or u.get("new_value") or "", + "created_at": u.get("created_at"), + "updater": (u.get("updater") or {}).get("name", "system"), + "new_status": u.get("new_incident_status", {}).get("name") if u.get("new_incident_status") else None, + "new_severity": u.get("new_severity", {}).get("name") if u.get("new_severity") else None, + }) + + return _truncate_output({ + "incident_id": incident_id, + "updates": formatted, + "total_returned": len(formatted), + }) diff --git a/server/chat/background/rca_prompt_builder.py b/server/chat/background/rca_prompt_builder.py index 443b33a83..c3eb3f88e 100644 --- a/server/chat/background/rca_prompt_builder.py +++ b/server/chat/background/rca_prompt_builder.py @@ -1206,3 +1206,67 @@ def build_opsgenie_rca_prompt( alert_details['entity'] = entity return build_rca_prompt('opsgenie', alert_details, providers, user_id) + + +def _incidentio_dict_name(obj, default: str = "") -> str: + """Extract .name from a dict-or-scalar incident.io field.""" + if isinstance(obj, dict): + return obj.get("name", default) + return str(obj) if obj else default + + +def _incidentio_format_roles(roles: list) -> str: + return ", ".join( + f"{r.get('role', {}).get('name', '?')}: {r.get('assignee', {}).get('name', 'unassigned')}" + for r in roles[:5] + ) + + +def _incidentio_format_custom_fields(custom_fields: list) -> str: + return ", ".join( + f"{cf.get('custom_field', {}).get('name', '?')}=" + f"{(cf.get('values') or [{}])[0].get('label', '?')}" + for cf in custom_fields[:5] + if cf.get("values") + ) + + +def build_incidentio_rca_prompt( + payload: Dict[str, Any], + providers: Optional[List[str]] = None, + user_id: Optional[str] = None, +) -> str: + """Build RCA prompt from incident.io webhook event payload.""" + event = payload.get("event", {}) or {} + incident = event.get("incident") or payload.get("incident") or {} + + name = incident.get("name") or incident.get("title") or "Unknown Incident" + status = incident.get("status") or "unknown" + summary = incident.get("summary") or "" + permalink = incident.get("permalink") or "" + severity = _incidentio_dict_name(incident.get("severity")) + inc_type = _incidentio_dict_name(incident.get("incident_type")) + + role_str = _incidentio_format_roles(incident.get("incident_role_assignments") or []) + cf_str = _incidentio_format_custom_fields(incident.get("custom_field_entries") or []) + + message_parts = [f"Incident: {name}"] + for label, value in [("Summary", summary), ("Roles", role_str), + ("Fields", cf_str), ("Link", permalink)]: + if value: + message_parts.append(f"{label}: {value}") + + labels = {} + if severity: + labels['severity'] = severity + if inc_type: + labels['incident_type'] = inc_type + + alert_details = { + 'title': name, + 'status': f"{status} (severity: {severity})" if severity else status, + 'message': ". ".join(message_parts), + 'labels': labels, + } + + return build_rca_prompt('incidentio', alert_details, providers, user_id) diff --git a/server/chat/background/task.py b/server/chat/background/task.py index 50c58821f..415b2551b 100644 --- a/server/chat/background/task.py +++ b/server/chat/background/task.py @@ -272,7 +272,7 @@ def _ensure_llm_context_history( _RATE_LIMIT_MAX_REQUESTS = 5 # Max 5 background chats per window # RCA sources that use rca_context in system prompt -_RCA_SOURCES = {'grafana', 'datadog', 'netdata', 'splunk', 'slack', 'google_chat', 'pagerduty', 'dynatrace', 'jenkins', 'cloudbees', 'spinnaker', 'newrelic', 'chat', 'opsgenie'} +_RCA_SOURCES = {'grafana', 'datadog', 'netdata', 'splunk', 'slack', 'google_chat', 'pagerduty', 'dynatrace', 'jenkins', 'cloudbees', 'spinnaker', 'newrelic', 'chat', 'opsgenie', 'incidentio'} # Initialize Redis client at module load time - fails if Redis is unavailable _redis_client = get_redis_client() diff --git a/server/main_compute.py b/server/main_compute.py index 90f9dfae2..2ab70e255 100644 --- a/server/main_compute.py +++ b/server/main_compute.py @@ -110,6 +110,10 @@ "allow_headers": ["Content-Type", "X-Provider", "X-Requested-With", "X-User-ID", "Authorization", "X-Provider-Preference"], "methods": ["GET", "POST", "DELETE", "OPTIONS"]}, + r"/incidentio/*": {"origins": FRONTEND_URL, "supports_credentials": True, + "allow_headers": ["Content-Type", "X-Provider", "X-Requested-With", "X-User-ID", + "Authorization", "X-Provider-Preference"], + "methods": ["GET", "POST", "PUT", "DELETE", "OPTIONS"]}, r"/bigpanda/*": {"origins": FRONTEND_URL, "supports_credentials": True, "allow_headers": ["Content-Type", "X-Provider", "X-Requested-With", "X-User-ID", "Authorization", "X-Provider-Preference"], @@ -210,6 +214,7 @@ "/jenkins/webhook/", "/cloudbees/webhook/", "/spinnaker/webhook/", + "/incidentio/alerts/webhook/", "/ovh_api/ovh/oauth2/callback", "/azure/callback", "/azure/setup-script", @@ -380,6 +385,11 @@ def enforce_user_org_binding(): app.register_blueprint(splunk_bp, url_prefix="/splunk") app.register_blueprint(splunk_search_bp, url_prefix="/splunk") +# --- incident.io Integration Routes --- +from routes.incidentio import bp as incidentio_bp # noqa: F401 +import routes.incidentio.tasks # noqa: F401 +app.register_blueprint(incidentio_bp, url_prefix="/incidentio") + # --- Coroot Integration Routes --- from routes.coroot import bp as coroot_bp # noqa: F401 app.register_blueprint(coroot_bp, url_prefix="/coroot") diff --git a/server/routes/incidentio/__init__.py b/server/routes/incidentio/__init__.py new file mode 100644 index 000000000..5675cccaf --- /dev/null +++ b/server/routes/incidentio/__init__.py @@ -0,0 +1 @@ +from .incidentio_routes import incidentio_bp as bp diff --git a/server/routes/incidentio/incidentio_routes.py b/server/routes/incidentio/incidentio_routes.py new file mode 100644 index 000000000..bf4064cd1 --- /dev/null +++ b/server/routes/incidentio/incidentio_routes.py @@ -0,0 +1,385 @@ +"""incident.io connector routes: connect, status, disconnect, webhook, alerts, settings.""" + +import base64 +import hashlib +import hmac +import logging +import os +import re +from typing import Any, Dict, Optional + +import requests +from flask import Blueprint, jsonify, request + +from routes.incidentio.tasks import process_incidentio_event +from utils.db.connection_pool import db_pool +from utils.auth.stateless_auth import ( + get_user_preference, + set_rls_context, + store_user_preference, +) +from utils.auth.token_management import get_token_data, store_tokens_in_db +from utils.auth.rbac_decorators import require_permission +from utils.secrets.secret_ref_utils import delete_user_secret + +INCIDENTIO_API_BASE = "https://api.incident.io/v2" +INCIDENTIO_TIMEOUT = 15 + +_SAFE_LOG_RE = re.compile(r"[^a-zA-Z0-9._\-]") + +logger = logging.getLogger(__name__) + +incidentio_bp = Blueprint("incidentio", __name__) + + +def _sanitize_for_log(value: str, max_len: int = 80) -> str: + """Strip any character that isn't alphanumeric, dot, dash, or underscore.""" + return _SAFE_LOG_RE.sub("", value)[:max_len] + + +class IncidentioAPIError(Exception): + """Error codes avoid leaking HTTP response bodies through str(exc).""" + INVALID_KEY = "invalid_key" + FORBIDDEN = "forbidden" + TIMEOUT = "timeout" + UNREACHABLE = "unreachable" + API_ERROR = "api_error" + + _USER_MESSAGES = { + INVALID_KEY: "Invalid API key", + FORBIDDEN: "API key lacks required permissions", + TIMEOUT: "Connection to incident.io timed out", + UNREACHABLE: "Unable to reach incident.io API", + API_ERROR: "Failed to validate API key with incident.io", + } + + def __init__(self, code: str): + self.code = code + super().__init__(self._USER_MESSAGES.get(code, self._USER_MESSAGES[self.API_ERROR])) + + +class IncidentioClient: + """Client for the incident.io REST API.""" + + def __init__(self, api_key: str): + self.api_key = api_key + + @property + def headers(self) -> Dict[str, str]: + return { + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json", + "Accept": "application/json", + } + + def _request(self, method: str, path: str, **kwargs) -> requests.Response: + url = f"{INCIDENTIO_API_BASE}{path}" + try: + response = requests.request( + method, url, headers=self.headers, timeout=INCIDENTIO_TIMEOUT, **kwargs + ) + response.raise_for_status() + return response + except requests.exceptions.Timeout: + raise IncidentioAPIError(IncidentioAPIError.TIMEOUT) + except requests.exceptions.ConnectionError: + raise IncidentioAPIError(IncidentioAPIError.UNREACHABLE) + except requests.HTTPError as exc: + status = exc.response.status_code if exc.response is not None else None + logger.warning("[INCIDENTIO] HTTP %s from %s", status, path) + if status == 401: + raise IncidentioAPIError(IncidentioAPIError.INVALID_KEY) + if status == 403: + raise IncidentioAPIError(IncidentioAPIError.FORBIDDEN) + raise IncidentioAPIError(IncidentioAPIError.API_ERROR) + + def list_incidents(self, page_size: int = 5) -> Dict[str, Any]: + return self._request("GET", "/incidents", params={"page_size": page_size}).json() + + def get_incident(self, incident_id: str) -> Dict[str, Any]: + return self._request("GET", f"/incidents/{incident_id}").json() + + def get_incident_updates(self, incident_id: str) -> Dict[str, Any]: + return self._request("GET", "/incident_updates", params={"incident_id": incident_id}).json() + + def post_incident_update(self, incident_id: str, message: str) -> Dict[str, Any]: + return self._request( + "POST", + "/incident_updates", + json={"incident_id": incident_id, "message": message}, + ).json() + + +def _get_stored_credentials(user_id: str) -> Optional[Dict[str, Any]]: + try: + creds = get_token_data(user_id, "incidentio") + return creds if creds else None + except Exception: + logger.exception("[INCIDENTIO] Failed to retrieve credentials for user %s", user_id) + return None + + +# ── Routes ────────────────────────────────────────────────────────── + + +@incidentio_bp.route("/connect", methods=["POST"]) +@require_permission("connectors", "write") +def connect(user_id): + """Validate API key against incident.io and store credentials.""" + try: + data = request.get_json(force=True, silent=True) or {} + except Exception: + data = {} + + api_key = data.get("apiKey") + if not api_key or not isinstance(api_key, str): + return jsonify({"error": "apiKey is required"}), 400 + + if len(api_key) < 20 or len(api_key) > 500: + return jsonify({"error": "Invalid API key format"}), 400 + + client = IncidentioClient(api_key) + try: + client.list_incidents(page_size=1) + except IncidentioAPIError as exc: + logger.warning("[INCIDENTIO] Connection validation failed: %s", exc.code) + return jsonify({"error": str(exc)}), 502 + + token_payload = {"api_key": api_key} + + try: + store_tokens_in_db(user_id, token_payload, "incidentio") + except Exception: + logger.exception("[INCIDENTIO] Failed to store credentials for user %s", user_id) + return jsonify({"error": "Failed to store credentials"}), 500 + + return jsonify({"success": True, "connected": True}) + + +@incidentio_bp.route("/status", methods=["GET"]) +@require_permission("connectors", "read") +def status(user_id): + """Check incident.io connection status.""" + creds = _get_stored_credentials(user_id) + if not creds: + return jsonify({"connected": False}) + + api_key = creds.get("api_key") + if not api_key: + return jsonify({"connected": False}) + + client = IncidentioClient(api_key) + try: + client.list_incidents(page_size=1) + except IncidentioAPIError: + logger.warning("[INCIDENTIO] Status check failed for user %s", user_id) + return jsonify({"connected": False, "error": "Connection check failed"}) + + return jsonify({"connected": True}) + + +@incidentio_bp.route("/disconnect", methods=["POST", "DELETE"]) +@require_permission("connectors", "write") +def disconnect(user_id): + """Remove stored incident.io credentials.""" + try: + success, _ = delete_user_secret(user_id, "incidentio") + if not success: + return jsonify({"error": "Failed to delete stored credentials"}), 500 + + logger.info("[INCIDENTIO] Disconnected user %s", user_id) + return jsonify({"success": True, "message": "incident.io disconnected successfully"}) + except Exception: + logger.exception("[INCIDENTIO] Failed to disconnect user %s", user_id) + return jsonify({"error": "Failed to disconnect incident.io"}), 500 + + +@incidentio_bp.route("/alerts/webhook/", methods=["POST"]) +def alert_webhook(user_id: str): + """Receive webhook events from incident.io.""" + if not user_id: + return jsonify({"error": "user_id is required"}), 400 + + log_uid = _sanitize_for_log(user_id, 36) + + creds = get_token_data(user_id, "incidentio") + if not creds: + logger.warning("[INCIDENTIO] Webhook with no connection: %s", log_uid) + return jsonify({"error": "incident.io not connected for this user"}), 404 + + webhook_secret = creds.get("webhook_secret") + if webhook_secret: + msg_id = request.headers.get("webhook-id", "") + timestamp = request.headers.get("webhook-timestamp", "") + signature_header = request.headers.get("webhook-signature", "") + if not msg_id or not timestamp or not signature_header: + logger.warning("[INCIDENTIO] Webhook rejected: missing Svix headers: %s", log_uid) + return jsonify({"error": "Missing webhook signature headers"}), 401 + to_sign = f"{msg_id}.{timestamp}.{request.get_data(as_text=True)}" + secret_bytes = base64.b64decode(webhook_secret.split("_")[-1]) if webhook_secret.startswith("whsec_") else webhook_secret.encode() + expected = base64.b64encode(hmac.new(secret_bytes, to_sign.encode(), hashlib.sha256).digest()).decode() + signatures = [s.split(",")[-1] for s in signature_header.split(" ")] + if not any(hmac.compare_digest(expected, s) for s in signatures): + logger.warning("[INCIDENTIO] Webhook rejected: invalid signature: %s", log_uid) + return jsonify({"error": "Invalid webhook signature"}), 401 + + payload = request.get_json(silent=True) or {} + + event_type = payload.get("event_type") or (payload.get("event", {}) or {}).get("type", "unknown") + logger.info("[INCIDENTIO] Webhook received: user=%s event=%s", log_uid, _sanitize_for_log(str(event_type))) + + metadata = {"remote_addr": request.remote_addr} + process_incidentio_event.delay(payload, metadata, user_id) + + return jsonify({"received": True}) + + +@incidentio_bp.route("/alerts", methods=["GET"]) +@require_permission("connectors", "read") +def get_alerts(user_id): + """Fetch stored incident.io events.""" + limit = min(max(request.args.get("limit", 50, type=int), 1), 200) + offset = max(request.args.get("offset", 0, type=int), 0) + severity_filter = request.args.get("severity") + + try: + with db_pool.get_admin_connection() as conn: + cursor = conn.cursor() + org_id = set_rls_context(cursor, conn, user_id, log_prefix="[INCIDENTIO:alerts]") + + where = "WHERE org_id = %s" + params = [org_id] + if severity_filter: + where += " AND severity = %s" + params.append(severity_filter) + + cursor.execute( + f""" + SELECT id, incident_id, incident_name, incident_status, severity, + incident_type, payload, received_at, created_at + FROM incidentio_alerts + {where} + ORDER BY received_at DESC + LIMIT %s OFFSET %s + """, + (*params, limit, offset), + ) + rows = cursor.fetchall() + + cursor.execute(f"SELECT COUNT(*) FROM incidentio_alerts {where}", params) + total = cursor.fetchone()[0] + + return jsonify({ + "alerts": [ + { + "id": r[0], + "incidentId": r[1], + "name": r[2], + "status": r[3], + "severity": r[4], + "incidentType": r[5], + "payload": r[6], + "receivedAt": r[7].isoformat() if r[7] else None, + "createdAt": r[8].isoformat() if r[8] else None, + } + for r in rows + ], + "total": total, + "limit": limit, + "offset": offset, + }) + except Exception: + logger.exception("[INCIDENTIO] Failed to fetch alerts") + return jsonify({"error": "Failed to fetch alerts"}), 500 + + +@incidentio_bp.route("/alerts/webhook-url", methods=["GET"]) +@require_permission("connectors", "read") +def get_webhook_url(user_id): + """Return the webhook URL for this user's incident.io configuration.""" + ngrok_url = os.getenv("NGROK_URL", "").rstrip("/") + backend_url = os.getenv("NEXT_PUBLIC_BACKEND_URL", "").rstrip("/") + base_url = ngrok_url if ngrok_url and backend_url.startswith("http://localhost") else backend_url + if not base_url: + base_url = request.host_url.rstrip("/") + + webhook_url = f"{base_url}/incidentio/alerts/webhook/{user_id}" + + creds = _get_stored_credentials(user_id) + has_secret = bool(creds and creds.get("webhook_secret")) + + return jsonify({ + "webhookUrl": webhook_url, + "hasWebhookSecret": has_secret, + "instructions": [ + "1. Go to incident.io → Settings → Webhooks", + "2. Click 'Add endpoint'", + "3. Paste the webhook URL above", + "4. Select events: incident.created, incident.updated", + "5. Save the endpoint, then copy the signing secret from the endpoint settings", + "6. Paste the signing secret (starts with whsec_) into the field above", + ], + }) + + +@incidentio_bp.route("/webhook-secret", methods=["PUT"]) +@require_permission("connectors", "write") +def save_webhook_secret(user_id): + """Store the webhook signing secret from incident.io's endpoint settings.""" + try: + data = request.get_json(force=True, silent=True) or {} + except Exception: + data = {} + + secret = data.get("webhookSecret", "").strip() + if not secret: + return jsonify({"error": "webhookSecret is required"}), 400 + + creds = _get_stored_credentials(user_id) + if not creds: + return jsonify({"error": "incident.io not connected"}), 400 + + creds["webhook_secret"] = secret + try: + store_tokens_in_db(user_id, creds, "incidentio") + except Exception: + logger.exception("[INCIDENTIO] Failed to store webhook secret for user %s", user_id) + return jsonify({"error": "Failed to store webhook secret"}), 500 + + return jsonify({"success": True}) + + +@incidentio_bp.route("/rca-settings", methods=["GET"]) +@require_permission("connectors", "read") +def get_rca_settings(user_id): + rca_enabled = get_user_preference(user_id, "incidentio_rca_enabled", default=True) + postback_enabled = get_user_preference(user_id, "incidentio_postback_enabled", default=False) + return jsonify({"rcaEnabled": rca_enabled, "postbackEnabled": postback_enabled}) + + +@incidentio_bp.route("/rca-settings", methods=["PUT"]) +@require_permission("connectors", "write") +def update_rca_settings(user_id): + try: + data = request.get_json(force=True, silent=True) or {} + except Exception: + data = {} + + rca_enabled = data.get("rcaEnabled") + postback_enabled = data.get("postbackEnabled") + + if rca_enabled is not None: + if not isinstance(rca_enabled, bool): + return jsonify({"error": "rcaEnabled must be a boolean"}), 400 + store_user_preference(user_id, "incidentio_rca_enabled", rca_enabled) + + if postback_enabled is not None: + if not isinstance(postback_enabled, bool): + return jsonify({"error": "postbackEnabled must be a boolean"}), 400 + store_user_preference(user_id, "incidentio_postback_enabled", postback_enabled) + + return jsonify({ + "success": True, + "rcaEnabled": get_user_preference(user_id, "incidentio_rca_enabled", default=True), + "postbackEnabled": get_user_preference(user_id, "incidentio_postback_enabled", default=False), + }) diff --git a/server/routes/incidentio/tasks.py b/server/routes/incidentio/tasks.py new file mode 100644 index 000000000..6cb7f2776 --- /dev/null +++ b/server/routes/incidentio/tasks.py @@ -0,0 +1,458 @@ +"""Celery tasks for incident.io webhook event processing.""" + +from __future__ import annotations + +import json +import logging +from datetime import datetime, timezone +from typing import Any, Dict, Optional + +from celery_config import celery_app +from chat.background.rca_prompt_builder import build_incidentio_rca_prompt +from services.correlation.alert_correlator import AlertCorrelator +from services.correlation import handle_correlated_alert + +logger = logging.getLogger(__name__) + + +def _should_trigger_rca(user_id: str) -> bool: + from utils.auth.stateless_auth import get_user_preference + return get_user_preference(user_id, "incidentio_rca_enabled", default=True) + + +def _should_postback(user_id: str) -> bool: + from utils.auth.stateless_auth import get_user_preference + return get_user_preference(user_id, "incidentio_postback_enabled", default=False) + + +def _resolve_incident_object(payload: Dict[str, Any]) -> Dict[str, Any]: + """Find the incident dict inside an incident.io webhook payload.""" + event = payload.get("event", {}) or {} + incident = event.get("incident") or payload.get("incident") or None + + if not incident: + for key, value in payload.items(): + if key.startswith("public_incident.") and isinstance(value, dict): + incident = value.get("incident") or value + break + + return incident or {} + + +def _safe_name(obj, default: str = "") -> str: + """Extract .name from a dict-or-scalar field.""" + if isinstance(obj, dict): + return obj.get("name", default) + return str(obj) if obj else default + + +def _extract_incident_fields(payload: Dict[str, Any]) -> Dict[str, Any]: + """Extract normalized incident fields from the webhook event envelope.""" + event = payload.get("event", {}) or {} + incident = _resolve_incident_object(payload) + + return { + "incident_id": incident.get("id") or payload.get("id"), + "incident_name": incident.get("name") or incident.get("title") or "Untitled Incident", + "incident_status": incident.get("status") or event.get("status") or "unknown", + "severity": _safe_name(incident.get("severity"), "unknown"), + "incident_type": _safe_name(incident.get("incident_type")), + "summary": incident.get("summary") or "", + "created_at": incident.get("created_at"), + "updated_at": incident.get("updated_at"), + "permalink": incident.get("permalink") or "", + "custom_fields": incident.get("custom_field_entries") or [], + "roles": incident.get("incident_role_assignments") or [], + } + + +def _map_severity(severity_name: str) -> str: + """Normalize incident.io severity names to standard levels.""" + s = severity_name.lower().strip() + if s in ("critical", "sev0", "sev1", "p0", "p1"): + return "critical" + if s in ("high", "major", "sev2", "p2"): + return "high" + if s in ("medium", "moderate", "sev3", "p3"): + return "medium" + if s in ("low", "minor", "sev4", "sev5", "p4", "p5"): + return "low" + return "unknown" + + +_NEW_INCIDENT_EVENTS = frozenset(( + "incident.created", "v2.incidents.created", + "incident.declared", "public_incident.incident_created", + "public_incident.incident_created_v2", +)) + + +def _build_alert_metadata(fields: Dict[str, Any], event_type: str) -> Dict[str, Any]: + meta: Dict[str, Any] = { + "permalink": fields["permalink"], + "summary": fields["summary"], + "event_type": event_type, + } + if fields["roles"]: + meta["roles"] = [ + {"role": r.get("role", {}).get("name", ""), + "assignee": r.get("assignee", {}).get("name", "")} + for r in fields["roles"][:5] + ] + return meta + + +def _try_correlate(cursor, conn, *, user_id, alert_db_id, fields, service, + normalized_severity, alert_metadata, payload, org_id) -> bool: + """Attempt alert correlation. Returns True if correlated (and committed).""" + try: + cursor.execute("SAVEPOINT sp_correlation") + correlator = AlertCorrelator() + result = correlator.correlate( + cursor=cursor, user_id=user_id, source_type="incidentio", + source_alert_id=alert_db_id, alert_title=fields["incident_name"], + alert_service=service, alert_severity=normalized_severity, + alert_metadata=alert_metadata, org_id=org_id, + ) + if result.is_correlated: + handle_correlated_alert( + cursor=cursor, user_id=user_id, incident_id=result.incident_id, + source_type="incidentio", source_alert_id=alert_db_id, + alert_title=fields["incident_name"], alert_service=service, + alert_severity=normalized_severity, correlation_result=result, + alert_metadata=alert_metadata, raw_payload=payload, org_id=org_id, + ) + conn.commit() + return True + cursor.execute("RELEASE SAVEPOINT sp_correlation") + except Exception as corr_exc: + cursor.execute("ROLLBACK TO SAVEPOINT sp_correlation") + logger.warning("[INCIDENTIO] Correlation failed, continuing: %s", corr_exc) + return False + + +def _create_and_link_incident(cursor, conn, *, user_id, org_id, alert_db_id, + fields, service, normalized_severity, + alert_metadata, received_at) -> Optional[str]: + """Create Aurora incident record and link the alert. Returns incident_id or None.""" + cursor.execute( + """ + INSERT INTO incidents + (user_id, org_id, source_type, source_alert_id, alert_title, + alert_service, severity, status, started_at, alert_metadata) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (org_id, source_type, source_alert_id, user_id) DO UPDATE + SET updated_at = CURRENT_TIMESTAMP, + alert_metadata = EXCLUDED.alert_metadata + RETURNING id + """, + ( + user_id, org_id, "incidentio", alert_db_id, + fields["incident_name"], service, normalized_severity, + "investigating", received_at, json.dumps(alert_metadata), + ), + ) + row = cursor.fetchone() + incident_id = row[0] if row else None + conn.commit() + + if not incident_id: + return None + + try: + cursor.execute( + """INSERT INTO incident_alerts + (user_id, org_id, incident_id, source_type, source_alert_id, + alert_title, alert_service, alert_severity, correlation_strategy, + correlation_score, alert_metadata) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""", + ( + user_id, org_id, incident_id, "incidentio", alert_db_id, + fields["incident_name"], service, normalized_severity, + "primary", 1.0, json.dumps(alert_metadata), + ), + ) + cursor.execute( + "UPDATE incidents SET affected_services = ARRAY[%s] WHERE id = %s", + (service, incident_id), + ) + conn.commit() + except Exception as e: + conn.rollback() + logger.warning("[INCIDENTIO] Failed to link alert: %s", e) + + return str(incident_id) + + +@celery_app.task( + bind=True, max_retries=3, default_retry_delay=30, name="incidentio.process_event" +) +def process_incidentio_event( + self, + payload: Dict[str, Any], + metadata: Optional[Dict[str, Any]] = None, + user_id: Optional[str] = None, +) -> None: + """Process an incident.io webhook event.""" + try: + event_type = payload.get("event_type") or (payload.get("event", {}) or {}).get("type", "unknown") + fields = _extract_incident_fields(payload) + logger.info( + "[INCIDENTIO][EVENT][USER:%s] type=%s incident=%s status=%s severity=%s", + user_id or "unknown", event_type, fields["incident_name"], + fields["incident_status"], fields["severity"], + ) + + if not user_id: + logger.warning("[INCIDENTIO] No user_id — event not stored") + return + + _store_and_process_event(user_id, event_type, fields, payload) + + except Exception as exc: + logger.exception("[INCIDENTIO] Failed to process event") + raise self.retry(exc=exc) + + +def _store_and_process_event(user_id: str, event_type: str, + fields: Dict[str, Any], payload: Dict[str, Any]) -> None: + """Store the alert and optionally trigger correlation/RCA.""" + from utils.db.connection_pool import db_pool + from utils.auth.stateless_auth import set_rls_context + + incident_id = None + service = "" + normalized_severity = "" + alert_metadata: Dict[str, Any] = {} + + with db_pool.get_admin_connection() as conn: + with conn.cursor() as cursor: + org_id = set_rls_context(cursor, conn, user_id, log_prefix="[INCIDENTIO]") + if not org_id: + return + + received_at = datetime.now(timezone.utc) + normalized_severity = _map_severity(fields["severity"]) + + alert_db_id = _upsert_alert(cursor, conn, user_id=user_id, org_id=org_id, + fields=fields, payload=payload, + severity=normalized_severity, received_at=received_at) + if not alert_db_id: + conn.rollback() + logger.error("[INCIDENTIO] Failed to store event for user %s", user_id) + return + + if event_type not in _NEW_INCIDENT_EVENTS: + conn.commit() + logger.info("[INCIDENTIO] Stored update event (no RCA trigger)") + return + + service = _extract_service(fields) + alert_metadata = _build_alert_metadata(fields, event_type) + + if _try_correlate(cursor, conn, user_id=user_id, alert_db_id=alert_db_id, + fields=fields, service=service, + normalized_severity=normalized_severity, + alert_metadata=alert_metadata, payload=payload, org_id=org_id): + return + + if not _should_trigger_rca(user_id): + conn.commit() + logger.info("[INCIDENTIO] Stored incident (RCA disabled)") + return + + incident_id = _create_and_link_incident( + cursor, conn, user_id=user_id, org_id=org_id, + alert_db_id=alert_db_id, fields=fields, service=service, + normalized_severity=normalized_severity, + alert_metadata=alert_metadata, received_at=received_at, + ) + + if incident_id: + _trigger_rca_pipeline( + user_id=user_id, incident_id=incident_id, fields=fields, + payload=payload, alert_metadata=alert_metadata, + service=service, severity=normalized_severity, + ) + + +def _upsert_alert(cursor, conn, *, user_id, org_id, fields, payload, + severity, received_at) -> Optional[int]: + """Insert or update the incidentio_alerts row. Returns the DB id or None.""" + cursor.execute( + """ + INSERT INTO incidentio_alerts + (user_id, org_id, incident_id, incident_name, incident_status, + severity, incident_type, payload, received_at) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (org_id, incident_id) DO UPDATE + SET incident_name = EXCLUDED.incident_name, + incident_status = EXCLUDED.incident_status, + severity = EXCLUDED.severity, + incident_type = EXCLUDED.incident_type, + payload = EXCLUDED.payload, + received_at = EXCLUDED.received_at + RETURNING id + """, + ( + user_id, org_id, fields["incident_id"], + fields["incident_name"], fields["incident_status"], + severity, fields["incident_type"], + json.dumps(payload), received_at, + ), + ) + row = cursor.fetchone() + return row[0] if row else None + + +def _extract_service(fields: Dict[str, Any]) -> str: + """Best-effort service extraction from incident fields.""" + for cf in fields.get("custom_fields") or []: + field_def = cf.get("custom_field", {}) + if field_def.get("name", "").lower() in ("service", "affected_service", "component"): + values = cf.get("values") or [] + if values: + return str(values[0].get("label") or values[0].get("value", ""))[:255] + + name = fields.get("incident_name", "") + if ":" in name: + return name.split(":")[0].strip()[:255] + + return fields.get("incident_type") or "unknown" + + +def _trigger_rca_pipeline( + user_id: str, + incident_id: str, + fields: Dict[str, Any], + payload: Dict[str, Any], + alert_metadata: Dict[str, Any], + service: str, + severity: str, +) -> None: + """Trigger summary generation and background RCA for an incident.""" + from chat.background.summarization import generate_incident_summary + + generate_incident_summary.delay( + incident_id=str(incident_id), + user_id=user_id, + source_type="incidentio", + alert_title=fields["incident_name"], + severity=severity, + service=service, + raw_payload=payload, + alert_metadata=alert_metadata, + ) + + try: + from chat.background.task import ( + run_background_chat, + create_background_chat_session, + is_background_chat_allowed, + ) + + if not is_background_chat_allowed(user_id): + logger.info("[INCIDENTIO] Background RCA rate-limited for user %s", user_id) + return + + chat_title = f"RCA: {fields['incident_name']}" + session_id = create_background_chat_session( + user_id=user_id, + title=chat_title, + trigger_metadata={ + "source": "incidentio", + "incident_id": fields["incident_id"], + "incident_name": fields["incident_name"], + "permalink": fields["permalink"], + }, + incident_id=str(incident_id), + ) + + rca_prompt = build_incidentio_rca_prompt(payload, user_id=user_id) + + task = run_background_chat.delay( + user_id=user_id, + session_id=session_id, + initial_message=rca_prompt, + trigger_metadata={ + "source": "incidentio", + "incident_id": fields["incident_id"], + "incident_name": fields["incident_name"], + }, + incident_id=str(incident_id), + ) + + # Store task ID for cancellation support + from utils.db.connection_pool import db_pool + try: + with db_pool.get_admin_connection() as conn: + with conn.cursor() as cursor: + cursor.execute( + "UPDATE incidents SET rca_celery_task_id = %s WHERE id = %s", + (task.id, str(incident_id)), + ) + conn.commit() + except Exception as exc: + logger.warning("[INCIDENTIO] Failed to store RCA task ID for incident %s: %s", incident_id, exc) + + logger.info("[INCIDENTIO] Triggered RCA for incident %s (task=%s)", incident_id, task.id) + + # Post-back RCA summary if enabled + if _should_postback(user_id): + postback_rca_to_incidentio.delay(user_id, str(incident_id), fields["incident_id"]) + + except Exception as exc: + logger.exception("[INCIDENTIO] Failed to trigger RCA: %s", exc) + + +@celery_app.task( + bind=True, max_retries=2, default_retry_delay=120, name="incidentio.postback_rca" +) +def postback_rca_to_incidentio( + self, + user_id: str, + aurora_incident_id: str, + incidentio_incident_id: str, +) -> None: + """Post RCA results back to incident.io timeline once analysis completes.""" + try: + from utils.db.connection_pool import db_pool + from utils.auth.token_management import get_token_data + from routes.incidentio.incidentio_routes import IncidentioClient + + # Wait for RCA to complete — check for summary in incidents table + with db_pool.get_admin_connection() as conn: + with conn.cursor() as cursor: + cursor.execute( + "SELECT aurora_summary, aurora_status FROM incidents WHERE id = %s", + (aurora_incident_id,), + ) + row = cursor.fetchone() + + if not row or not row[0]: + if self.request.retries < self.max_retries: + raise self.retry(countdown=120) + logger.info("[INCIDENTIO] No RCA summary after retries, skipping postback") + return + + summary, aurora_status = row + if aurora_status not in ("analyzed", "completed"): + if self.request.retries < self.max_retries: + raise self.retry(countdown=120) + return + + creds = get_token_data(user_id, "incidentio") + if not creds or not creds.get("api_key"): + logger.warning("[INCIDENTIO] No credentials for postback") + return + + client = IncidentioClient(creds["api_key"]) + message = f"🔍 **Aurora RCA Summary**\n\n{summary}" + client.post_incident_update(incidentio_incident_id, message) + logger.info("[INCIDENTIO] Posted RCA back to incident %s", incidentio_incident_id) + + except Exception as exc: + if "retry" not in str(type(exc).__name__).lower(): + logger.exception("[INCIDENTIO] Postback failed: %s", exc) + raise self.retry(exc=exc) + raise diff --git a/server/routes/incidents_routes.py b/server/routes/incidents_routes.py index 9f273a881..ad081fb43 100644 --- a/server/routes/incidents_routes.py +++ b/server/routes/incidents_routes.py @@ -561,6 +561,19 @@ def get_incident(user_id, incident_id: str): logger.debug("[INCIDENTS] Found OpsGenie/JSM payload for alert") except (ValueError, TypeError): logger.debug("[INCIDENTS] Skipping payload fetch for opsgenie alert (non-integer id)") + elif source_type == "incidentio": + try: + alert_id_int = int(source_alert_id) + cursor.execute( + "SELECT payload FROM incidentio_alerts WHERE id = %s AND org_id = %s", + (alert_id_int, org_id), + ) + alert_row = cursor.fetchone() + if alert_row and alert_row[0] is not None: + raw_payload = alert_row[0] + logger.debug("[INCIDENTS] Found incident.io payload for alert") + except (ValueError, TypeError): + logger.debug("[INCIDENTS] Skipping payload fetch for incidentio alert (non-integer id)") # Log warning if no payload found for any source type if not raw_payload: diff --git a/server/utils/auth/stateless_auth.py b/server/utils/auth/stateless_auth.py index ceaf9ac93..6c7e50232 100644 --- a/server/utils/auth/stateless_auth.py +++ b/server/utils/auth/stateless_auth.py @@ -345,68 +345,63 @@ def store_user_preference(user_id: str, key: str, value: Any): conn = connect_to_db_as_user() cursor = conn.cursor() org_id = set_rls_context(cursor, conn, user_id, log_prefix="[StoreUserPref]") - - if org_id: - cursor.execute(""" - INSERT INTO user_preferences (user_id, org_id, preference_key, preference_value) - VALUES (%s, %s, %s, %s) - ON CONFLICT (user_id, org_id, preference_key) WHERE org_id IS NOT NULL DO UPDATE SET - preference_value = EXCLUDED.preference_value, - updated_at = CURRENT_TIMESTAMP - """, (user_id, org_id, key, json.dumps(value))) - else: - cursor.execute(""" - DELETE FROM user_preferences - WHERE user_id = %s AND org_id IS NULL AND preference_key = %s - """, (user_id, key)) - cursor.execute(""" - INSERT INTO user_preferences (user_id, org_id, preference_key, preference_value) - VALUES (%s, NULL, %s, %s) - """, (user_id, key, json.dumps(value))) + if not org_id: + return + + cursor.execute( + "DELETE FROM user_preferences WHERE org_id = %s AND preference_key = %s", + (org_id, key), + ) + cursor.execute(""" + INSERT INTO user_preferences (user_id, org_id, preference_key, preference_value) + VALUES (%s, %s, %s, %s) + """, (user_id, org_id, key, json.dumps(value))) conn.commit() - logger.debug(f"Stored preference {key} for user {user_id}") - except Exception as e: - logger.error(f"Error storing user preference: {e}") + logger.debug("Stored org preference successfully") + except Exception: + logger.exception("Error storing user preference") finally: if 'cursor' in locals() and cursor: cursor.close() if 'conn' in locals() and conn: conn.close() +def _parse_preference_value(raw, default=None): + """Decode a preference_value column, which may be JSON text or already decoded.""" + if raw is None: + return default + if not isinstance(raw, str): + return raw + try: + return json.loads(raw) + except (json.JSONDecodeError, ValueError): + return raw + def get_user_preference(user_id: str, key: str, default=None): """Get user preference from database.""" + conn = None + cursor = None try: conn = connect_to_db_as_user() cursor = conn.cursor() - set_rls_context(cursor, conn, user_id, log_prefix="[Prefs:get]") - + org_id = set_rls_context(cursor, conn, user_id, log_prefix="[Prefs:get]") + + lookup_col, lookup_val = ("org_id", org_id) if org_id else ("user_id", user_id) cursor.execute( - "SELECT preference_value FROM user_preferences WHERE user_id = %s AND preference_key = %s", - (user_id, key) + f"SELECT preference_value FROM user_preferences WHERE {lookup_col} = %s AND preference_key = %s", + (lookup_val, key), ) result = cursor.fetchone() - if result: - logger.debug(f"Retrieved preference {key} for user {user_id}") - try: - # Try to parse as JSON, but handle cases where it might already be decoded - # Use 'is not None' to handle boolean False values correctly - value = result[0] if result[0] is not None else default - if isinstance(value, str): - return json.loads(value) - return value - except json.JSONDecodeError: - # If JSON parsing fails, return the raw value - return result[0] if result[0] is not None else default - - logger.debug(f"No preference {key} found for user {user_id}, returning default") - return default - except Exception as e: - logger.error(f"Error retrieving user preference: {e}") + if not result: + return default + return _parse_preference_value(result[0], default) + except Exception: + logger.exception("Error retrieving user preference") return default finally: - if 'cursor' in locals() and cursor: + if cursor: cursor.close() - if 'conn' in locals() and conn: + if conn: conn.close() def get_connected_providers(user_id: str) -> List[str]: diff --git a/server/utils/db/db_utils.py b/server/utils/db/db_utils.py index 247e3a535..4877fcab0 100644 --- a/server/utils/db/db_utils.py +++ b/server/utils/db/db_utils.py @@ -823,6 +823,30 @@ def initialize_tables(): CREATE INDEX IF NOT EXISTS idx_splunk_alerts_state ON splunk_alerts(alert_state); CREATE INDEX IF NOT EXISTS idx_splunk_alerts_received_at ON splunk_alerts(received_at DESC); """, + "incidentio_alerts": """ + CREATE TABLE IF NOT EXISTS incidentio_alerts ( + id SERIAL PRIMARY KEY, + user_id VARCHAR(255) NOT NULL, + org_id VARCHAR(255), + incident_id VARCHAR(255), + incident_name TEXT, + incident_status VARCHAR(100), + severity VARCHAR(50), + incident_type VARCHAR(255), + payload JSONB NOT NULL, + received_at TIMESTAMP NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + + CREATE UNIQUE INDEX IF NOT EXISTS idx_incidentio_alerts_org_incident + ON incidentio_alerts(org_id, incident_id); + CREATE INDEX IF NOT EXISTS idx_incidentio_alerts_user_id + ON incidentio_alerts(user_id, received_at DESC); + CREATE INDEX IF NOT EXISTS idx_incidentio_alerts_status + ON incidentio_alerts(incident_status); + CREATE INDEX IF NOT EXISTS idx_incidentio_alerts_severity + ON incidentio_alerts(severity); + """, "jenkins_deployment_events": """ CREATE TABLE IF NOT EXISTS jenkins_deployment_events ( id SERIAL PRIMARY KEY, @@ -990,7 +1014,7 @@ def initialize_tables(): created_at TIMESTAMP DEFAULT NOW(), updated_at TIMESTAMP DEFAULT NOW() ); - + CREATE INDEX IF NOT EXISTS idx_users_email ON users(email); """, "organizations": """ @@ -1179,6 +1203,7 @@ def initialize_tables(): rls_tables.append("netdata_alerts") rls_tables.append("netdata_verification_tokens") rls_tables.append("splunk_alerts") + rls_tables.append("incidentio_alerts") rls_tables.append("bigpanda_events") rls_tables.append("jenkins_deployment_events") rls_tables.append("spinnaker_deployment_events") diff --git a/server/utils/providers.py b/server/utils/providers.py index 4fb16da39..2096c6eb3 100644 --- a/server/utils/providers.py +++ b/server/utils/providers.py @@ -24,6 +24,7 @@ "github", "google_chat", "grafana", + "incidentio", "jenkins", "jira", "netdata", diff --git a/server/utils/secrets/secret_ref_utils.py b/server/utils/secrets/secret_ref_utils.py index b66c16a1e..0ba773511 100644 --- a/server/utils/secrets/secret_ref_utils.py +++ b/server/utils/secrets/secret_ref_utils.py @@ -66,6 +66,7 @@ "newrelic", # New Relic connector tokens "notion", # Notion (documentation platform) "google", # Google Chat — provider is "google_chat", split('_')[0] matches this + "incidentio", # incident.io connector tokens }