diff --git a/.env.example b/.env.example index 5a7a14a37..2d0b84e05 100644 --- a/.env.example +++ b/.env.example @@ -373,3 +373,6 @@ MCP_PORT=8811 # Development # ----------------------------------------------------------------------------- NGROK_URL= + +# Show the initial user RCA message (the auto-generated prompt) in the chat UI. +DISPLAY__RCA_USER_MSG=false diff --git a/client/src/components/tool-calls/ToolExecutionWidget.tsx b/client/src/components/tool-calls/ToolExecutionWidget.tsx index a83667bd1..2441aad0c 100644 --- a/client/src/components/tool-calls/ToolExecutionWidget.tsx +++ b/client/src/components/tool-calls/ToolExecutionWidget.tsx @@ -188,6 +188,18 @@ const ToolExecutionWidget = ({ tool, className, sendMessage, sendRaw, onToolUpda else if (tool.tool_name === "list_slack_channels" || tool.tool_name === "get_channel_history" || tool.tool_name === "get_thread_replies") { command = parseSlackCommand(tool.tool_name, normalizedInput) } + // Alert payload drill-down tool: show the json_path being queried + else if (tool.tool_name === "get_alert_field") { + try { + const parsed = JSON.parse(normalizedInput) + const path = parsed.json_path || parsed.kwargs?.json_path || '' + if (path) { + command = `get_alert_field: ${path}` + } + } catch { + // Keep the precomputed fallback + } + } // If command is still JSON blob, use default if (typeof command === "string" && command.trim().startsWith("{")) { diff --git a/server/chat/backend/agent/tools/alert_payload_tool.py b/server/chat/backend/agent/tools/alert_payload_tool.py new file mode 100644 index 000000000..7e3d90909 --- /dev/null +++ b/server/chat/backend/agent/tools/alert_payload_tool.py @@ -0,0 +1,190 @@ +""" +Alert Payload Drill-Down Tool + +Retrieves full (untruncated) field values from the stored webhook payload. +Used by the RCA agent when the initial prompt contained a truncated payload +and the agent needs to inspect a specific field in full. +""" + +import json +import logging +from datetime import timedelta +from typing import Any, Optional, Tuple + +from pydantic import BaseModel, Field + +logger = logging.getLogger(__name__) + +_SOURCE_TABLE_MAP = { + "grafana": "grafana_alerts", + "datadog": "datadog_events", + "newrelic": "newrelic_events", + "pagerduty": "pagerduty_events", + "opsgenie": "opsgenie_events", + "sentry": "sentry_events", + "splunk": "splunk_alerts", + "dynatrace": "dynatrace_problems", + "bigpanda": "bigpanda_events", + "netdata": "netdata_alerts", + "incidentio": "incidentio_alerts", + "jenkins": "jenkins_deployment_events", + "cloudbees": "jenkins_deployment_events", + "spinnaker": "spinnaker_deployment_events", +} + +_PAYLOAD_LOOKUP_WINDOW = timedelta(minutes=10) + + +class GetAlertFieldArgs(BaseModel): + json_path: str = Field( + description=( + "Dot-separated path to the field in the webhook payload. " + "Use numeric indices for arrays. " + "Examples: 'alerts.0.labels', 'event.incident.summary', 'results.0'" + ) + ) + + +GET_ALERT_FIELD_DESCRIPTION = ( + "Retrieve the full (untruncated) value of a field from the original webhook payload. " + "Use when the RCA prompt shows a truncated field you need to inspect fully. " + "Provide a dot-separated JSON path (e.g. 'event.incident.custom_field_entries', 'alerts.0.annotations')." +) + + +def _validate_inputs(json_path: str, user_id: Optional[str], incident_id: Optional[str]) -> Optional[str]: + """Validate required inputs; returns an error string or None if valid.""" + if not user_id: + return "Error: User authentication required." + if not incident_id: + return "Error: No incident context. This tool is only available during RCA investigations." + if not json_path or not json_path.strip(): + return "Error: json_path is required. Provide a dot-separated path like 'event.incident.summary'." + return None + + +def _fetch_payload(cursor, conn, incident_id: str, user_id: str) -> Tuple[Optional[Any], Optional[str]]: + """Fetch the raw payload for an incident. Returns (payload_dict, error_string).""" + from utils.auth.stateless_auth import set_rls_context + + set_rls_context(cursor, conn, user_id, log_prefix="[GetAlertField]") + + cursor.execute( + "SELECT source_type, source_alert_id, created_at FROM incidents WHERE id = %s", + (incident_id,), + ) + row = cursor.fetchone() + if not row: + return None, f"Error: Incident {incident_id} not found." + + source_type, source_alert_id, incident_created_at = row[0], row[1], row[2] + table = _SOURCE_TABLE_MAP.get(source_type) + if not table: + return None, f"Error: Unknown source type '{source_type}'. Cannot look up payload." + + # Primary: direct ID lookup + cursor.execute( + f"SELECT payload FROM {table} WHERE id = %s", + (source_alert_id,), + ) + payload_row = cursor.fetchone() + + # Fallback: time-window constrained lookup (fail-closed if ambiguous) + if not payload_row or not payload_row[0]: + if incident_created_at: + window_start = incident_created_at - _PAYLOAD_LOOKUP_WINDOW + window_end = incident_created_at + _PAYLOAD_LOOKUP_WINDOW + cursor.execute( + f"SELECT payload FROM {table} " + f"WHERE user_id = %s AND received_at BETWEEN %s AND %s", + (user_id, window_start, window_end), + ) + rows = cursor.fetchall() + if len(rows) == 1: + payload_row = rows[0] + else: + payload_row = None + + if not payload_row or not payload_row[0]: + return None, f"Error: No payload found in {table} for source_alert_id {source_alert_id}." + + payload = payload_row[0] + if isinstance(payload, str): + payload = json.loads(payload) + + return payload, None + + +def _traverse_path(payload: Any, json_path: str) -> Tuple[Optional[Any], Optional[str]]: + """Walk a dot-separated path through the payload. Returns (value, error_string).""" + path_parts = json_path.strip().split(".") + current = payload + for part in path_parts: + if current is None: + return None, f"Error: Path '{json_path}' not found. Value is null at '{part}'." + if isinstance(current, dict): + if part in current: + current = current[part] + else: + available = list(current.keys())[:20] + return None, ( + f"Error: Key '{part}' not found at this level.\n" + f"Available keys: {available}" + ) + elif isinstance(current, list): + try: + idx = int(part) + except ValueError: + return None, f"Error: Expected numeric index for list, got '{part}'." + if 0 <= idx < len(current): + current = current[idx] + else: + return None, f"Error: Index {idx} out of range (list has {len(current)} items)." + else: + return None, f"Error: Cannot traverse into {type(current).__name__} at '{part}'." + return current, None + + +def _format_output(value: Any) -> str: + """Serialize and truncate the extracted value for tool output.""" + from chat.backend.constants import MAX_TOOL_OUTPUT_CHARS + + if isinstance(value, (dict, list)): + result = json.dumps(value, ensure_ascii=False, default=str, indent=2) + else: + result = str(value) if value is not None else "null" + + if len(result) > MAX_TOOL_OUTPUT_CHARS: + result = result[:MAX_TOOL_OUTPUT_CHARS] + "\n... [output truncated]" + return result + + +def get_alert_field( + json_path: str, + user_id: Optional[str] = None, + incident_id: Optional[str] = None, + **kwargs, +) -> str: + """Retrieve a specific field from the stored webhook payload.""" + error = _validate_inputs(json_path, user_id, incident_id) + if error: + return error + + from utils.db.connection_pool import db_pool + + try: + with db_pool.get_admin_connection() as conn: + with conn.cursor() as cursor: + payload, err = _fetch_payload(cursor, conn, incident_id, user_id) + if err: + return err + + value, err = _traverse_path(payload, json_path) + if err: + return err + + return _format_output(value) + + except Exception as e: + logger.exception("[GetAlertField] Error retrieving field: %s", e) + return f"Error retrieving alert field: {e}" diff --git a/server/chat/backend/agent/tools/cloud_tools.py b/server/chat/backend/agent/tools/cloud_tools.py index 33fac03cc..f978d9842 100644 --- a/server/chat/backend/agent/tools/cloud_tools.py +++ b/server/chat/backend/agent/tools/cloud_tools.py @@ -2366,6 +2366,24 @@ def _pinned_trigger(action_id: str = "", _pid=pinned_id, _fn=final_func, **kw): except Exception as e: logging.warning(f"Failed to add Cloudflare tools (treating as not connected): {e}") + # Add alert payload drill-down tool for RCA sessions with an incident + incident_id = getattr(state_context, 'incident_id', None) if state_context else None + if incident_id and is_background: + try: + from .alert_payload_tool import get_alert_field, GetAlertFieldArgs, GET_ALERT_FIELD_DESCRIPTION + _ctx = with_forced_context(get_alert_field) + _notif = with_completion_notification(_ctx) + _final = wrap_func_with_capture(_notif, "get_alert_field") if tool_capture else _notif + tools.append(StructuredTool.from_function( + func=_final, + name="get_alert_field", + description=GET_ALERT_FIELD_DESCRIPTION, + args_schema=GetAlertFieldArgs, + )) + logging.info(f"Added get_alert_field tool for incident {incident_id}") + except Exception as e: + logging.warning(f"Failed to add get_alert_field tool: {e}") + logging.info(f"Created {len(tools)} Aurora native tools") # Add real MCP tools if available diff --git a/server/chat/backend/agent/tools/output_sanitizer.py b/server/chat/backend/agent/tools/output_sanitizer.py index e13947098..714dbbfe4 100644 --- a/server/chat/backend/agent/tools/output_sanitizer.py +++ b/server/chat/backend/agent/tools/output_sanitizer.py @@ -9,11 +9,24 @@ logger = logging.getLogger(__name__) -def truncate_json_fields(data, max_field_length=10000): +def truncate_json_fields(data, max_field_length=10000, max_depth=None, _current_depth=0): """ Recursively truncate string fields in JSON data while preserving the JSON structure. Only truncates individual string values, not the entire JSON object. + When max_depth is set, nested structures beyond that depth are replaced with summaries. """ + if max_depth is not None and _current_depth >= max_depth: + if isinstance(data, dict): + return f"{{object, {len(data)} keys: {', '.join(list(data.keys())[:5])}{', ...' if len(data) > 5 else ''}}}" + elif isinstance(data, list): + return f"[array, {len(data)} items]" + elif isinstance(data, str): + if len(data) > max_field_length: + return data[:max_field_length] + "... [field truncated]" + return data + else: + return data + if isinstance(data, str): if len(data) > max_field_length: return data[:max_field_length] + "... [field truncated]" @@ -21,14 +34,13 @@ def truncate_json_fields(data, max_field_length=10000): elif isinstance(data, dict): truncated_dict = {} for key, value in data.items(): - # Truncate the key if it's too long safe_key = str(key) if key is not None else "null_key" - if len(safe_key) > 200: # Reasonable limit for keys + if len(safe_key) > 200: safe_key = safe_key[:200] + "..." - truncated_dict[safe_key] = truncate_json_fields(value, max_field_length) + truncated_dict[safe_key] = truncate_json_fields(value, max_field_length, max_depth, _current_depth + 1) return truncated_dict elif isinstance(data, list): - return [truncate_json_fields(item, max_field_length) for item in data] + return [truncate_json_fields(item, max_field_length, max_depth, _current_depth + 1) for item in data] else: return data diff --git a/server/chat/backend/agent/tools/trigger_rca_tool.py b/server/chat/backend/agent/tools/trigger_rca_tool.py index 47ddd5fd1..a4af5310c 100644 --- a/server/chat/backend/agent/tools/trigger_rca_tool.py +++ b/server/chat/backend/agent/tools/trigger_rca_tool.py @@ -181,7 +181,7 @@ def trigger_rca( create_background_chat_session, run_background_chat, ) - from chat.background.rca_prompt_builder import build_chat_rca_prompt + from chat.background.rca_prompt_builder import build_rca_prompt trigger_metadata = {"source": "chat", "incident_id": incident_id} @@ -191,9 +191,17 @@ def trigger_rca( trigger_metadata=trigger_metadata, incident_id=incident_id, ) - rca_prompt, rail_text = build_chat_rca_prompt( - description=issue_description, title=incident_title, - service=service, severity=severity, user_id=user_id, + # Small enough to always pass verbatim (no truncation/get_alert_field needed) + payload: dict = { + "title": incident_title, + "status": "investigating", + "description": issue_description, + "service": service, + "severity": severity, + } + + rca_prompt, rail_text = build_rca_prompt( + "chat", incident_title, payload, user_id=user_id, ) task = run_background_chat.delay( diff --git a/server/chat/background/rca_prompt_builder.py b/server/chat/background/rca_prompt_builder.py index aab26f6b0..104d11d2d 100644 --- a/server/chat/background/rca_prompt_builder.py +++ b/server/chat/background/rca_prompt_builder.py @@ -1,9 +1,10 @@ """ -Shared RCA (Root Cause Analysis) prompt builder for background alert processing. +RCA (Root Cause Analysis) prompt builder for background alert processing. -This module creates provider-aware RCA prompts that pair alert data with -infrastructure context. Behavioral investigation guidance lives in the system -prompt (rca_sections/). The user message contains only alert facts and context. +build_rca_prompt() is the single entry point for all RCA prompt +construction — both webhook-triggered and user-initiated (chat) RCAs. +It passes the raw payload directly to the LLM, with conditional truncation +for large payloads. Aurora Learn Integration: - When Aurora Learn is enabled, searches for similar past incidents with positive feedback @@ -11,6 +12,7 @@ """ from typing import Any, Dict, List, Optional +import json import logging logger = logging.getLogger(__name__) @@ -296,272 +298,119 @@ def get_user_providers(user_id: str) -> List[str]: return result -def _build_provider_investigation_section(providers: List[str], user_id: Optional[str] = None) -> str: - """Provider investigation now loaded from skills/rca/ files.""" - return "" - -def _get_github_connected(user_id: str) -> bool: - """Return True when the user has a usable GitHub credential. - - Checks both auth paths: an active App installation OR a stored OAuth - token (hybrid deployments may have either or both). Without the OAuth - branch, OAuth-only users would have GitHub-investigation guidance - suppressed in the RCA prompt even though their connection works. - """ - try: - from utils.auth.github_auth_mode import is_oauth_enabled - from utils.auth.github_auth_router import _lookup_any_active_installation - if _lookup_any_active_installation(user_id) is not None: - return True - if is_oauth_enabled(): - from utils.auth.token_management import get_token_data - creds = get_token_data(user_id, "github") - if creds and creds.get("access_token"): - return True - return False - except Exception as e: - logger.warning(f"Error checking GitHub connection for user {user_id}: {e}") - return False +# ============================================================================ +# Unified Raw Payload RCA Prompt Builder +# ============================================================================ +PAYLOAD_CHAR_THRESHOLD = 1_000 +CHAT_PAYLOAD_MAX =60_000 -def _has_jenkins_connected(user_id: str) -> bool: - """Check if user has Jenkins connected.""" - try: - from utils.auth.token_management import get_token_data - creds = get_token_data(user_id, "jenkins") - return bool(creds and creds.get("base_url")) - except Exception as e: - logger.warning(f"Error checking Jenkins context: {e}") - return False - +def _extract_rail_text_from_payload(payload: Dict[str, Any]) -> str: + """Extract attacker-controllable text from a raw payload for guardrail evaluation.""" + _RAIL_FIELDS = { + 'title', 'message', 'body', 'description', 'text', 'summary', + 'alert_title', 'event_title', 'rulename', 'name', 'condition_name', + } + parts: List[str] = [] -def _has_cloudbees_connected(user_id: str) -> bool: - """Check if user has CloudBees CI connected.""" - try: - from utils.auth.token_management import get_token_data - creds = get_token_data(user_id, "cloudbees") - return bool(creds and creds.get("base_url")) - except Exception as e: - logger.warning(f"Error checking CloudBees context: {e}") - return False + def _collect(obj: Any, depth: int = 0) -> None: + if depth > 2: + return + if isinstance(obj, dict): + for key, val in obj.items(): + if isinstance(val, str) and key.lower().rstrip('_') in _RAIL_FIELDS: + stripped = val.strip() + if stripped: + parts.append(stripped) + elif isinstance(val, (dict, list)): + _collect(val, depth + 1) + elif isinstance(obj, list): + for item in obj[:5]: + _collect(item, depth + 1) + + _collect(payload) + combined = "\n\n".join(parts) + return combined[:3000] -def _has_jira_connected(user_id: str) -> bool: - """Check if user has Jira connected and the feature flag is enabled.""" - try: - from utils.flags.feature_flags import is_jira_enabled - if not is_jira_enabled(): - return False - from utils.auth.token_management import get_token_data - creds = get_token_data(user_id, "jira") - return bool(creds and (creds.get("access_token") or creds.get("pat_token"))) - except Exception as e: - logger.warning(f"Error checking Jira context: {e}") - return False +def build_rca_prompt( + source: str, + title: str, + payload: Dict[str, Any], + user_id: Optional[str] = None, +) -> tuple[str, str]: + """Build an RCA prompt by passing the raw payload directly to the LLM. + Instead of manually extracting fields, we pass the raw JSON so the LLM + parses it directly. Payloads under PAYLOAD_CHAR_THRESHOLD are passed + verbatim; larger ones get per-field truncation so the agent can drill + down via the get_alert_field tool. -def _has_confluence_connected(user_id: str) -> bool: - """Check if user has Confluence connected and the feature flag is enabled.""" - try: - from utils.flags.feature_flags import is_confluence_enabled - if not is_confluence_enabled(): - return False - from utils.auth.token_management import get_token_data - creds = get_token_data(user_id, "confluence") - return bool(creds and (creds.get("access_token") or creds.get("pat_token"))) - except Exception as e: - logger.warning(f"Error checking Confluence context: {e}") - return False + Args: + source: Provider name (grafana, datadog, incidentio, chat, etc.) + title: Alert title (already extracted by the caller for incident creation) + payload: The raw webhook payload dict (or synthetic payload for chat RCAs) + user_id: For provider lookup, Aurora Learn, and prediscovery context + Returns: + (prompt, rail_text) tuple + """ + from chat.backend.agent.tools.output_sanitizer import truncate_json_fields -def _get_recent_jenkins_deployments(user_id: str, service: str = "", lookback_minutes: int = 60, provider: str = "") -> List[Dict[str, Any]]: - """Query jenkins_deployment_events for recent deployments matching a service. + providers = get_user_providers(user_id) if user_id else [] - Used to inject deployment context into ANY RCA prompt (not just Jenkins-sourced). - """ - if not user_id: - return [] - lookback_minutes = max(1, min(int(lookback_minutes), 10080)) # 1 min to 7 days try: - from utils.db.connection_pool import db_pool - from utils.auth.stateless_auth import set_rls_context - with db_pool.get_admin_connection() as conn: - with conn.cursor() as cursor: - set_rls_context(cursor, conn, user_id, log_prefix="[RCAPrompt:_get_recent_jenkins_deployments]") - conditions = ["user_id = %s", "received_at >= NOW() - make_interval(mins => %s)"] - params: list = [user_id, lookback_minutes] - - if service and service != "unknown": - conditions.append("service = %s") - params.append(service) - - if provider: - conditions.append("provider = %s") - params.append(provider) - - where = " AND ".join(conditions) - cursor.execute( - f"""SELECT service, environment, result, build_number, build_url, - commit_sha, branch, deployer, trace_id, received_at - FROM jenkins_deployment_events - WHERE {where} - ORDER BY received_at DESC LIMIT 5""", - tuple(params), - ) - rows = cursor.fetchall() - return [ - { - "service": r[0], "environment": r[1], "result": r[2], - "build_number": r[3], "build_url": r[4], "commit_sha": r[5] or "", - "branch": r[6], "deployer": r[7], "trace_id": r[8], - "webhook_received_at": r[9].isoformat() if r[9] else None, - } - for r in rows - ] + serialized = json.dumps(payload, ensure_ascii=False, default=str) + payload_size = len(serialized) + + if source == "chat": + if payload_size > CHAT_PAYLOAD_MAX: + json_content = serialized[:CHAT_PAYLOAD_MAX] + "\n... [message truncated]" + else: + json_content = serialized + truncation_note = "" + elif payload_size <= PAYLOAD_CHAR_THRESHOLD: + json_content = serialized + truncation_note = "" + else: + truncated = truncate_json_fields(payload, max_field_length=250) + json_content = json.dumps(truncated, ensure_ascii=False, default=str) + if len(json_content) > 15_000: + truncated = truncate_json_fields(payload, max_field_length=80, max_depth=1) + json_content = json.dumps(truncated, indent=2, ensure_ascii=False, default=str) + truncation_note = ( + "Fields ending with '... [field truncated]' were too long to include in full. " + "`get_alert_field` tool for fields that show this marker if you need to inspect them. " + ) except Exception as e: - logger.warning(f"Error fetching recent Jenkins deployments: {e}") - return [] - + logger.warning(f"Failed to serialize alert payload: {e}") + json_content = f"[Payload could not be serialized — use get_alert_field to inspect fields. Keys: {list(payload.keys())[:20]}]" + truncation_note = "" -def build_rca_prompt( - source: str, - alert_details: Dict[str, Any], - providers: Optional[List[str]] = None, - user_id: Optional[str] = None, - integrations: Optional[Dict[str, bool]] = None, -) -> tuple[str, str]: - """Build a comprehensive, provider-aware RCA prompt. - - Returns: - (prompt, rail_text) tuple where ``prompt`` is the full synthesized - RCA instruction scaffold sent to the agent as the initial message, - and ``rail_text`` is the webhook-authored subset that input guardrails - should evaluate for prompt injection. Callers must forward rail_text - into ``run_background_chat`` via the ``rail_text`` parameter. - """ - # Fetch providers if not provided - if not providers and user_id: - providers = get_user_providers(user_id) - - providers = providers or [] - providers_lower = [p.lower() for p in providers] - - # Derive integrations from skill registry when not passed by caller - if integrations is None and user_id: - try: - from chat.backend.agent.skills.registry import SkillRegistry - registry = SkillRegistry.get_instance() - connected_ids = registry.get_connected_skill_ids(user_id) - integrations = {sid: True for sid in connected_ids} - except Exception: - integrations = {} - - # Extract alert service name early — used by multiple sections below. - # Start with the explicit service label, then try richer fallbacks from - # the alert message (Condition/Policy/Targets fields) so downstream - # consumers (Jenkins deploy lookup, Aurora Learn, prediscovery) get the - # same specificity as the Jira search term. - _label_service = alert_details.get('labels', {}).get('service', '') - alert_service = _label_service if _label_service and _label_service != 'unknown' else '' - if not alert_service and source == 'netdata': - alert_service = alert_details.get('host', '') or '' - - # Format alert details - title = alert_details.get('title', 'Unknown Alert') - status = alert_details.get('status', 'unknown') - labels = alert_details.get('labels', {}) - message = alert_details.get('message', '') - values = alert_details.get('values', '') - - # If alert_service is still empty, try to extract a meaningful service/component - # name from the message (same heuristic the Jira search uses). - _generic_titles = { - 'new relic alert', 'unknown alert', 'alert', 'unknown', - 'grafana alert', 'datadog alert', 'splunk alert', - } - if not alert_service: - _candidate = title if title.lower().strip() not in _generic_titles else '' - if not _candidate: - _msg = message or '' - for _part in _msg.replace('.', ',').split(','): - _part = _part.strip() - for _prefix in ('Condition:', 'Targets:', 'Entities:', 'Policy:', 'Search:'): - if _part.startswith(_prefix): - _candidate = _part[len(_prefix):].strip() - break - if _candidate: - break - if _candidate: - alert_service = _candidate - - # Source-specific labels formatting - if source == 'grafana': - labels_str = ", ".join(f"{k}={v}" for k, v in labels.items()) if labels else "none" - elif source == 'datadog': - tags = alert_details.get('tags', []) - labels_str = ", ".join(tags[:10]) if tags else "none" - elif source == 'netdata': - host = alert_details.get('host', 'unknown') - chart = alert_details.get('chart', 'unknown') - labels_str = f"host={host}, chart={chart}" - elif source == 'pagerduty': - labels_str = ", ".join(f"{k}={v}" for k, v in labels.items()) if labels else "none" - elif source == 'splunk': - labels_str = ", ".join(f"{k}={v}" for k, v in labels.items()) if labels else "none" - elif source == 'dynatrace': - entity = alert_details.get('impacted_entity', 'unknown') - impact = alert_details.get('impact', 'unknown') - labels_str = f"entity={entity}, impact={impact}" - elif source == 'bigpanda': - labels_str = ", ".join(f"{k}={v}" for k, v in labels.items()) if labels else "none" - elif source == 'newrelic': - labels_str = ", ".join(f"{k}={v}" for k, v in labels.items()) if labels else "none" - elif source == 'chat': - labels_str = ", ".join(f"{k}={v}" for k, v in labels.items()) if labels else "user-reported" - elif source == 'opsgenie': - tags = alert_details.get('tags', []) - labels_str = ", ".join(tags[:10]) if tags else "none" - else: - labels_str = str(labels) - - # Build the prompt prompt_parts = [ f"# ROOT CAUSE ANALYSIS REQUIRED - {source.upper()} ALERT", "", - "## ALERT DETAILS:", - f"- **Title**: {title}", - f"- **Status**: {status}", - f"- **Source**: {source}", - f"- **Labels/Tags**: {labels_str}", - ] - - if message: - prompt_parts.append(f"- **Message**: {message}") - if values: - prompt_parts.append(f"- **Values**: {values}") - if source == 'datadog' and 'monitor_id' in alert_details: - prompt_parts.append(f"- **Monitor ID**: {alert_details['monitor_id']}") - if source == 'pagerduty': - if 'incident_id' in alert_details: - prompt_parts.append(f"- **Incident ID**: {alert_details['incident_id']}") - if 'incident_url' in alert_details: - prompt_parts.append(f"- **Incident URL**: {alert_details['incident_url']}") - if source == 'netdata': - prompt_parts.append(f"- **Host**: {alert_details.get('host', 'unknown')}") - prompt_parts.append(f"- **Chart**: {alert_details.get('chart', 'unknown')}") - if source == 'newrelic': - if 'issueUrl' in alert_details: - prompt_parts.append(f"- **Issue URL**: {alert_details['issueUrl']}") - - # providers list is already verified by get_user_providers() — only - # cloud providers with valid role-auth + SkillRegistry-validated integrations. - prompt_parts.extend([ + f"## ALERT: {title}", "", - "## CONNECTED INFRASTRUCTURE & MONITORING:", + "## CONNECTED INFRASTRUCTURE:", f"You have access to: {', '.join(providers) if providers else 'No cloud/monitoring providers connected'}", - ]) + "", + "## WEBHOOK PAYLOAD:", + truncation_note + "", + json_content, + "", + ] - # Aurora Learn: Inject context from similar past incidents + # Aurora Learn: inject context from similar past incidents + metadata = payload.get("metadata") + metadata_service = metadata.get("service", "") if isinstance(metadata, dict) else "" + alert_service = ( + payload.get("service") + or payload.get("resource") + or payload.get("component") + or metadata_service + or "" + ) if user_id: similar_context = _get_similar_good_rcas_context( user_id=user_id, @@ -572,7 +421,7 @@ def build_rca_prompt( if similar_context: prompt_parts.append(similar_context) - # Prediscovery: Inject infrastructure topology context + # Prediscovery: inject infrastructure topology context if user_id: prediscovery_context = _get_prediscovery_context( user_id=user_id, @@ -582,710 +431,7 @@ def build_rca_prompt( if prediscovery_context: prompt_parts.append(prediscovery_context) - # User message contains only alert data + context. - # All behavioral guidance lives in the system prompt (rca_sections/). - return "\n".join(prompt_parts), build_alert_rail_text(alert_details) - - -def _format_cloudwatch_dimensions(dimensions: list) -> str: - """Format CloudWatch trigger dimensions into a comma-separated string.""" - parts = [] - for d in dimensions: - if not isinstance(d, dict): - continue - name = d.get('name') or d.get('Name') - value = d.get('value') or d.get('Value') - if name and value: - parts.append(f"{name}={value}") - return ", ".join(parts) - - -def _build_cloudwatch_message(namespace: str, metric_name: str, reason: str, dim_str: str) -> str: - """Compose the CloudWatch alert message from metric info and reason.""" - message = reason - if namespace and metric_name: - message = f"Metric: {namespace}/{metric_name}. {reason}" - if dim_str: - message += f" Dimensions: {dim_str}." - return message.strip() - - -def build_cloudwatch_rca_prompt( - payload: Dict[str, Any], - providers: Optional[List[str]] = None, - user_id: Optional[str] = None, -) -> tuple[str, str]: - """Build RCA prompt from a CloudWatch alarm payload.""" - alarm_name = payload.get("AlarmName") or "Unknown Alarm" - state_value = payload.get("NewStateValue") or payload.get("state_value") or "ALARM" - reason = payload.get("NewStateReason") or payload.get("reason") or "" - - trigger = payload.get("Trigger") or {} - namespace = trigger.get("Namespace") or "" - metric_name = trigger.get("MetricName") or "" - dim_str = _format_cloudwatch_dimensions(trigger.get("Dimensions") or []) - message = _build_cloudwatch_message(namespace, metric_name, reason, dim_str) - - alert_details = { - "title": alarm_name, - "status": state_value, - "message": message, - "namespace": namespace, - "metric_name": metric_name, - } - - return build_rca_prompt("cloudwatch", alert_details, providers, user_id) - - -def build_grafana_rca_prompt( - payload: Dict[str, Any], - providers: Optional[List[str]] = None, - user_id: Optional[str] = None, -) -> tuple[str, str]: - """Build RCA prompt from Grafana alert payload.""" - title = payload.get("title") or payload.get("ruleName") or "Unknown Alert" - status = payload.get("state") or payload.get("status") or "unknown" - message = payload.get("message") or payload.get("annotations", {}).get("description") or "" - labels = payload.get("commonLabels", {}) or payload.get("labels", {}) - - values = payload.get("values") or payload.get("evalMatches", []) - values_str = "" - if values: - if isinstance(values, list): - values_str = ", ".join(str(v) for v in values[:5]) - elif isinstance(values, dict): - values_str = ", ".join(f"{k}: {v}" for k, v in list(values.items())[:5]) - - alert_details = { - 'title': title, - 'status': status, - 'message': message, - 'labels': labels, - 'values': values_str, - } - - return build_rca_prompt('grafana', alert_details, providers, user_id) - - -def build_datadog_rca_prompt( - payload: Dict[str, Any], - providers: Optional[List[str]] = None, - user_id: Optional[str] = None, -) -> tuple[str, str]: - """Build RCA prompt from Datadog alert payload.""" - title = payload.get("title") or payload.get("event_title") or payload.get("event", {}).get("title") or "Unknown Alert" - status = payload.get("status") or payload.get("state") or payload.get("alert_type") or "unknown" - event_type = payload.get("event_type") or payload.get("alert_type") or "unknown" - scope = payload.get("scope") or payload.get("event", {}).get("scope") or "none" - tags = payload.get("tags", []) - monitor_id = payload.get("monitor_id") or payload.get("alert_id") or "unknown" - message = payload.get("body") or payload.get("message") or payload.get("event", {}).get("text") or "" - - alert_details = { - 'title': title, - 'status': f"{status} ({event_type})", - 'message': message, - 'tags': tags, - 'monitor_id': monitor_id, - 'scope': scope, - } - - return build_rca_prompt('datadog', alert_details, providers, user_id) - - -def build_dynatrace_rca_prompt( - payload: Dict[str, Any], - providers: Optional[List[str]] = None, - user_id: Optional[str] = None, -) -> tuple[str, str]: - """Build RCA prompt from Dynatrace problem notification payload.""" - title = payload.get("ProblemTitle") or "Unknown Problem" - impact = payload.get("ProblemImpact") or "unknown" - entity = payload.get("ImpactedEntity") or "unknown" - problem_url = payload.get("ProblemURL") or "" - tags = payload.get("Tags") or "" - - alert_details = { - 'title': title, - 'status': payload.get("State", "OPEN"), - 'message': f"Impact: {impact}. Entity: {entity}", - 'labels': {}, - 'impacted_entity': entity, - 'impact': impact, - } - if problem_url: - alert_details['problemUrl'] = problem_url - if tags: - alert_details['tags'] = tags - - return build_rca_prompt('dynatrace', alert_details, providers, user_id) - - -def build_netdata_rca_prompt( - payload: Dict[str, Any], - providers: Optional[List[str]] = None, - user_id: Optional[str] = None, -) -> tuple[str, str]: - """Build RCA prompt from Netdata alert payload.""" - alarm = payload.get("name") or payload.get("alarm") or payload.get("title") or "Unknown Alert" - status = payload.get("status") or "unknown" - host = payload.get("host") or "unknown" - chart = payload.get("chart") or "unknown" - alert_class = payload.get("class") or "unknown" - family = payload.get("family") or "unknown" - space = payload.get("space") or "unknown" - room = payload.get("room") or "unknown" - value = payload.get("value") - message = payload.get("message") or payload.get("info") or "" - - values_str = str(value) if value is not None else "" - - alert_details = { - 'title': alarm, - 'status': status, - 'message': message, - 'host': host, - 'chart': chart, - 'labels': { - 'class': alert_class, - 'family': family, - 'space': space, - 'room': room, - }, - 'values': values_str, - } - - return build_rca_prompt('netdata', alert_details, providers, user_id) - - -def build_pagerduty_rca_prompt( - incident: Dict[str, Any], - providers: Optional[List[str]] = None, - user_id: Optional[str] = None, -) -> tuple[str, str]: - """Build RCA prompt from PagerDuty V3 incident data.""" - title = incident.get("title", "Untitled Incident") - incident_number = incident.get("number", "unknown") - incident_id = incident.get("id", "unknown") - status = incident.get("status", "unknown") - urgency = incident.get("urgency", "unknown") - - # Service information - service = incident.get("service", {}) - service_name = service.get("summary", "unknown") if isinstance(service, dict) else "unknown" - - # Priority information - priority = incident.get("priority", {}) - priority_name = priority.get("summary") or priority.get("name", "none") if isinstance(priority, dict) else "none" - - # Description - description = incident.get("body", {}).get("details", "") - - # HTML URL - html_url = incident.get("html_url", "") - - # Incident key - incident_key = incident.get("incident_key", "") - - # Build alert details for the unified prompt builder - alert_details = { - 'title': f"#{incident_number}: {title}", - 'status': f"{status} (urgency: {urgency})", - 'message': description, - 'labels': { - 'incident_id': incident_id, - 'incident_number': str(incident_number), - 'urgency': urgency, - 'priority': priority_name, - 'service': service_name, - }, - 'incident_url': html_url, - 'incident_id': incident_id, - } - - if incident_key: - alert_details['labels']['incident_key'] = incident_key - - # Add escalation policy - if escalation_policy := incident.get("escalation_policy", {}): - if isinstance(escalation_policy, dict): - ep_name = escalation_policy.get("summary") or escalation_policy.get("name", "") - if ep_name: - alert_details['labels']['escalation_policy'] = ep_name - - # Add assignments - if assignments := incident.get("assignments", []): - if isinstance(assignments, list) and assignments: - assignees = [] - for assignment in assignments[:3]: - if isinstance(assignment, dict): - assignee = assignment.get("assignee", {}) - if isinstance(assignee, dict): - assignee_name = assignee.get("summary") or assignee.get("name", "") - if assignee_name: - assignees.append(assignee_name) - if assignees: - alert_details['labels']['assigned_to'] = ', '.join(assignees) - - # Add teams - if teams := incident.get("teams", []): - if isinstance(teams, list) and teams: - team_names = [] - for team in teams[:3]: - if isinstance(team, dict): - team_name = team.get("summary") or team.get("name", "") - if team_name: - team_names.append(team_name) - if team_names: - alert_details['labels']['teams'] = ', '.join(team_names) - - # Add custom fields - if custom_fields := incident.get("customFields", {}): - if isinstance(custom_fields, dict) and custom_fields: - for field_name, field_value in custom_fields.items(): - alert_details['labels'][f"custom_{field_name}"] = str(field_value) - - return build_rca_prompt('pagerduty', alert_details, providers, user_id) - - -def build_jenkins_rca_prompt( - payload: Dict[str, Any], - providers: Optional[List[str]] = None, - user_id: Optional[str] = None, -) -> tuple[str, str]: - """Build RCA prompt from a Jenkins deployment failure event.""" - service = payload.get("service") or payload.get("job_name") or "Unknown Service" - result = payload.get("result", "FAILURE") - environment = payload.get("environment", "unknown") - git = payload.get("git", {}) - - alert_details = { - 'title': f"Jenkins Deployment {result}: {service}", - 'status': result, - 'message': f"Build #{payload.get('build_number', '?')} deployed to {environment}", - 'labels': { - 'service': service, - 'environment': environment, - 'deployer': payload.get('deployer', ''), - }, - } - - if git.get("commit_sha"): - alert_details['labels']['commit'] = git['commit_sha'] - if git.get("branch"): - alert_details['labels']['branch'] = git['branch'] - if payload.get("trace_id"): - alert_details['labels']['trace_id'] = payload['trace_id'] - - return build_rca_prompt('jenkins', alert_details, providers, user_id) - - -def build_cloudbees_rca_prompt( - payload: Dict[str, Any], - providers: Optional[List[str]] = None, - user_id: Optional[str] = None, -) -> tuple[str, str]: - """Build RCA prompt from a CloudBees CI deployment failure event.""" - service = payload.get("service") or payload.get("job_name") or "Unknown Service" - result = payload.get("result", "FAILURE") - environment = payload.get("environment", "unknown") - git = payload.get("git", {}) - - alert_details = { - 'title': f"CloudBees CI Deployment {result}: {service}", - 'status': result, - 'message': f"Build #{payload.get('build_number', '?')} deployed to {environment}", - 'labels': { - 'service': service, - 'environment': environment, - 'deployer': payload.get('deployer', ''), - }, - } - - if git.get("commit_sha"): - alert_details['labels']['commit'] = git['commit_sha'] - if git.get("branch"): - alert_details['labels']['branch'] = git['branch'] - if payload.get("trace_id"): - alert_details['labels']['trace_id'] = payload['trace_id'] - - return build_rca_prompt('cloudbees', alert_details, providers, user_id) - - -def build_spinnaker_rca_prompt( - payload: Dict[str, Any], - providers: Optional[List[str]] = None, - user_id: Optional[str] = None, -) -> tuple[str, str]: - """Build RCA prompt from a Spinnaker pipeline failure event.""" - application = payload.get("application") or "Unknown Application" - pipeline_name = payload.get("pipeline_name") or payload.get("pipeline", "Unknown Pipeline") - status = payload.get("status", "TERMINAL") - trigger_type = payload.get("trigger_type", "unknown") - trigger_user = payload.get("trigger_user", "unknown") - - alert_details = { - 'title': f"Spinnaker Pipeline {status}: {application}/{pipeline_name}", - 'status': status, - 'message': f"Pipeline '{pipeline_name}' for application '{application}' ended with status {status}", - 'labels': { - 'service': application, - 'pipeline': pipeline_name, - 'trigger_type': trigger_type, - 'trigger_user': trigger_user, - }, - } - - execution_id = payload.get("execution_id") - if execution_id: - alert_details['labels']['execution_id'] = execution_id - - return build_rca_prompt('spinnaker', alert_details, providers, user_id) - - -def build_bigpanda_rca_prompt( - incident: Dict[str, Any], - alerts: list, - providers: Optional[List[str]] = None, - user_id: Optional[str] = None, -) -> tuple[str, str]: - """Build RCA prompt from BigPanda incident payload.""" - first_alert = alerts[0] if alerts else {} - title = ( - first_alert.get("description") - or first_alert.get("condition_name") - or f"BigPanda Incident {incident.get('id', 'unknown')}" - ) - service = str( - first_alert.get("primary_property") - or first_alert.get("source_system") - or "unknown" - ) - bp_status = incident.get("status", "active") - - message_parts = [f"Child alerts: {len(alerts)}"] - if envs := incident.get("environments"): - message_parts.append(f"Environments: {envs}") - if tags := incident.get("incident_tags"): - message_parts.append(f"Tags: {tags}") - if alerts: - summaries = [] - for a in alerts[:5]: - desc = a.get("description") or a.get("condition_name") or "no description" - src = a.get("source_system") or "unknown" - summaries.append(f"[{src}] {desc}") - message_parts.append("Top alerts: " + "; ".join(summaries)) - - alert_details = { - 'title': title, - 'status': bp_status, - 'message': ". ".join(message_parts), - 'labels': { - 'service': service, - 'severity': incident.get("severity", "unknown"), - 'child_alert_count': str(len(alerts)), - }, - } - - return build_rca_prompt('bigpanda', alert_details, providers, user_id) - - -def build_splunk_rca_prompt( - payload: Dict[str, Any], - providers: Optional[List[str]] = None, - user_id: Optional[str] = None, -) -> tuple[str, str]: - """Build RCA prompt from Splunk alert payload.""" - search_name = payload.get("search_name") or payload.get("name") or "Unknown Alert" - result_count = payload.get("result_count") or payload.get("results_count") or 0 - search_query = payload.get("search") or payload.get("search_query") or "" - app = payload.get("app") or payload.get("source") or "" - severity = payload.get("severity") or payload.get("alert_severity") or "" - - results = payload.get("results") or payload.get("result") or [] - results_str = "" - if results: - if isinstance(results, list): - results_str = ", ".join(str(r) for r in results[:5]) - elif isinstance(results, dict): - results_str = str(results) - - message_parts = [f"Search: {search_name}", f"Result count: {result_count}"] - if search_query: - message_parts.append(f"SPL: {search_query}") - if results_str: - message_parts.append(f"Sample: {results_str}") - - alert_details = { - 'title': search_name, - 'status': f"triggered ({result_count} results)", - 'message': ". ".join(message_parts), - 'labels': {}, - } - - if app: - alert_details['labels']['app'] = app - if severity: - alert_details['labels']['severity'] = str(severity) - - return build_rca_prompt('splunk', alert_details, providers, user_id) - - -def build_newrelic_rca_prompt( - payload: Dict[str, Any], - providers: Optional[List[str]] = None, - user_id: Optional[str] = None, -) -> tuple[str, str]: - """Build RCA prompt from New Relic alert/issue webhook payload.""" - from routes.newrelic.tasks import extract_newrelic_title - title = extract_newrelic_title(payload) - state = payload.get("state") or payload.get("currentState") or payload.get("current_state") or "unknown" - priority = payload.get("priority") or payload.get("severity") or "unknown" - condition_name = payload.get("conditionName") or payload.get("condition_name") or "" - policy_name = payload.get("policyName") or payload.get("policy_name") or "" - issue_url = payload.get("issueUrl") or payload.get("violationChartUrl") or payload.get("incident_url") or "" - account_id = payload.get("accountId") or payload.get("account_id") or "" - - entities = payload.get("entitiesData", {}).get("entities", []) - entity_names = [e.get("name", "unknown") for e in entities[:5]] if entities else [] - targets = payload.get("targets", []) - target_names = [t.get("name", "unknown") for t in targets[:5]] if targets else [] - - details = payload.get("details") or "" - - message_parts = [] - if condition_name: - message_parts.append(f"Condition: {condition_name}") - if policy_name: - message_parts.append(f"Policy: {policy_name}") - if entity_names: - message_parts.append(f"Entities: {', '.join(entity_names)}") - elif target_names: - message_parts.append(f"Targets: {', '.join(target_names)}") - if payload.get("totalIncidents"): - message_parts.append(f"Total incidents: {payload['totalIncidents']}") - if details: - message_parts.append(f"Details: {details[:500]}") - - labels: Dict[str, str] = {} - if priority and priority != "unknown": - labels["priority"] = priority - if account_id: - labels["accountId"] = str(account_id) - - alert_details = { - 'title': title, - 'status': f"{state} (priority: {priority})", - 'message': ". ".join(message_parts) if message_parts else title, - 'labels': labels, - } - if issue_url: - alert_details['issueUrl'] = issue_url - - return build_rca_prompt('newrelic', alert_details, providers, user_id) - - -def build_sentry_rca_prompt( - payload: Dict[str, Any], - providers: Optional[List[str]] = None, - user_id: Optional[str] = None, -) -> tuple[str, str]: - """Build RCA prompt from a Sentry Integration Platform webhook payload.""" - from routes.sentry.tasks import extract_sentry_title - data = payload.get("data") or {} - issue = data.get("issue") if isinstance(data.get("issue"), dict) else {} - event = data.get("event") if isinstance(data.get("event"), dict) else {} - error = data.get("error") if isinstance(data.get("error"), dict) else {} - - title = extract_sentry_title(payload) - action = payload.get("action") or "unknown" - level = ( - issue.get("level") - or event.get("level") - or error.get("level") - or "unknown" - ) - - project = issue.get("project") or event.get("project") or {} - project_slug = project.get("slug") if isinstance(project, dict) else None - project_name = project.get("name") if isinstance(project, dict) else None - - culprit = issue.get("culprit") or event.get("culprit") or "" - short_id = issue.get("shortId") or "" - permalink = issue.get("permalink") or issue.get("web_url") or event.get("web_url") or "" - environment = event.get("environment") or "" - release = event.get("release") or "" - count = issue.get("count") - user_count = issue.get("userCount") - first_seen = issue.get("firstSeen") or "" - last_seen = issue.get("lastSeen") or "" - - message_parts: List[str] = [] - if culprit: - message_parts.append(f"Culprit: {culprit}") - if project_slug or project_name: - message_parts.append(f"Project: {project_slug or project_name}") - if environment: - message_parts.append(f"Environment: {environment}") - if release: - message_parts.append(f"Release: {release}") - if count is not None: - message_parts.append(f"Event count: {count}") - if user_count is not None: - message_parts.append(f"Users affected: {user_count}") - if first_seen: - message_parts.append(f"First seen: {first_seen}") - if last_seen and last_seen != first_seen: - message_parts.append(f"Last seen: {last_seen}") - - labels: Dict[str, str] = {} - if level and level != "unknown": - labels["level"] = str(level) - if short_id: - labels["shortId"] = str(short_id) - if project_slug: - labels["projectSlug"] = str(project_slug) - - alert_details = { - "title": title, - "status": f"{action} (level: {level})", - "message": ". ".join(message_parts) if message_parts else title, - "labels": labels, - } - if permalink: - alert_details["issueUrl"] = permalink - - return build_rca_prompt("sentry", alert_details, providers, user_id) - - -def build_chat_rca_prompt( - description: str, - title: str = "", - service: str = "", - severity: str = "medium", - providers: Optional[List[str]] = None, - user_id: Optional[str] = None, -) -> tuple[str, str]: - """Build RCA prompt from a user-reported incident in chat. - - Wraps the user's free-text description into the standard alert_details - format and delegates to the shared build_rca_prompt(). - """ - alert_title = title or f"User-reported: {description[:80]}" - - labels: Dict[str, str] = {} - if service: - labels["service"] = service - if severity: - labels["severity"] = severity - - alert_details = { - "title": alert_title, - "status": "investigating", - "message": description, - "labels": labels, - } - - return build_rca_prompt("chat", alert_details, providers, user_id) - - -def build_opsgenie_rca_prompt( - payload: Dict[str, Any], - providers: Optional[List[str]] = None, - user_id: Optional[str] = None, -) -> tuple[str, str]: - """Build RCA prompt from OpsGenie alert webhook payload.""" - alert = payload.get("alert", {}) - message = alert.get("message") or "Unknown Alert" - action = payload.get("action") or "unknown" - priority = alert.get("priority") or "unknown" - status = alert.get("status") or "unknown" - source = alert.get("source") or "unknown" - description = alert.get("description") or "" - entity = alert.get("entity") or "" - tags = alert.get("tags", []) - teams = alert.get("teams", []) - - message_parts = [] - if description: - message_parts.append(description) - if entity: - message_parts.append(f"Entity: {entity}") - if teams: - message_parts.append(f"Teams: {', '.join(teams) if isinstance(teams, list) else str(teams)}") - - alert_details = { - 'title': message, - 'status': f"{status} (action: {action}, priority: {priority})", - 'message': ". ".join(message_parts) if message_parts else message, - 'tags': tags, - 'source': source, - } - if entity: - alert_details['entity'] = entity - - return build_rca_prompt('opsgenie', alert_details, providers, user_id) - - -def _incidentio_dict_name(obj, default: str = "") -> str: - """Extract .name from a dict-or-scalar incident.io field.""" - if isinstance(obj, dict): - return obj.get("name", default) - return str(obj) if obj else default - - -def _incidentio_format_roles(roles: list) -> str: - return ", ".join( - f"{r.get('role', {}).get('name', '?')}: {r.get('assignee', {}).get('name', 'unassigned')}" - for r in roles[:5] - ) - - -def _incidentio_format_custom_fields(custom_fields: list) -> str: - return ", ".join( - f"{cf.get('custom_field', {}).get('name', '?')}=" - f"{(cf.get('values') or [{}])[0].get('label', '?')}" - for cf in custom_fields[:5] - if cf.get("values") - ) - - -def build_incidentio_rca_prompt( - payload: Dict[str, Any], - providers: Optional[List[str]] = None, - user_id: Optional[str] = None, -) -> tuple[str, str]: - """Build RCA prompt from incident.io webhook event payload.""" - event = payload.get("event", {}) or {} - incident = event.get("incident") or payload.get("incident") or {} - - name = incident.get("name") or incident.get("title") or "Unknown Incident" - status = incident.get("status") or "unknown" - summary = incident.get("summary") or "" - permalink = incident.get("permalink") or "" - severity = _incidentio_dict_name(incident.get("severity")) - inc_type = _incidentio_dict_name(incident.get("incident_type")) - - role_str = _incidentio_format_roles(incident.get("incident_role_assignments") or []) - cf_str = _incidentio_format_custom_fields(incident.get("custom_field_entries") or []) - - message_parts = [f"Incident: {name}"] - for label, value in [("Summary", summary), ("Roles", role_str), - ("Fields", cf_str), ("Link", permalink)]: - if value: - message_parts.append(f"{label}: {value}") - - labels = {} - if severity: - labels['severity'] = severity - if inc_type: - labels['incident_type'] = inc_type - - alert_details = { - 'title': name, - 'status': f"{status} (severity: {severity})" if severity else status, - 'message': ". ".join(message_parts), - 'labels': labels, - } + prompt = "\n".join(prompt_parts) + rail_text = _extract_rail_text_from_payload(payload) - return build_rca_prompt('incidentio', alert_details, providers, user_id) + return prompt, rail_text diff --git a/server/chat/background/task.py b/server/chat/background/task.py index 1426e59c9..4f4bd24be 100644 --- a/server/chat/background/task.py +++ b/server/chat/background/task.py @@ -452,6 +452,43 @@ def run_background_chat( logger.info(f"[BackgroundChat] Starting for user {user_id}, session {session_id}") logger.info(f"[BackgroundChat] Trigger: {trigger_metadata}") + + # Eagerly persist the initial user message so it's visible in the UI immediately (opt-in) + if os.environ.get("DISPLAY__RCA_USER_MSG", "").lower() in ("1", "true", "yes"): + try: + with db_pool.get_admin_connection() as conn: + with conn.cursor() as cursor: + set_rls_context(cursor, conn, user_id, log_prefix="[BackgroundChat:EagerMsg]") + cursor.execute("SELECT messages FROM chat_sessions WHERE id = %s", (session_id,)) + row = cursor.fetchone() + messages = row[0] if row and row[0] else [] + if isinstance(messages, str): + messages = json.loads(messages) + if not isinstance(messages, list): + messages = [] + has_user_message = any( + isinstance(m, dict) and m.get("sender") == "user" + for m in messages + ) + if not has_user_message: + existing_numbers = [ + int(m.get("message_number", 0)) + for m in messages + if isinstance(m, dict) + ] + next_num = max(existing_numbers, default=0) + 1 + messages.append({ + "sender": "user", + "text": initial_message, + "message_number": next_num, + }) + cursor.execute( + "UPDATE chat_sessions SET messages = %s::jsonb WHERE id = %s", + (json.dumps(messages), session_id), + ) + conn.commit() + except Exception as e: + logger.warning(f"[BackgroundChat] Failed to eagerly persist user message: {e}") completed_successfully = False diff --git a/server/routes/bigpanda/tasks.py b/server/routes/bigpanda/tasks.py index f03741c52..205b97058 100644 --- a/server/routes/bigpanda/tasks.py +++ b/server/routes/bigpanda/tasks.py @@ -12,6 +12,7 @@ from typing import Any, Dict, Optional from celery_config import celery_app +from chat.background.rca_prompt_builder import build_rca_prompt from services.correlation.alert_correlator import AlertCorrelator from services.correlation import handle_correlated_alert from utils.auth.stateless_auth import get_user_preference @@ -89,11 +90,6 @@ def _should_trigger_rca(user_id: str) -> bool: return get_user_preference(user_id, "bigpanda_rca_enabled", default=False) -def _build_rca_prompt(incident: dict[str, Any], alerts: list[dict[str, Any]], user_id: str | None = None) -> tuple[str, str]: - from chat.background.rca_prompt_builder import build_bigpanda_rca_prompt - return build_bigpanda_rca_prompt(incident, alerts, user_id=user_id) - - @celery_app.task( bind=True, max_retries=3, default_retry_delay=30, name="bigpanda.process_event", @@ -269,7 +265,7 @@ def process_bigpanda_event( trigger_metadata={"source": "bigpanda", "incident_id": incident_id}, incident_id=str(aurora_incident_id), ) - rca_prompt, rail_text = _build_rca_prompt(incident, alerts, user_id=user_id) + rca_prompt, rail_text = build_rca_prompt("bigpanda", title, raw_payload or {"incident": incident, "alerts": alerts}, user_id=user_id) task = run_background_chat.delay( user_id=user_id, session_id=session_id, initial_message=rca_prompt, diff --git a/server/routes/datadog/tasks.py b/server/routes/datadog/tasks.py index b81987c22..a1401bcc6 100644 --- a/server/routes/datadog/tasks.py +++ b/server/routes/datadog/tasks.py @@ -8,7 +8,7 @@ from typing import Any, Dict, Optional from celery_config import celery_app -from chat.background.rca_prompt_builder import build_datadog_rca_prompt +from chat.background.rca_prompt_builder import build_rca_prompt from services.correlation.alert_correlator import AlertCorrelator from services.correlation import handle_correlated_alert @@ -407,8 +407,8 @@ def process_datadog_event( ) # Build comprehensive RCA prompt with provider context - rca_prompt, rail_text = build_datadog_rca_prompt( - payload, user_id=user_id + rca_prompt, rail_text = build_rca_prompt( + "datadog", event_title, payload, user_id=user_id ) # Start RCA task and immediately store task ID diff --git a/server/routes/dynatrace/tasks.py b/server/routes/dynatrace/tasks.py index 0a9c64fa0..36735a70b 100644 --- a/server/routes/dynatrace/tasks.py +++ b/server/routes/dynatrace/tasks.py @@ -8,7 +8,7 @@ from typing import Any from celery_config import celery_app -from chat.background.rca_prompt_builder import build_dynatrace_rca_prompt +from chat.background.rca_prompt_builder import build_rca_prompt from services.correlation.alert_correlator import AlertCorrelator from services.correlation import handle_correlated_alert from utils.auth.stateless_auth import get_user_preference @@ -187,7 +187,7 @@ def process_dynatrace_problem( trigger_metadata={"source": "dynatrace", "problem_id": payload.get("ProblemID")}, incident_id=str(incident_id), ) - rca_prompt, rail_text = build_dynatrace_rca_prompt(payload, user_id=user_id) + rca_prompt, rail_text = build_rca_prompt("dynatrace", title, payload, user_id=user_id) task = run_background_chat.delay( user_id=user_id, session_id=session_id, initial_message=rca_prompt, diff --git a/server/routes/grafana/tasks.py b/server/routes/grafana/tasks.py index df973d894..89200cf45 100644 --- a/server/routes/grafana/tasks.py +++ b/server/routes/grafana/tasks.py @@ -18,7 +18,7 @@ from typing import Any, Dict, Optional from celery_config import celery_app -from chat.background.rca_prompt_builder import build_grafana_rca_prompt +from chat.background.rca_prompt_builder import build_rca_prompt from services.correlation.alert_correlator import AlertCorrelator from services.correlation import handle_correlated_alert @@ -517,7 +517,7 @@ def process_grafana_alert( user_id=user_id, title=chat_title, trigger_metadata={"source": "grafana", "alert_uid": alert_uid, "alert_state": alert_state}, ) - rca_prompt, rail_text = build_grafana_rca_prompt(alert_payload, user_id=user_id) + rca_prompt, rail_text = build_rca_prompt("grafana", per_alert_title, alert_payload, user_id=user_id) task = run_background_chat.delay( user_id=user_id, session_id=session_id, initial_message=rca_prompt, trigger_metadata={"source": "grafana", "alert_uid": alert_uid, diff --git a/server/routes/incidentio/tasks.py b/server/routes/incidentio/tasks.py index 08dfd4bd0..ffa17afa8 100644 --- a/server/routes/incidentio/tasks.py +++ b/server/routes/incidentio/tasks.py @@ -8,7 +8,7 @@ from typing import Any, Dict, Optional from celery_config import celery_app -from chat.background.rca_prompt_builder import build_incidentio_rca_prompt +from chat.background.rca_prompt_builder import build_rca_prompt from services.correlation.alert_correlator import AlertCorrelator from services.correlation import handle_correlated_alert @@ -414,7 +414,7 @@ def _trigger_rca_pipeline( incident_id=str(incident_id), ) - rca_prompt, rail_text = build_incidentio_rca_prompt(payload, user_id=user_id) + rca_prompt, rail_text = build_rca_prompt("incidentio", fields["incident_name"], payload, user_id=user_id) task = run_background_chat.delay( user_id=user_id, diff --git a/server/routes/jenkins/tasks.py b/server/routes/jenkins/tasks.py index d9a619abb..9838b85f2 100644 --- a/server/routes/jenkins/tasks.py +++ b/server/routes/jenkins/tasks.py @@ -8,6 +8,7 @@ from typing import Any, Dict, Optional from celery_config import celery_app +from chat.background.rca_prompt_builder import build_rca_prompt from services.correlation.alert_correlator import AlertCorrelator from services.correlation import handle_correlated_alert @@ -56,15 +57,6 @@ def _extract_git(payload: Dict[str, Any]) -> Dict[str, str]: } -def _build_rca_prompt(payload: Dict[str, Any], user_id: Optional[str] = None, source: str = "jenkins") -> tuple[str, str]: - """Build an RCA prompt from a deployment failure using the full prompt builder.""" - if source == "cloudbees": - from chat.background.rca_prompt_builder import build_cloudbees_rca_prompt - return build_cloudbees_rca_prompt(payload, user_id=user_id) - from chat.background.rca_prompt_builder import build_jenkins_rca_prompt - return build_jenkins_rca_prompt(payload, user_id=user_id) - - @celery_app.task( bind=True, max_retries=3, default_retry_delay=30, name="jenkins.process_deployment" ) @@ -375,7 +367,7 @@ def _trigger_rca( }, incident_id=str(incident_id), ) - rca_prompt, rail_text = _build_rca_prompt(payload, user_id=user_id, source=source) + rca_prompt, rail_text = build_rca_prompt(source, alert_title, payload, user_id=user_id) task = run_background_chat.delay( user_id=user_id, session_id=session_id, diff --git a/server/routes/netdata/helpers.py b/server/routes/netdata/helpers.py index b8f5cdcec..f24789052 100644 --- a/server/routes/netdata/helpers.py +++ b/server/routes/netdata/helpers.py @@ -108,55 +108,3 @@ def should_trigger_background_chat(user_id: str, payload: Dict[str, Any]) -> boo # Always trigger RCA for any webhook received return True - - -def build_rca_prompt_from_alert(normalized: Dict[str, Any], user_id: Optional[str] = None) -> str: - """Build an RCA analysis prompt from normalized Netdata alert data. - - Args: - normalized: The normalized Netdata alert data (from normalize_netdata_payload) - user_id: Optional user ID for Aurora Learn context injection - - Returns: - A prompt string for the background chat agent - """ - alarm = normalized.get("name") or "Unknown Alert" - status = normalized.get("status") or "unknown" - host = normalized.get("host") or "unknown" - chart = normalized.get("chart") or "unknown" - alert_class = normalized.get("class") or "unknown" - family = normalized.get("family") or "unknown" - space = normalized.get("space") or "unknown" - room = normalized.get("room") or "unknown" - value = normalized.get("value") - message = normalized.get("message") or "" - - # Build the prompt parts - prompt_parts = [ - "A Netdata alert has been triggered and requires Root Cause Analysis.", - "", - "ALERT DETAILS:", - f"- Alarm: {alarm}", - f"- Status: {status}", - f"- Host: {host}", - f"- Chart: {chart}", - f"- Class: {alert_class}", - f"- Family: {family}", - f"- Space: {space}", - f"- Room: {room}", - ] - - if value: - prompt_parts.append(f"- Value: {value}") - - if message: - prompt_parts.append(f"- Message: {message}") - - # Add Aurora Learn context if available - try: - from chat.background.rca_prompt_builder import inject_aurora_learn_context - inject_aurora_learn_context(prompt_parts, user_id, alarm, host, "netdata") - except Exception as e: - logger.warning(f"[AURORA LEARN] Failed to get context: {e}") - - return "\n".join(prompt_parts) diff --git a/server/routes/netdata/tasks.py b/server/routes/netdata/tasks.py index f161b9bd7..57870970d 100644 --- a/server/routes/netdata/tasks.py +++ b/server/routes/netdata/tasks.py @@ -20,7 +20,7 @@ normalize_netdata_payload, should_trigger_background_chat, ) -from chat.background.rca_prompt_builder import build_netdata_rca_prompt +from chat.background.rca_prompt_builder import build_rca_prompt from services.correlation.alert_correlator import AlertCorrelator from services.correlation import handle_correlated_alert from utils.payload_timestamp import extract_alert_fired_at @@ -312,8 +312,8 @@ def process_netdata_alert( ) # Build simple RCA prompt with Aurora Learn context injection - rca_prompt, rail_text = build_netdata_rca_prompt( - data, user_id=user_id + rca_prompt, rail_text = build_rca_prompt( + "netdata", data.get('name', 'Netdata Alert'), payload, user_id=user_id ) # Start RCA task and immediately store task ID diff --git a/server/routes/newrelic/tasks.py b/server/routes/newrelic/tasks.py index 169fe6380..d54ca3073 100644 --- a/server/routes/newrelic/tasks.py +++ b/server/routes/newrelic/tasks.py @@ -8,7 +8,7 @@ from typing import Any, Dict, List, Optional from celery_config import celery_app -from chat.background.rca_prompt_builder import build_newrelic_rca_prompt +from chat.background.rca_prompt_builder import build_rca_prompt from services.correlation.alert_correlator import AlertCorrelator from services.correlation import handle_correlated_alert from utils.payload_timestamp import extract_alert_fired_at @@ -424,7 +424,7 @@ def process_newrelic_event( incident_id=str(incident_id), ) - rca_prompt, rail_text = build_newrelic_rca_prompt(payload, user_id=user_id) + rca_prompt, rail_text = build_rca_prompt("newrelic", event_title, payload, user_id=user_id) task = run_background_chat.delay( user_id=user_id, diff --git a/server/routes/opsgenie/tasks.py b/server/routes/opsgenie/tasks.py index 8cbf64aa0..3008cd538 100644 --- a/server/routes/opsgenie/tasks.py +++ b/server/routes/opsgenie/tasks.py @@ -9,7 +9,7 @@ from typing import Any, Dict, Optional from celery_config import celery_app -from chat.background.rca_prompt_builder import build_opsgenie_rca_prompt +from chat.background.rca_prompt_builder import build_rca_prompt from services.correlation.alert_correlator import AlertCorrelator from services.correlation import handle_correlated_alert @@ -341,8 +341,8 @@ def process_opsgenie_event( incident_id=str(incident_id), ) - rca_prompt, rail_text = build_opsgenie_rca_prompt( - payload, user_id=user_id + rca_prompt, rail_text = build_rca_prompt( + "opsgenie", alert_message, payload, user_id=user_id ) task = run_background_chat.delay( diff --git a/server/routes/pagerduty/tasks.py b/server/routes/pagerduty/tasks.py index ba33b73ba..b72144c94 100644 --- a/server/routes/pagerduty/tasks.py +++ b/server/routes/pagerduty/tasks.py @@ -8,7 +8,7 @@ from typing import Any, Dict, Optional from celery_config import celery_app -from chat.background.rca_prompt_builder import build_pagerduty_rca_prompt +from chat.background.rca_prompt_builder import build_rca_prompt from services.correlation.alert_correlator import AlertCorrelator from services.correlation import handle_correlated_alert from utils.auth.stateless_auth import set_rls_context @@ -348,8 +348,8 @@ def trigger_delayed_rca( event_data = consolidated_payload.get("event", {}) incident_data = event_data.get("data", {}) - rca_prompt, rail_text = build_pagerduty_rca_prompt( - incident_data, user_id=user_id + rca_prompt, rail_text = build_rca_prompt( + "pagerduty", incident_title, incident_data, user_id=user_id ) except (json.JSONDecodeError, KeyError, TypeError) as e: rca_prompt = ( diff --git a/server/routes/sentry/tasks.py b/server/routes/sentry/tasks.py index 2a2d8c17c..6fe5cb3d1 100644 --- a/server/routes/sentry/tasks.py +++ b/server/routes/sentry/tasks.py @@ -11,7 +11,7 @@ import requests from celery_config import celery_app -from chat.background.rca_prompt_builder import build_sentry_rca_prompt +from chat.background.rca_prompt_builder import build_rca_prompt from services.correlation.alert_correlator import AlertCorrelator from services.correlation import handle_correlated_alert from utils.payload_timestamp import extract_alert_fired_at @@ -403,7 +403,7 @@ def process_sentry_event( incident_id=str(incident_id), ) - rca_prompt, rail_text = build_sentry_rca_prompt(payload, user_id=user_id) + rca_prompt, rail_text = build_rca_prompt("sentry", title, payload, user_id=user_id) task = run_background_chat.delay( user_id=user_id, diff --git a/server/routes/spinnaker/tasks.py b/server/routes/spinnaker/tasks.py index 8e9b4f544..778402d88 100644 --- a/server/routes/spinnaker/tasks.py +++ b/server/routes/spinnaker/tasks.py @@ -8,6 +8,7 @@ from typing import Any, Dict, Optional from celery_config import celery_app +from chat.background.rca_prompt_builder import build_rca_prompt from services.correlation.alert_correlator import AlertCorrelator from services.correlation import handle_correlated_alert from utils.auth.stateless_auth import set_rls_context @@ -61,12 +62,6 @@ def _extract_execution_fields(payload: Dict[str, Any]) -> Dict[str, Any]: } -def _build_rca_prompt(payload: Dict[str, Any], user_id: Optional[str] = None) -> tuple[str, str]: - """Build an RCA prompt from a deployment failure.""" - from chat.background.rca_prompt_builder import build_spinnaker_rca_prompt - return build_spinnaker_rca_prompt(payload, user_id=user_id) - - @celery_app.task( bind=True, max_retries=3, default_retry_delay=30, name="spinnaker.process_deployment" ) @@ -342,7 +337,7 @@ def _trigger_rca( }, incident_id=str(incident_id), ) - rca_prompt, rail_text = _build_rca_prompt(payload, user_id=user_id) + rca_prompt, rail_text = build_rca_prompt("spinnaker", alert_title, payload, user_id=user_id) task = run_background_chat.delay( user_id=user_id, session_id=session_id, diff --git a/server/routes/splunk/tasks.py b/server/routes/splunk/tasks.py index a8cbc7daa..5970e928d 100644 --- a/server/routes/splunk/tasks.py +++ b/server/routes/splunk/tasks.py @@ -8,7 +8,7 @@ from typing import Any, Dict, Optional from celery_config import celery_app -from chat.background.rca_prompt_builder import build_splunk_rca_prompt +from chat.background.rca_prompt_builder import build_rca_prompt from services.correlation.alert_correlator import AlertCorrelator from services.correlation import handle_correlated_alert @@ -345,8 +345,8 @@ def process_splunk_alert( ) # Build comprehensive RCA prompt with provider context - rca_prompt, rail_text = build_splunk_rca_prompt( - payload, user_id=user_id + rca_prompt, rail_text = build_rca_prompt( + "splunk", alert_title, payload, user_id=user_id ) # Start RCA task and immediately store task ID