diff --git a/server/chat/backend/agent/prompt/composer.py b/server/chat/backend/agent/prompt/composer.py index c36d19620..ffcf7da1f 100644 --- a/server/chat/backend/agent/prompt/composer.py +++ b/server/chat/backend/agent/prompt/composer.py @@ -22,34 +22,43 @@ ) from .schema import PromptSegments +_RCA_SECTIONS_DIR = os.path.normpath( + os.path.join(os.path.dirname(__file__), "rca_sections") +) -def build_system_invariant(is_background: bool = False) -> str: - """Load core system prompt from modular markdown files under skills/core/. +_RCA_SECTION_ORDER = [ + "identity", + "investigation", + "context_mgmt", + "error_recovery", + "evidence_standard", + "conclusion_gate", +] - Segments are loaded in a fixed order that mirrors the original monolithic - prompt so that cached prefixes remain stable across deployments. - In background RCA mode, Terraform/IaC, SSH setup, and cloud CLI discovery - segments are omitted (~3,300 tokens) since background investigations are - read-only and the freed budget is better spent on integration skills. - """ - from chat.backend.agent.skills.loader import load_core_prompt +def _build_rca_system_prompt() -> str: + """Assemble the background RCA system prompt from rca_sections/*.md.""" + parts = [] + for name in _RCA_SECTION_ORDER: + path = os.path.join(_RCA_SECTIONS_DIR, f"{name}.md") + with open(path, "r", encoding="utf-8") as f: + content = f.read().strip() + if content: + parts.append(content) + + return "\n\n".join(parts) - core_dir = os.path.join( - os.path.dirname(__file__), os.pardir, "skills", "core" - ) - core_dir = os.path.normpath(core_dir) + +def build_system_invariant(is_background: bool = False) -> str: + """Build the system prompt for interactive or background RCA mode.""" + from chat.backend.agent.skills.loader import load_core_prompt if is_background: - return load_core_prompt(core_dir, segments=[ - "identity", - "security", - "knowledge_base", - "error_handling", - "investigation", - "behavioral_rules", - ]) + return _build_rca_system_prompt() + core_dir = os.path.normpath( + os.path.join(os.path.dirname(__file__), os.pardir, "skills", "core") + ) return load_core_prompt(core_dir, segments=[ "identity", "security", diff --git a/server/chat/backend/agent/prompt/rca_sections/conclusion_gate.md b/server/chat/backend/agent/prompt/rca_sections/conclusion_gate.md new file mode 100644 index 000000000..3083f59e0 --- /dev/null +++ b/server/chat/backend/agent/prompt/rca_sections/conclusion_gate.md @@ -0,0 +1,14 @@ +# Before Concluding + +You will conclude too early. Recognize these traps: +- "The timing correlates": correlation is not causation. Find the mechanism. +- "This is the most common cause": common does not mean actual for THIS incident. +- "I found one log line that matches": one data point is not a pattern. +- "The service restarted, so resource exhaustion": check actual resource metrics. +- "We need to scale up resources": that's a band-aid, not a root cause. Why are resources insufficient now? Did something change or was it always underprovisioned? +- "The cluster is unstable": what specifically is making it unstable? Which node, which component, what changed? + +Before stating root cause, answer: +1. What alternative did you rule out, and how? +2. What specific evidence (tool output) proves the mechanism, not just the correlation? +3. Does your root cause explain the timing of the alert? diff --git a/server/chat/backend/agent/prompt/rca_sections/context_mgmt.md b/server/chat/backend/agent/prompt/rca_sections/context_mgmt.md new file mode 100644 index 000000000..740979d8c --- /dev/null +++ b/server/chat/backend/agent/prompt/rca_sections/context_mgmt.md @@ -0,0 +1,5 @@ +# Recording Findings + +After each tool call, write down what you found before making the next call. Earlier tool results may be cleared from context. If a finding only exists in a tool result you didn't record, it's gone. + +Be concise: what you found, what it means, what you'll check next. Nothing else. diff --git a/server/chat/backend/agent/prompt/rca_sections/error_recovery.md b/server/chat/backend/agent/prompt/rca_sections/error_recovery.md new file mode 100644 index 000000000..4ada34b78 --- /dev/null +++ b/server/chat/backend/agent/prompt/rca_sections/error_recovery.md @@ -0,0 +1,9 @@ +# Error Recovery + +When a tool call returns empty or errors: +- Broaden the time window (-2h, -6h, -24h) +- Simplify the query (fewer filters, broader match) +- Try a different resource type (metrics instead of logs, events instead of traces) +- Verify the service/host name exists by listing available resources + +Empty results are data as they rule things out. Do not repeat a failed query unchanged. If all avenues are exhausted, state what was ruled out and stop. diff --git a/server/chat/backend/agent/prompt/rca_sections/evidence_standard.md b/server/chat/backend/agent/prompt/rca_sections/evidence_standard.md new file mode 100644 index 000000000..2e7616c8a --- /dev/null +++ b/server/chat/backend/agent/prompt/rca_sections/evidence_standard.md @@ -0,0 +1,7 @@ +# Evidence + +Never state a root cause without citing the specific tool output that proves it. Exact timestamps, error messages, metric values. + +If you cannot determine root cause, say what you confirmed, what you ruled out, and what remains unverified. Distinguish facts from hypotheses in your reporting. + +Do not fabricate log lines, metrics, or timestamps. Do not hedge confirmed findings or overclaim uncertain ones. Match your confidence to the evidence. diff --git a/server/chat/backend/agent/prompt/rca_sections/identity.md b/server/chat/backend/agent/prompt/rca_sections/identity.md new file mode 100644 index 000000000..e58e3e7ce --- /dev/null +++ b/server/chat/backend/agent/prompt/rca_sections/identity.md @@ -0,0 +1,3 @@ +# Identity + +You are Aurora, an SRE agent investigating a production alert. Your goal is not just to name the root cause. It is to understand the incident deeply enough to remediate it, prevent recurrence, and write the post-mortem. diff --git a/server/chat/backend/agent/prompt/rca_sections/investigation.md b/server/chat/backend/agent/prompt/rca_sections/investigation.md new file mode 100644 index 000000000..152934a39 --- /dev/null +++ b/server/chat/backend/agent/prompt/rca_sections/investigation.md @@ -0,0 +1,9 @@ +# Investigation + +Before your first tool call, state your hypothesis and what you will query to test it. + +Work from the outside in. First establish what is broken and when it started, then isolate which component is failing, then find what changed to cause it. Something changed. A deploy, a config, a dependency, traffic, resources. Find that change. + +A symptom is not a root cause. "The pod is OOMKilled" is a symptom. "Memory leak in the request parser introduced in commit X" is a root cause. "The pod needs more resources" is not specific enough. Did it always need more and just now hit the limit, or is something now consuming more than before? If consumption changed, what changed it? "The cluster is unstable" is not specific. Which component, which node, what changed? Keep drilling until you reach something specific and actionable. + +Design queries to disprove your hypothesis, not confirm it. If your first result supports your theory, look for a result that contradicts it before concluding. diff --git a/server/chat/backend/agent/skills/rca/segments/critical_requirements_header.md b/server/chat/backend/agent/skills/rca/segments/critical_requirements_header.md deleted file mode 100644 index 68278f4db..000000000 --- a/server/chat/backend/agent/skills/rca/segments/critical_requirements_header.md +++ /dev/null @@ -1 +0,0 @@ -## CRITICAL INVESTIGATION REQUIREMENTS: diff --git a/server/chat/backend/agent/skills/rca/segments/error_resilience_infra.md b/server/chat/backend/agent/skills/rca/segments/error_resilience_infra.md deleted file mode 100644 index a2bbc973b..000000000 --- a/server/chat/backend/agent/skills/rca/segments/error_resilience_infra.md +++ /dev/null @@ -1,4 +0,0 @@ -- If cloud monitoring/metrics commands fail -> use kubectl directly -- If kubectl fails -> check cloud provider CLI alternatives -- If one log source fails -> try another (kubectl logs, cloud logging, container logs) -- If a resource isn't found -> check other namespaces, regions, or projects diff --git a/server/chat/backend/agent/skills/rca/segments/error_resilience_intro.md b/server/chat/backend/agent/skills/rca/segments/error_resilience_intro.md deleted file mode 100644 index 09ce24eb9..000000000 --- a/server/chat/backend/agent/skills/rca/segments/error_resilience_intro.md +++ /dev/null @@ -1,2 +0,0 @@ -### ERROR RESILIENCE: -- If one tool or data source fails, try an alternative immediately diff --git a/server/chat/backend/agent/skills/rca/segments/error_resilience_outro.md b/server/chat/backend/agent/skills/rca/segments/error_resilience_outro.md deleted file mode 100644 index e4e0bd3b4..000000000 --- a/server/chat/backend/agent/skills/rca/segments/error_resilience_outro.md +++ /dev/null @@ -1 +0,0 @@ -- **ALWAYS have 3-4 backup data sources or diagnostic approaches ready** diff --git a/server/chat/backend/agent/skills/rca/segments/output_requirements.md b/server/chat/backend/agent/skills/rca/segments/output_requirements.md deleted file mode 100644 index f7367f8e5..000000000 --- a/server/chat/backend/agent/skills/rca/segments/output_requirements.md +++ /dev/null @@ -1,21 +0,0 @@ -## OUTPUT REQUIREMENTS: - -### Your analysis MUST include: -1. **Summary**: Brief description of the incident -2. **Investigation Steps**: Document EVERY tool call and what it revealed -3. **Evidence**: Show specific log entries, metric values, config snippets -4. **Root Cause**: Clearly state the EXACT root cause with supporting evidence -5. **Impact**: Describe what was affected and how -6. **Remediation**: Specific, actionable steps to fix the issue -7. **Code Fix** (if applicable): If the root cause is a code defect and GitHub is connected, you MUST call `github_fix` to propose the fix. This creates a review-only suggestion - it is safe and expected. - -### Remember: -- During the initial automated investigation, do NOT make direct infrastructure changes (no scaling, restarts, config writes) — investigate only. -- However, if the **user explicitly asks** you to execute a command or take an action (e.g., restart a service, apply a fix), you MUST comply. User instructions always override investigation-mode constraints. -- `github_fix` is always safe: it creates a *suggestion* for user review, not a direct change. Always use it when you find a code defect. -- The user expects you to find the EXACT root cause, not surface-level symptoms -- Keep digging until you have definitive answers -- Never conclude with 'unable to determine' without exhausting all investigation avenues - -## BEGIN INVESTIGATION NOW -Start by understanding the scope of the issue, then systematically investigate using the tools and approaches above. diff --git a/server/chat/backend/agent/skills/rca/segments/persistence_and_immediate_action.md b/server/chat/backend/agent/skills/rca/segments/persistence_and_immediate_action.md deleted file mode 100644 index 53ef3dff3..000000000 --- a/server/chat/backend/agent/skills/rca/segments/persistence_and_immediate_action.md +++ /dev/null @@ -1,15 +0,0 @@ -### PERSISTENCE IS MANDATORY: -- **MINIMUM**: Make AT LEAST 15-20 tool calls before concluding -- **DO NOT STOP** after 2-3 commands - keep investigating until you find the EXACT root cause -- **SPEND TIME**: Investigation should take AT LEAST 3-5 minutes of active tool usage -- **IF BLOCKED**: Try 3-5 alternative data sources or diagnostic tools before giving up on any single avenue -- **COMMAND FAILURES ARE NOT STOPPING POINTS**: When a diagnostic command fails, try alternative data sources immediately -- **ACCESS DENIED IS A STOPPING POINT**: When access is denied or authentication fails, do NOT attempt to bypass - pivot to other authorized tools and data sources - -### IMMEDIATE ACTION REQUIRED: -- **DO NOT** output a plan or text explanation first. -- **DO NOT** say 'I will start by...' -- **If Jira is connected, your FIRST tool call MUST be jira_search_issues.** -- After {after_context_label} context, proceed to infrastructure/CI tools. -- UNLESS YOU ARE DONE, your response MUST contain a tool call. -- NOT PROVIDING A TOOL CALL WILL END THE INVESTIGATION AUTOMATICALLY diff --git a/server/chat/backend/agent/skills/rca/segments/what_to_investigate.md b/server/chat/backend/agent/skills/rca/segments/what_to_investigate.md deleted file mode 100644 index 44da5cbe5..000000000 --- a/server/chat/backend/agent/skills/rca/segments/what_to_investigate.md +++ /dev/null @@ -1,8 +0,0 @@ -### WHAT TO INVESTIGATE: -- Resource STATUS and HEALTH (running, pending, failed, etc.) -- LOGS for error messages, warnings, stack traces -- METRICS for CPU, memory, disk, network anomalies -- CONFIGURATIONS for misconfigurations or invalid values -- EVENTS for recent state changes -- DEPENDENCIES for cascading failures -- RECENT DEPLOYMENTS (not just commits) that correlate with the issue — verify the change reached production before blaming it diff --git a/server/chat/background/rca_prompt_builder.py b/server/chat/background/rca_prompt_builder.py index 9c0f38011..2ea3630ee 100644 --- a/server/chat/background/rca_prompt_builder.py +++ b/server/chat/background/rca_prompt_builder.py @@ -1,33 +1,20 @@ """ Shared RCA (Root Cause Analysis) prompt builder for background alert processing. -This module creates provider-aware, persistence-focused RCA prompts that leverage -all available tools and follow the detailed investigation guidelines in the system prompt. +This module creates provider-aware RCA prompts that pair alert data with +infrastructure context. Behavioral investigation guidance lives in the system +prompt (rca_sections/). The user message contains only alert facts and context. Aurora Learn Integration: - When Aurora Learn is enabled, searches for similar past incidents with positive feedback - Injects context from helpful RCAs to improve new investigations """ -from functools import lru_cache from typing import Any, Dict, List, Optional import logging -import os logger = logging.getLogger(__name__) -RCA_SEGMENTS_DIR = os.path.normpath( - os.path.join( - os.path.dirname(__file__), - os.pardir, - "backend", - "agent", - "skills", - "rca", - "segments", - ) -) - def build_alert_rail_text(alert_details: Dict[str, Any]) -> str: """Extract the webhook-authored subset of an alert for input-rail evaluation. @@ -52,59 +39,6 @@ def build_alert_rail_text(alert_details: Dict[str, Any]) -> str: return "\n\n".join(parts) -@lru_cache(maxsize=32) -def _load_rca_segment_template(segment_name: str) -> str: - """ - Load an RCA markdown segment by name (filename without .md). - - Segment content is cached in-process for performance. - """ - try: - from chat.backend.agent.skills.loader import load_core_prompt - - return load_core_prompt(RCA_SEGMENTS_DIR, segments=[segment_name]).strip() - except Exception as e: - logger.warning(f"Failed to load RCA segment '{segment_name}': {e}") - return "" - - -def _render_rca_segment(segment_name: str, context: Optional[Dict[str, Any]] = None) -> str: - """Render an RCA segment with optional {variable} template substitutions.""" - template = _load_rca_segment_template(segment_name) - if not template: - return "" - - if not context: - return template - - try: - from chat.backend.agent.skills.loader import resolve_template - - return resolve_template(template, context) - except Exception as e: - logger.warning(f"Failed to render RCA segment '{segment_name}': {e}") - return template - - -def _append_rca_segment( - prompt_parts: List[str], - segment_name: str, - context: Optional[Dict[str, Any]] = None, - leading_blank: bool = False, - trailing_blank: bool = False, -) -> None: - """Append rendered segment to prompt_parts with optional surrounding blank lines.""" - content = _render_rca_segment(segment_name, context=context) - if not content: - return - - if leading_blank: - prompt_parts.append("") - prompt_parts.append(content) - if trailing_blank: - prompt_parts.append("") - - # ============================================================================ # Aurora Learn - Similar RCA Context Injection # ============================================================================ @@ -627,11 +561,6 @@ def build_rca_prompt( f"You have access to: {', '.join(providers) if providers else 'No cloud/monitoring providers connected'}", ]) - # All integration guidance (GitHub, Jira, Confluence, Jenkins, CloudBees, - # provider investigation commands) loaded from skill files via SkillRegistry - # in the system prompt (background.py). No skill loading here — the user - # message should contain only alert details and investigation context. - # Aurora Learn: Inject context from similar past incidents if user_id: similar_context = _get_similar_good_rcas_context( @@ -653,59 +582,8 @@ def build_rca_prompt( if prediscovery_context: prompt_parts.append(prediscovery_context) - # Critical investigation requirements (modular markdown segments) - _append_rca_segment( - prompt_parts, - "critical_requirements_header", - leading_blank=True, - trailing_blank=True, - ) - - has_infra_providers = bool({'gcp', 'aws', 'azure', 'ovh', 'scaleway'}.intersection(set(providers_lower))) - has_jira = bool((integrations or {}).get('jira')) - has_confluence = bool((integrations or {}).get('confluence')) - after_context_label = 'Jira' if has_jira else 'Confluence' if has_confluence else 'change' - - # Add aggressive persistence prompts only if cost optimization is disabled - # The immediate action required due to the AgentExecutor which assumes agent is done when it sends a text chunk without a tool call. - if os.getenv("RCA_OPTIMIZE_COSTS", "").lower() != "true": - _append_rca_segment( - prompt_parts, - "persistence_and_immediate_action", - context={"after_context_label": after_context_label}, - trailing_blank=True, - ) - - depth_steps = [] - if has_jira or has_confluence: - depth_steps.append("**Search Jira/Confluence first** for recent changes, open bugs, and runbooks") - depth_steps.extend([ - "Start broad - understand the overall system state", - "Identify the affected component(s)", - "Drill down into specifics - logs, metrics, configurations", - "Check related/dependent resources", - "Look for recent changes that correlate with the issue", - ]) - if has_infra_providers: - depth_steps.extend([ - "Compare with healthy resources of the same type", - "Check resource quotas, limits, and constraints", - "Examine network connectivity and security rules", - "Verify IAM permissions and service accounts", - ]) - depth_steps.append("Review historical patterns if available") - prompt_parts.append("### INVESTIGATION DEPTH:") - for i, step in enumerate(depth_steps, 1): - prompt_parts.append(f"{i}. {step}") - - _append_rca_segment(prompt_parts, "error_resilience_intro", leading_blank=True) - if has_infra_providers: - _append_rca_segment(prompt_parts, "error_resilience_infra") - _append_rca_segment(prompt_parts, "error_resilience_outro") - - _append_rca_segment(prompt_parts, "what_to_investigate", leading_blank=True) - _append_rca_segment(prompt_parts, "output_requirements", leading_blank=True) - + # User message contains only alert data + context. + # All behavioral guidance lives in the system prompt (rca_sections/). return "\n".join(prompt_parts), build_alert_rail_text(alert_details)