Move RCA behavioral prompts to system prompt

server/chat/backend/agent/prompt/composer.py

@@ -22,34 +22,43 @@
 )
 from .schema import PromptSegments
 
+_RCA_SECTIONS_DIR = os.path.normpath(
+    os.path.join(os.path.dirname(__file__), "rca_sections")
+)
 
-def build_system_invariant(is_background: bool = False) -> str:
-    """Load core system prompt from modular markdown files under skills/core/.
+_RCA_SECTION_ORDER = [
+    "identity",
+    "investigation",
+    "context_mgmt",
+    "error_recovery",
+    "evidence_standard",
+    "conclusion_gate",
+]
 
-    Segments are loaded in a fixed order that mirrors the original monolithic
-    prompt so that cached prefixes remain stable across deployments.
 
-    In background RCA mode, Terraform/IaC, SSH setup, and cloud CLI discovery
-    segments are omitted (~3,300 tokens) since background investigations are
-    read-only and the freed budget is better spent on integration skills.
-    """
-    from chat.backend.agent.skills.loader import load_core_prompt
+def _build_rca_system_prompt() -> str:
+    """Assemble the background RCA system prompt from rca_sections/*.md."""
+    parts = []
+    for name in _RCA_SECTION_ORDER:
+        path = os.path.join(_RCA_SECTIONS_DIR, f"{name}.md")
+        with open(path, "r", encoding="utf-8") as f:
+            content = f.read().strip()
+        if content:
+            parts.append(content)
+
+    return "\n\n".join(parts)
 
-    core_dir = os.path.join(
-        os.path.dirname(__file__), os.pardir, "skills", "core"
-    )
-    core_dir = os.path.normpath(core_dir)
+
+def build_system_invariant(is_background: bool = False) -> str:
+    """Build the system prompt for interactive or background RCA mode."""
+    from chat.backend.agent.skills.loader import load_core_prompt
 
     if is_background:
-        return load_core_prompt(core_dir, segments=[
-            "identity",
-            "security",
-            "knowledge_base",
-            "error_handling",
-            "investigation",
-            "behavioral_rules",
-        ])
+        return _build_rca_system_prompt()
 
+    core_dir = os.path.normpath(
+        os.path.join(os.path.dirname(__file__), os.pardir, "skills", "core")
+    )
     return load_core_prompt(core_dir, segments=[
         "identity",
         "security",

server/chat/backend/agent/prompt/rca_sections/conclusion_gate.md

@@ -0,0 +1,14 @@
+# Before Concluding
+
+You will conclude too early. Recognize these traps:
+- "The timing correlates": correlation is not causation. Find the mechanism.
+- "This is the most common cause": common does not mean actual for THIS incident.
+- "I found one log line that matches": one data point is not a pattern.
+- "The service restarted, so resource exhaustion": check actual resource metrics.
+- "We need to scale up resources": that's a band-aid, not a root cause. Why are resources insufficient now? Did something change or was it always underprovisioned?
+- "The cluster is unstable": what specifically is making it unstable? Which node, which component, what changed?
+
+Before stating root cause, answer:
+1. What alternative did you rule out, and how?
+2. What specific evidence (tool output) proves the mechanism, not just the correlation?
+3. Does your root cause explain the timing of the alert?

server/chat/backend/agent/prompt/rca_sections/context_mgmt.md

@@ -0,0 +1,5 @@
+# Recording Findings
+
+After each tool call, write down what you found before making the next call. Earlier tool results may be cleared from context. If a finding only exists in a tool result you didn't record, it's gone.
+
+Be concise: what you found, what it means, what you'll check next. Nothing else.

server/chat/backend/agent/prompt/rca_sections/error_recovery.md

@@ -0,0 +1,9 @@
+# Error Recovery
+
+When a tool call returns empty or errors:
+- Broaden the time window (-2h, -6h, -24h)
+- Simplify the query (fewer filters, broader match)
+- Try a different resource type (metrics instead of logs, events instead of traces)
+- Verify the service/host name exists by listing available resources
+
+Empty results are data as they rule things out. Do not repeat a failed query unchanged. If all avenues are exhausted, state what was ruled out and stop.

server/chat/backend/agent/prompt/rca_sections/evidence_standard.md

@@ -0,0 +1,7 @@
+# Evidence
+
+Never state a root cause without citing the specific tool output that proves it. Exact timestamps, error messages, metric values.
+
+If you cannot determine root cause, say what you confirmed, what you ruled out, and what remains unverified. Distinguish facts from hypotheses in your reporting.
+
+Do not fabricate log lines, metrics, or timestamps. Do not hedge confirmed findings or overclaim uncertain ones. Match your confidence to the evidence.

server/chat/backend/agent/prompt/rca_sections/identity.md

@@ -0,0 +1,3 @@
+# Identity
+
+You are Aurora, an SRE agent investigating a production alert. Your goal is not just to name the root cause. It is to understand the incident deeply enough to remediate it, prevent recurrence, and write the post-mortem.

server/chat/backend/agent/prompt/rca_sections/investigation.md

@@ -0,0 +1,9 @@
+# Investigation
+
+Before your first tool call, state your hypothesis and what you will query to test it.
+
+Work from the outside in. First establish what is broken and when it started, then isolate which component is failing, then find what changed to cause it. Something changed. A deploy, a config, a dependency, traffic, resources. Find that change.
+
+A symptom is not a root cause. "The pod is OOMKilled" is a symptom. "Memory leak in the request parser introduced in commit X" is a root cause. "The pod needs more resources" is not specific enough. Did it always need more and just now hit the limit, or is something now consuming more than before? If consumption changed, what changed it? "The cluster is unstable" is not specific. Which component, which node, what changed? Keep drilling until you reach something specific and actionable.
+
+Design queries to disprove your hypothesis, not confirm it. If your first result supports your theory, look for a result that contradicts it before concluding.

server/chat/background/rca_prompt_builder.py

@@ -1,33 +1,20 @@
 """
 Shared RCA (Root Cause Analysis) prompt builder for background alert processing.
 
-This module creates provider-aware, persistence-focused RCA prompts that leverage
-all available tools and follow the detailed investigation guidelines in the system prompt.
+This module creates provider-aware RCA prompts that pair alert data with
+infrastructure context. Behavioral investigation guidance lives in the system
+prompt (rca_sections/). The user message contains only alert facts and context.
 
 Aurora Learn Integration:
 - When Aurora Learn is enabled, searches for similar past incidents with positive feedback
 - Injects context from helpful RCAs to improve new investigations
 """
 
-from functools import lru_cache
 from typing import Any, Dict, List, Optional
 import logging
-import os
 
 logger = logging.getLogger(__name__)
 
-RCA_SEGMENTS_DIR = os.path.normpath(
-    os.path.join(
-        os.path.dirname(__file__),
-        os.pardir,
-        "backend",
-        "agent",
-        "skills",
-        "rca",
-        "segments",
-    )
-)
-
 
 def build_alert_rail_text(alert_details: Dict[str, Any]) -> str:
     """Extract the webhook-authored subset of an alert for input-rail evaluation.
@@ -52,59 +39,6 @@
     return "\n\n".join(parts)
 
 
-@lru_cache(maxsize=32)
-def _load_rca_segment_template(segment_name: str) -> str:
-    """
-    Load an RCA markdown segment by name (filename without .md).
-
-    Segment content is cached in-process for performance.
-    """
-    try:
-        from chat.backend.agent.skills.loader import load_core_prompt
-
-        return load_core_prompt(RCA_SEGMENTS_DIR, segments=[segment_name]).strip()
-    except Exception as e:
-        logger.warning(f"Failed to load RCA segment '{segment_name}': {e}")
-        return ""
-
-
-def _render_rca_segment(segment_name: str, context: Optional[Dict[str, Any]] = None) -> str:
-    """Render an RCA segment with optional {variable} template substitutions."""
-    template = _load_rca_segment_template(segment_name)
-    if not template:
-        return ""
-
-    if not context:
-        return template
-
-    try:
-        from chat.backend.agent.skills.loader import resolve_template
-
-        return resolve_template(template, context)
-    except Exception as e:
-        logger.warning(f"Failed to render RCA segment '{segment_name}': {e}")
-        return template
-
-
-def _append_rca_segment(
-    prompt_parts: List[str],
-    segment_name: str,
-    context: Optional[Dict[str, Any]] = None,
-    leading_blank: bool = False,
-    trailing_blank: bool = False,
-) -> None:
-    """Append rendered segment to prompt_parts with optional surrounding blank lines."""
-    content = _render_rca_segment(segment_name, context=context)
-    if not content:
-        return
-
-    if leading_blank:
-        prompt_parts.append("")
-    prompt_parts.append(content)
-    if trailing_blank:
-        prompt_parts.append("")
-
-
 # ============================================================================
 # Aurora Learn - Similar RCA Context Injection
 # ============================================================================
@@ -510,7 +444,7 @@
         providers = get_user_providers(user_id)
 
     providers = providers or []
     providers_lower = [p.lower() for p in providers]
 
     # Derive integrations from skill registry when not passed by caller
     if integrations is None and user_id:
@@ -627,11 +561,6 @@
         f"You have access to: {', '.join(providers) if providers else 'No cloud/monitoring providers connected'}",
     ])
 
-    # All integration guidance (GitHub, Jira, Confluence, Jenkins, CloudBees,
-    # provider investigation commands) loaded from skill files via SkillRegistry
-    # in the system prompt (background.py). No skill loading here — the user
-    # message should contain only alert details and investigation context.
-
     # Aurora Learn: Inject context from similar past incidents
     if user_id:
         similar_context = _get_similar_good_rcas_context(
@@ -653,59 +582,8 @@
         if prediscovery_context:
             prompt_parts.append(prediscovery_context)
 
-    # Critical investigation requirements (modular markdown segments)
-    _append_rca_segment(
-        prompt_parts,
-        "critical_requirements_header",
-        leading_blank=True,
-        trailing_blank=True,
-    )
-
-    has_infra_providers = bool({'gcp', 'aws', 'azure', 'ovh', 'scaleway'}.intersection(set(providers_lower)))
-    has_jira = bool((integrations or {}).get('jira'))
-    has_confluence = bool((integrations or {}).get('confluence'))
-    after_context_label = 'Jira' if has_jira else 'Confluence' if has_confluence else 'change'
-
-    # Add aggressive persistence prompts only if cost optimization is disabled
-    # The immediate action required due to the AgentExecutor which assumes agent is done when it sends a text chunk without a tool call.
-    if os.getenv("RCA_OPTIMIZE_COSTS", "").lower() != "true":
-        _append_rca_segment(
-            prompt_parts,
-            "persistence_and_immediate_action",
-            context={"after_context_label": after_context_label},
-            trailing_blank=True,
-        )
-
-    depth_steps = []
-    if has_jira or has_confluence:
-        depth_steps.append("**Search Jira/Confluence first** for recent changes, open bugs, and runbooks")
-    depth_steps.extend([
-        "Start broad - understand the overall system state",
-        "Identify the affected component(s)",
-        "Drill down into specifics - logs, metrics, configurations",
-        "Check related/dependent resources",
-        "Look for recent changes that correlate with the issue",
-    ])
-    if has_infra_providers:
-        depth_steps.extend([
-            "Compare with healthy resources of the same type",
-            "Check resource quotas, limits, and constraints",
-            "Examine network connectivity and security rules",
-            "Verify IAM permissions and service accounts",
-        ])
-    depth_steps.append("Review historical patterns if available")
-    prompt_parts.append("### INVESTIGATION DEPTH:")
-    for i, step in enumerate(depth_steps, 1):
-        prompt_parts.append(f"{i}. {step}")
-
-    _append_rca_segment(prompt_parts, "error_resilience_intro", leading_blank=True)
-    if has_infra_providers:
-        _append_rca_segment(prompt_parts, "error_resilience_infra")
-    _append_rca_segment(prompt_parts, "error_resilience_outro")
-
-    _append_rca_segment(prompt_parts, "what_to_investigate", leading_blank=True)
-    _append_rca_segment(prompt_parts, "output_requirements", leading_blank=True)
-
+    # User message contains only alert data + context.
+    # All behavioral guidance lives in the system prompt (rca_sections/).
     return "\n".join(prompt_parts), build_alert_rail_text(alert_details)