From Alert Fatigue to Autonomous Remediation: Building the Modern AI SRE Stack

Alert fires
    ↓
PagerDuty pages on-call engineer
    ↓
Engineer wakes up, opens blog post
    ↓
Engineer manually gathers context (logs, metrics, recent deploys)
    ↓
Engineer diagnoses the problem (20-40 minutes)
    ↓
Engineer remediates or escalates
    ↓
Engineer writes incident summary
    ↓
Engineer goes back to sleep (or not)

Alert fires
    ↓
AI triage agent evaluates severity and category
    ↓
    ├── Known, automatable → AI remediates, notifies human, done
    ├── Known, needs approval → AI diagnoses, proposes fix, human approves, done
    └── Novel, needs human → AI pre-investigates, human wakes up with full context

# triage_agent.py
TRIAGE_PROMPT = """
You are an SRE triage agent. Your job is to classify incoming alerts and 
determine the correct response path.

Given an alert, classify it as:
1. AUTO_REMEDIATE: Known issue type with a safe, tested automated fix
2. DIAGNOSE_FIRST: Needs investigation before action; human should see the 
   diagnosis before waking up
3. PAGE_NOW: Novel, high-severity, or ambiguous; wake the human immediately
   with whatever context you can gather quickly

For AUTO_REMEDIATE, specify the exact remediation action.
For DIAGNOSE_FIRST, specify what investigation to run.
For PAGE_NOW, summarize what you know in 3 sentences.

Always include your confidence level (high/medium/low) and reasoning.
"""

async def triage_alert(alert: dict) -> TriageDecision:
    # Enrich with context before classification
    recent_alerts = get_similar_recent_alerts(alert, lookback_hours=24)
    service_health = get_service_health_snapshot(alert['service'])
    recent_deployments = get_recent_deployments(alert['service'], hours=4)
    
    context = {
        "alert": alert,
        "similar_recent_alerts": recent_alerts,
        "service_health": service_health,
        "recent_deployments": recent_deployments
    }
    
    response = await claude.messages.create(
        model="claude-haiku-4-5-20251001",  # Fast/cheap for triage
        max_tokens=1024,
        system=TRIAGE_PROMPT,
        messages=[{
            "role": "user", 
            "content": f"Triage this alert: {json.dumps(context)}"
        }]
    )
    
    return parse_triage_decision(response)

REMEDIATION_REGISTRY = {
    "pod_crashloop": {
        "action": restart_pod,
        "safety_check": lambda ctx: ctx['pod_restart_count_1h'] < 3,
        "rollback": None,  # Restart is inherently reversible
        "cooldown_minutes": 5,
        "notify_channel": "#sre-alerts"
    },
    "high_memory_service": {
        "action": trigger_rolling_restart,
        "safety_check": lambda ctx: ctx['traffic_level'] < 0.8,  # Don't restart at peak
        "rollback": None,
        "cooldown_minutes": 30,
        "notify_channel": "#sre-alerts"
    },
    "stuck_batch_job": {
        "action": terminate_and_resubmit_job,
        "safety_check": lambda ctx: ctx['job_idempotent'] == True,
        "rollback": cancel_resubmitted_job,
        "cooldown_minutes": 60,
        "notify_channel": "#data-engineering"
    },
    "kinesis_throttling": {
        "action": scale_kinesis_shards,
        "safety_check": lambda ctx: ctx['current_shard_count'] < ctx['max_shard_limit'],
        "rollback": scale_kinesis_shards,  # Same function, fewer shards
        "cooldown_minutes": 15,
        "notify_channel": "#sre-alerts"
    }
}

async def execute_remediation(alert: dict, action: str, triage_reasoning: str):
    playbook = REMEDIATION_REGISTRY.get(action)
    if not playbook:
        raise ValueError(f"Unknown remediation action: {action}")
    
    context = gather_action_context(alert)
    
    # Safety check before executing
    if not playbook['safety_check'](context):
        # Safety check failed — escalate instead
        await page_engineer(alert, reason="Safety check failed for auto-remediation")
        return
    
    # Execute with full audit trail
    execution_id = generate_execution_id()
    log_remediation_start(execution_id, alert, action, triage_reasoning)
    
    try:
        result = await playbook['action'](alert, context)
        log_remediation_success(execution_id, result)
        await notify_slack(playbook['notify_channel'], 
                          f"Auto-remediated: {alert['description']}\n"
                          f"Action: {action}\n"
                          f"Reasoning: {triage_reasoning}\n"
                          f"Execution ID: {execution_id}")
    except Exception as e:
        log_remediation_failure(execution_id, e)
        await page_engineer(alert, reason=f"Auto-remediation failed: {e}")

🔍 INCIDENT INVESTIGATION — order-service latency spike (P95 >2s)

DIAGNOSIS (High Confidence):
The latency increase started at 14:23 UTC and correlates with a deployment 
of order-service v2.4.1 at 14:20 UTC. The deployment changed the 
database query for order retrieval — the new query is performing a full 
table scan on large orders (>50 line items) due to a missing index.

EVIDENCE:
• P95 latency: 240ms → 2,100ms starting 14:23
• order-service v2.4.1 deployed at 14:20 (git: abc123)
• DB slow query log: SELECT * FROM order_items WHERE order_id=? 
  showing full scans for orders with >50 items (affecting ~3% of orders)
• Error rate: unchanged (latency issue, not error issue)

RECOMMENDED ACTIONS:
1. [FAST] Roll back to v2.4.0 to restore service — estimated 5 min
2. [PROPER FIX] Add index: CREATE INDEX idx_order_items_order_id 
   ON order_items(order_id) — coordinate with DB team for maintenance window

RISK FLAGS:
• Rollback is safe — no schema changes in v2.4.1
• 3% of active orders are affected (heavy orders with >50 items)

Investigated by: SRE-Agent | Investigation time: 2m 14s | 
Tokens used: 8,240 | Confidence: High

# After incident resolves, capture outcome
async def record_incident_outcome(
    investigation_id: str,
    actual_root_cause: str,
    agent_diagnosis_correct: bool,
    time_to_resolution_minutes: int,
    human_notes: str
):
    outcome = {
        "investigation_id": investigation_id,
        "agent_diagnosis": get_investigation_diagnosis(investigation_id),
        "actual_root_cause": actual_root_cause,
        "correct": agent_diagnosis_correct,
        "ttd_minutes": time_to_resolution_minutes,
        "notes": human_notes,
        "timestamp": datetime.utcnow().isoformat()
    }
    
    # Store for analysis
    store_outcome_to_s3(outcome)
    
    # Update metrics
    increment_metric("agent.diagnosis_total")
    if agent_diagnosis_correct:
        increment_metric("agent.diagnosis_correct")
    
    # Feed back into agent training data
    if not agent_diagnosis_correct:
        flag_for_prompt_review(investigation_id, actual_root_cause)