Alert Review

weekly_audit:
  participants:
    - "On-call engineer who handled the alerts"
    - "SRE lead"
    - "Rotating team member"

  review_criteria:
    - question: "Did this alert fire in the past week?"
      action: "If no, consider removing it"

    - question: "Did the on-call engineer take action?"
      action: "If no, consider converting to ticket or dashboard"

    - question: "Did the alert fire multiple times for the same issue?"
      action: "If yes, add grouping or increase duration"

    - question: "Was the alert actionable within 5 minutes?"
      action: "If no, consider increasing the threshold"

    - question: "Did the alert correctly identify the problem?"
      action: "If no, refine the query or threshold"

    - question: "Did the alert have a runbook link?"
      action: "If no, add one before next week"

  output:
    - "List of alerts to delete"
    - "List of alerts to tune"
    - "List of alerts missing runbooks"
    - "List of alerts needing new runbooks"

page_volume_metrics:
  - name: "Total pages per week"
    target: "Decreasing or stable"
    alert: "If increasing for 3 consecutive weeks"

  - name: "Pages per engineer per week"
    target: "< 10"
    alert: "If > 15 for any engineer"

  - name: "Night pages per week (10 PM - 7 AM)"
    target: "< 20% of total"
    alert: "If > 30% of total"

  - name: "Pages per service"
    target: "Proportional to service criticality"
    alert: "If disproportionate to importance"

  - name: "False positive rate"
    target: "< 10%"
    alert: "If > 15%"

  dashboard:
    panels:
      - title: "Page Volume (4-week trend)"
        type: "timeseries"
        query: "sum(increase(fired_alerts_total{severity="critical"}[1w]))"

      - title: "Pages by Team"
        type: "bar_chart"
        query: "sum(increase(fired_alerts_total{severity="critical"}[1w])) by (team)"

      - title: "Pages by Service"
        type: "bar_chart"
        query: "sum(increase(fired_alerts_total{severity="critical"}[1w])) by (service)"

missed_page_tracking:
  definition: "A page not acknowledged within 5 minutes"

  investigation:
    - "Was the on-call engineer available?"
    - "Did the notification reach them?"
    - "Was the alert correctly routed?"
    - "Was the schedule accurate?"

  common_causes:
    - "Schedule not updated after team change"
    - "Notification channel not configured"
    - "Alert routed to wrong team"
    - "Engineer overwhelmed by noise"
    - "Phone issues (Do Not Disturb, airplane mode)"

  prevention:
    - "Weekly schedule verification"
    - "Monthly notification channel testing"
    - "Alert routing quarterly audit"
    - "Noise reduction to prevent fatigue"

  metrics:
    - name: "Missed page rate"
      formula: "missed_pages / total_pages"
      target: "< 2%"
      alert: "If > 5%"

    - name: "Mean time to acknowledge"
      target: "< 5 minutes"
      alert: "If > 10 minutes average"

tuning_action_queue:
  categories:
    delete:
      description: "Alert is not actionable"
      priority: "high"
      sla: "Complete within 1 week"

    tune:
      description: "Alert threshold or duration needs adjustment"
      priority: "medium"
      sla: "Complete within 2 weeks"

    add_runbook:
      description: "Alert is missing a runbook link"
      priority: "high"
      sla: "Complete within 1 week"

    add_grouping:
      description: "Alert fires too many times per incident"
      priority: "medium"
      sla: "Complete within 2 weeks"

    convert_to_ticket:
      description: "Alert should be a scheduled task, not a page"
      priority: "medium"
      sla: "Complete within 1 week"

  tracking:
    - "Use Jira or GitHub Issues"
    - "Tag with 'alert-tuning' label"
    - "Assign owner and due date"
    - "Review completion rate monthly"

  metrics:
    - name: "Tuning action completion rate"
      target: "> 90%"
      frequency: "Monthly"

    - name: "Average time to complete tuning"
      target: "< 7 days"
      frequency: "Monthly"

## Weekly Alert Review: [Date]

### Page Volume Summary
- Total pages: [count]
- Pages per engineer: [count]
- Night pages: [count] ([percentage]%)
- Missed pages: [count]

### Alert Quality Metrics
- Actionable rate: [percentage]%
- False positive rate: [percentage]%
- Mean time to acknowledge: [minutes]
- Mean time to resolve: [minutes]

### Alerts Reviewed
| Alert | Actionable | Action Taken | Owner |
|-------|------------|--------------|-------|
| [name] | Yes/No | [delete/tune/keep] | [name] |

### Tuning Actions
| ID | Description | Category | Owner | Due |
|----|-------------|----------|-------|-----|
| [id] | [desc] | [cat] | [name] | [date] |

### Trends
- [Observation about page volume trend]
- [Observation about alert quality trend]