Continuity Exercises

tabletop_scenarios:
  scenario_1:
    name: "Complete region failure"
    description: "AWS us-east-1 region becomes completely unavailable"
    participants: "SRE, Engineering, Product, Executive"
    discussion_points:
      - "How do we declare DR?"
      - "Who activates secondary region?"
      - "How do we communicate to customers?"
      - "What data might be lost?"
      - "When do we attempt failback?"

  scenario_2:
    name: "Database corruption"
    description: "Production database has corrupted data from bad migration"
    participants: "SRE, Database team, Engineering"
    discussion_points:
      - "When do we stop writes?"
      - "How do we restore from backup?"
      - "How do we handle data written after corruption?"
      - "What is the data loss window?"

  scenario_3:
    name: "CDN provider failure"
    description: "Primary CDN provider has global outage"
    participants: "SRE, Infrastructure, Product"
    discussion_points:
      - "How do we switch to backup CDN?"
      - "How do we handle cached content?"
      - "What is the user impact?"
      - "How do we communicate the issue?"

  scenario_4:
    name: "Security breach"
    description: "Confidential data exposed due to misconfiguration"
    participants: "SRE, Security, Legal, Executive"
    discussion_points:
      - "How do we contain the breach?"
      - "When do we notify affected users?"
      - "How do we notify regulators?"
      - "How do we preserve evidence?"

regional_evacuation:
  scenario: "Evacuate us-east-1 to eu-west-1"
  duration: "4 hours"
  participants: "SRE, Database, Infrastructure"

  steps:
    - step: "Preparation (30 min)"
      actions:
        - "Verify secondary region is healthy"
        - "Verify database replication is current"
        - "Open DR war room"
        - "Notify stakeholders"

    - step: "Failover (60 min)"
      actions:
        - "Stop writes to primary database"
        - "Verify replication lag is zero"
        - "Promote secondary database"
        - "Update DNS to secondary region"
        - "Verify traffic flowing to secondary"

    - step: "Validation (60 min)"
      actions:
        - "Test all critical user journeys"
        - "Verify data integrity"
        - "Monitor error rates and latency"
        - "Verify backup systems working"

    - step: "Communication (30 min)"
      actions:
        - "Update status page"
        - "Notify customers"
        - "Update leadership"

  success_criteria:
    - "Failover completes within RTO"
    - "No data loss (RPO = 0)"
    - "All services operational in secondary region"
    - "Communication sent within 15 minutes"

  debrief:
    timing: "Within 24 hours"
    participants: "All drill participants"
    output: "Improvement actions and runbook updates"

supplier_failure_scenarios:
  cloud_provider_outage:
    description: "Major cloud provider has regional outage"
    impact: "All services in affected region down"
    mitigation: "Multi-region failover"
    exercise: "Regional evacuation drill"

  cdn_outage:
    description: "CDN provider has global outage"
    impact: "Static assets unavailable, slow page loads"
    mitigation: "Fallback to origin, backup CDN"
    exercise: "CDN failover drill"

  dns_provider_outage:
    description: "DNS provider has outage"
    impact: "All services unreachable by name"
    mitigation: "Anycast DNS, backup DNS provider"
    exercise: "DNS failover drill"

  payment_provider_outage:
    description: "Payment processor is down"
    impact: "Cannot process payments"
    mitigation: "Backup payment provider"
    exercise: "Payment failover drill"

  database_provider_outage:
    description: "Managed database service is down"
    impact: "All database-dependent services fail"
    mitigation: "Self-managed database, cross-region replica"
    exercise: "Database failover drill"

executive_escalation:
  triggers:
    - "DR declared"
    - "DR exceeds RTO"
    - "Data loss detected"
    - "Customer data compromised"
    - "Regulatory notification required"

  escalation_path:
    - level: 1
      contact: "SRE Manager"
      timing: "Immediate on DR declaration"
      channel: "PagerDuty + Phone"

    - level: 2
      contact: "VP Engineering"
      timing: "Within 15 minutes"
      channel: "Phone"

    - level: 3
      contact: "CTO"
      timing: "Within 30 minutes"
      channel: "Phone"

    - level: 4
      contact: "CEO"
      timing: "Within 1 hour"
      channel: "Phone"

  executive_briefing_template:
    - "What happened: [1-2 sentences]"
    - "Current status: [one sentence]"
    - "Impact: [users, revenue, duration]"
    - "What we are doing: [current actions]"
    - "What we need: [decisions, resources]"
    - "Next update: [time]"

exercise_schedule:
  quarterly:
    Q1:
      exercise: "Tabletop — region failure scenario"
      participants: "SRE, Engineering, Product"
      duration: "2 hours"

    Q2:
      exercise: "Functional — database failover drill"
      participants: "SRE, Database team"
      duration: "4 hours"

    Q3:
      exercise: "Tabletop — security breach scenario"
      participants: "SRE, Security, Legal, Executive"
      duration: "2 hours"

    Q4:
      exercise: "Full-scale — regional evacuation drill"
      participants: "All teams"
      duration: "8 hours"

  monthly:
    - "Backup restore test"
    - "DR runbook review"
    - "Access verification"

  tracking:
    - "Exercise completion rate: target 100%"
    - "Action item completion rate: target > 90%"
    - "DR readiness score: target > 80%"