Recovery Metrics

recovery_time_tracking:
  metrics:
    - name: "Time to Detect"
      description: "Time from disaster occurrence to detection"
      target: "< 5 minutes"
      measurement: "disaster_time - detection_time"

    - name: "Time to Declare"
      description: "Time from detection to DR declaration"
      target: "< 15 minutes"
      measurement: "detection_time - declaration_time"

    - name: "Time to Activate"
      description: "Time from declaration to secondary region activation"
      target: "< 30 minutes"
      measurement: "declaration_time - activation_time"

    - name: "Time to Verify"
      description: "Time from activation to verification"
      target: "< 15 minutes"
      measurement: "activation_time - verification_time"

    - name: "Total Recovery Time"
      description: "Total time from disaster to full recovery"
      target: "Within RTO"
      measurement: "disaster_time - recovery_time"

  dashboard:
    panels:
      - title: "Recovery Time Components"
        type: "bar_chart"
        metrics:
          - "Time to Detect"
          - "Time to Declare"
          - "Time to Activate"
          - "Time to Verify"

      - title: "Recovery Time Trend"
        type: "timeseries"
        query: "total_recovery_time by (exercise_id)"

      - title: "RTO Compliance"
        type: "stat"
        query: "recovery_time <= rto_target"
        target: "100%"

data_freshness:
  metrics:
    - name: "Replication Lag"
      description: "Time delay between primary and secondary database"
      target: "< 1 second for synchronous, < 5 minutes for async"
      measurement: "primary_timestamp - secondary_timestamp"

    - name: "Data Loss Window"
      description: "Maximum data that could be lost"
      target: "Within RPO"
      measurement: "primary_timestamp - last_replicated_timestamp"

    - name: "Backup Freshness"
      description: "Time since last successful backup"
      target: "< 24 hours"
      measurement: "now() - last_backup_timestamp"

    - name: "Data Integrity Score"
      description: "Percentage of records that pass integrity checks"
      target: "100%"
      measurement: "valid_records / total_records * 100"

  monitoring:
    replication_lag:
      alert: "If lag > 5 minutes for async, > 1s for sync"
      dashboard: "https://grafana.internal/d/database-replication"

    backup_freshness:
      alert: "If no backup in 24 hours"
      dashboard: "https://grafana.internal/d/backup-status"

manual_step_tracking:
  purpose: "Track manual intervention required during DR"

  metrics:
    - name: "Manual Steps Count"
      description: "Number of manual steps required during DR"
      target: "Decreasing trend"
      tracking: "Per exercise"

    - name: "Automation Coverage"
      description: "Percentage of DR steps that are automated"
      target: "> 80%"
      calculation: "automated_steps / total_steps * 100"

    - name: "Mean Time per Manual Step"
      description: "Average time to complete each manual step"
      target: "< 5 minutes"
      measurement: "sum(manual_step_times) / count(manual_steps)"

  common_manual_steps:
    - "Database promotion"
    - "DNS update"
    - "Certificate rotation"
    - "Configuration update"
    - "Service restart"

  automation_opportunities:
    - "Database promotion: automate with operator"
    - "DNS update: automate with Terraform"
    - "Certificate rotation: automate with cert-manager"
    - "Configuration update: automate with ConfigMap sync"
    - "Service restart: automate with rollout"

failed_control_tracking:
  purpose: "Track DR controls that failed during exercises or real events"

  control_categories:
    - "Health checks"
    - "Failover automation"
    - "Data replication"
    - "DNS failover"
    - "Backup restoration"
    - "Monitoring and alerting"

  tracking:
    - control: "Health check failed to detect outage"
      impact: "Delayed failover"
      remediation: "Improve health check sensitivity"

    - control: "DNS failover did not propagate"
      impact: "Users still directed to failed region"
      remediation: "Reduce TTL, verify DNS provider SLA"

    - control: "Database replication lag exceeded RPO"
      impact: "Data loss during failover"
      remediation: "Upgrade to synchronous replication"

  metrics:
    - name: "Control Failure Rate"
      formula: "failed_controls / total_controls"
      target: "< 5%"
      frequency: "Per exercise"

    - name: "Mean Time to Detect Control Failure"
      formula: "avg(detection_time - failure_time)"
      target: "< 5 minutes"
      frequency: "Per exercise"

remediation_tracking:
  purpose: "Track improvements to DR capability based on exercise findings"

  process:
    - "Identify gaps during exercise"
    - "Create remediation action items"
    - "Assign owners and due dates"
    - "Track completion"
    - "Verify improvement in next exercise"

  metrics:
    - name: "Remediation Completion Rate"
      formula: "completed_rem / total_rem"
      target: "> 90%"
      frequency: "Monthly"

    - name: "DR Readiness Score"
      formula: "weighted_score of all DR metrics"
      target: "> 80%"
      frequency: "Quarterly"

    - name: "Exercise-to-Exercise Improvement"
      description: "Improvement in recovery time between exercises"
      target: "Positive trend"
      frequency: "Quarterly"

  quarterly_review:
    - "Review all DR exercise results"
    - "Identify systemic gaps"
    - "Prioritize remediation actions"
    - "Update DR roadmap"
    - "Report to leadership"