Dependency Failure Testing

dependency_failures:
  timeout:
    description: "Dependency does not respond within expected time"
    symptoms: "Increased latency, thread pool exhaustion"
    resilience_patterns: "Timeouts, bulkheads, fallbacks"

  connection_refused:
    description: "Dependency rejects connections"
    symptoms: "Connection errors, immediate failures"
    resilience_patterns: "Circuit breakers, retries with backoff"

  dns_failure:
    description: "Cannot resolve dependency hostname"
    symptoms: "DNS timeout, connection failures"
    resilience_patterns: "DNS caching, fallback IPs, health checks"

  partial_failure:
    description: "Dependency returns errors for some requests"
    symptoms: "Intermittent errors, inconsistent behavior"
    resilience_patterns: "Retries, circuit breakers, fallbacks"

  saturation:
    description: "Dependency is overloaded and slow"
    symptoms: "Increased latency, timeout errors"
    resilience_patterns: "Rate limiting, load shedding, backpressure"

  data_corruption:
    description: "Dependency returns incorrect data"
    symptoms: "Incorrect results, application errors"
    resilience_patterns: "Validation, checksums, fallback data"

timeout_experiment:
  hypothesis: "If downstream latency exceeds 5s, then timeouts
    prevent thread pool exhaustion"

  setup:
    - "Deploy application with 3s timeout to downstream"
    - "Inject 10s latency to downstream using Chaos Mesh"
    - "Send 100 requests per second"

  expected_behavior:
    - "Requests timeout after 3s"
    - "Thread pool does not exhaust"
    - "Application continues serving other requests"
    - "Error rate increases but remains below 10%"

  abort_criteria:
    - "Error rate exceeds 50%"
    - "Application crashes or becomes unresponsive"
    - "Thread pool exhaustion detected"

  validation_queries:
    error_rate: |
      sum(rate(http_request_duration_seconds_count{code=~"5.."}[1m]))
      /
      sum(rate(http_request_duration_seconds_count[1m]))
    thread_pool_usage: |
      application_thread_pool_active / application_thread_pool_max
    latency_p99: |
      histogram_quantile(0.99,
        sum(rate(http_request_duration_seconds_bucket[1m])) by (le)
      )

retry_experiment:
  hypothesis: "If downstream fails intermittently, then retries
    with exponential backoff recover without overwhelming
    the downstream"

  setup:
    - "Configure application with 3 retries, exponential backoff"
    - "Inject 30% failure rate to downstream"
    - "Send 100 requests per second"

  expected_behavior:
    - "Retries recover ~90% of failed requests"
    - "Downstream is not overwhelmed by retries"
    - "Overall error rate stays below 5%"
    - "Retry storm does not occur"

  abort_criteria:
    - "Downstream becomes unavailable due to retry storm"
    - "Error rate exceeds 20%"
    - "Application thread pool exhausts"

  retry_storm_detection:
    metric: |
      rate(http_request_duration_seconds_count{code=~"5.."}[1m])
      /
      rate(http_request_duration_seconds_count[1m])
    threshold: 0.3
    description: "If retry rate exceeds 30%, retry storm may be occurring"

circuit_breaker_experiment:
  hypothesis: "If downstream fails consistently, then circuit
    breaker opens and serves fallback responses"

  setup:
    - "Configure circuit breaker: open after 5 failures in 10s"
    - "Configure circuit breaker: half-open after 30s"
    - "Configure fallback: return cached response"
    - "Inject 100% failure rate to downstream"

  expected_behavior:
    - "Circuit breaker opens within 10 seconds"
    - "Fallback responses served after circuit opens"
    - "Error rate drops after circuit opens (fallback working)"
    - "Circuit breaker half-opens after 30 seconds"
    - "If downstream recovers, circuit breaker closes"

  phases:
    phase_1_closed:
      duration: "0-10 seconds"
      expected: "Circuit closed, requests fail"

    phase_2_open:
      duration: "10-40 seconds"
      expected: "Circuit open, fallback responses served"

    phase_3_half_open:
      duration: "40-70 seconds"
      expected: "Circuit half-open, test requests sent"

    phase_4_closed:
      duration: "70+ seconds"
      expected: "Circuit closed if downstream recovered"

  validation_queries:
    circuit_state: |
      application_circuit_breaker_state
    fallback_rate: |
      sum(rate(http_request_total{response_source="fallback"}[1m]))
      /
      sum(rate(http_request_total[1m]))

dns_failure_experiment:
  hypothesis: "If DNS resolution fails, then application uses
    cached DNS entries and continues serving requests"

  setup:
    - "Block DNS traffic to kube-dns for target service"
    - "Verify DNS cache has valid entries"
    - "Send traffic to application"

  expected_behavior:
    - "Application uses cached DNS entries"
    - "Requests continue to succeed for cache TTL duration"
    - "After cache expires, errors increase"
    - "After DNS restored, requests recover"

  abort_criteria:
    - "Application becomes completely unavailable"
    - "DNS restoration does not fix the issue"

  dns_cache_validation:
    metric: |
      application_dns_cache_entries
    expected: "> 0 entries for target service"

downstream_saturation_experiment:
  hypothesis: "If downstream is saturated, then rate limiting
    and load shedding prevent cascading failure"

  setup:
    - "Configure downstream to accept max 100 concurrent connections"
    - "Configure application rate limiter: 50 requests/second"
    - "Send 200 requests per second to application"

  expected_behavior:
    - "Rate limiter rejects excess requests"
    - "Application does not cascade failure to downstream"
    - "Accepted requests succeed"
    - "Rejected requests return 429 (Too Many Requests)"

  abort_criteria:
    - "Downstream becomes completely unavailable"
    - "Application crashes due to connection exhaustion"

  cascading_failure_detection:
    metric: |
      rate(http_request_duration_seconds_count{code="503"}[1m])
    threshold: 0.5
    description: "If 503 rate exceeds 50%, cascading failure may be occurring"