Game Day Planning

scenario:
  name: "Database failover during peak traffic"
  duration: "90 minutes"
  difficulty: "medium"

  objective: "Test database failover procedure and team coordination"

  pre_conditions:
    - "Staging environment healthy"
    - "Primary and replica databases running"
    - "Application serving traffic normally"
    - "All participants available"

  timeline:
    - time: "0:00"
      event: "Facilitator announces scenario start"
      action: "Attacker begins injecting failure"

    - time: "0:02"
      event: "Primary database becomes unavailable"
      action: "Attacker stops primary database pod"

    - time: "0:03"
      event: "Alerts fire"
      action: "Monitor alertmanager for response"

    - time: "0:05"
      event: "Expected: IC declares incident"
      action: "Observer notes time to declaration"

    - time: "0:10"
      event: "Expected: Team begins investigation"
      action: "Observer monitors war room activity"

    - time: "0:20"
      event: "Expected: Root cause identified"
      action: "Observer notes time to identification"

    - time: "0:30"
      event: "Expected: Remediation begins"
      action: "Observer monitors remediation steps"

    - time: "0:45"
      event: "Expected: Service restored"
      action: "Observer verifies recovery"

  success_criteria:
    - "Incident declared within 5 minutes"
    - "IC assigned within 10 minutes"
    - "Root cause identified within 20 minutes"
    - "Service restored within 45 minutes"
    - "All communication channels used correctly"

  abort_criteria:
    - "Real production impact detected"
    - "Staging environment becomes unstable"
    - "Participant requests abort"
    - "Duration exceeds 90 minutes"

participant_roles:
  facilitator:
    responsibilities:
      - "Run the exercise"
      - "Control the timeline"
      - "Make abort decisions"
      - "Lead the debrief"
    qualifications: "Experienced SRE, not participating as responder"

  attacker:
    responsibilities:
      - "Inject the failure"
      - "Respond to facilitator commands"
      - "Stop failure when told"
    qualifications: "Familiar with failure injection tools"

  responder:
    responsibilities:
      - "Respond as if real incident"
      - "Follow incident process"
      - "Document actions taken"
    qualifications: "On-call engineer or backup"

  observer:
    responsibilities:
      - "Take notes without participating"
      - "Track timeline"
      - "Record decisions and actions"
      - "Identify improvement areas"
    qualifications: "SRE with incident response experience"

  scribe:
    responsibilities:
      - "Document timeline in real-time"
      - "Record decisions and rationale"
      - "Capture questions and confusion"
    qualifications: "Detail-oriented, not participating as responder"

communication_channels:
  war_room:
    name: "#gameday-YYYY-MM-DD-description"
    purpose: "Real-time coordination"
    participants: "All game day participants"

  observer_notes:
    name: "#gameday-observer-notes"
    purpose: "Observer documentation"
    participants: "Observers and scribe only"

  status_updates:
    name: "#gameday-status"
    purpose: "Simulated status updates"
    participants: "Communications role player"

  facilitator_commands:
    name: "Facilitator direct message"
    purpose: "Commands to attacker and observer"
    participants: "Facilitator, attacker, observer"

guardrails:
  environment:
    - "Game days run in staging only"
    - "Never inject failure in production without approval"
    - "Verify staging environment health before starting"

  blast_radius:
    - "Maximum 1 service affected"
    - "Maximum 25% of pods affected"
    - "Maximum 30 minute failure duration"

  monitoring:
    - "Monitor production for spillover effects"
    - "Have rollback command ready"
    - "Abort if any production impact detected"

  approval:
    staging: "SRE lead approval"
    production: "VP Engineering approval"
    high_risk: "CTO approval"

debrief_process:
  timing: "Within 24 hours of game day"
  duration: "60 minutes"
  participants: "All game day participants"

  agenda:
    - time: "0-10 min"
      item: "Facilitator presents timeline"
    - time: "10-20 min"
      item: "Review performance against success criteria"
    - time: "20-35 min"
      item: "What worked well"
    - time: "35-50 min"
      item: "What needs improvement"
    - time: "50-60 min"
      item: "Assign action items and schedule next game day"

  questions:
    - "Did we detect the issue quickly?"
    - "Did we follow the incident process?"
    - "Was communication effective?"
    - "Were the right people involved?"
    - "Did we have the right tools?"
    - "Were runbooks accurate and useful?"

  output:
    - "Debrief document with findings"
    - "Action items with owners and due dates"
    - "Performance metrics for trend tracking"
    - "Recommendations for next game day"