Practice: On-Call Runbook

#!/usr/bin/env bash
set -euo pipefail

# Read Alertmanager webhook
PAYLOAD=$(cat)
STATUS=$(echo "$PAYLOAD" | jq -r '.status')
ALERTNAME=$(echo "$PAYLOAD" | jq -r '.alerts[0].labels.alertname')
SEVERITY=$(echo "$PAYLOAD" | jq -r '.alerts[0].labels.severity')
INSTANCE=$(echo "$PAYLOAD" | jq -r '.alerts[0].labels.instance')

echo "$(date): Alert received: $ALERTNAME ($SEVERITY) on $INSTANCE"

# Route to appropriate runbook
case "$ALERTNAME" in
    HighCPU)     ./runbooks/cpu.sh "$INSTANCE" ;;
    HighMemory)  ./runbooks/memory.sh "$INSTANCE" ;;
    ServiceDown) ./runbooks/service-down.sh "$INSTANCE" ;;
    *)           ./runbooks/default.sh "$INSTANCE" "$ALERTNAME" ;;
esac

#!/usr/bin/env bash

# Collect system diagnostics
collect_diagnostics() {
    local host="$1"
    local output_dir="/tmp/diagnostics-$(date +%s)"
    mkdir -p "$output_dir"

    echo "Collecting diagnostics from $host..."

    # System info
    ssh "$host" "uname -a" > "$output_dir/uname.txt"

    # Disk usage
    ssh "$host" "df -h" > "$output_dir/disk.txt"

    # Memory
    ssh "$host" "free -h" > "$output_dir/memory.txt"

    # Top processes
    ssh "$host" "ps aux --sort=-%cpu | head -20" > "$output_dir/top_cpu.txt"
    ssh "$host" "ps aux --sort=-%mem | head -20" > "$output_dir/top_mem.txt"

    # Network connections
    ssh "$host" "ss -tlnp" > "$output_dir/ports.txt"

    # Recent logs
    ssh "$host" "journalctl --since '1 hour ago' -p err" > "$output_dir/errors.txt"

    echo "Diagnostics saved to $output_dir"
    echo "$output_dir"
}

# Check service health
check_service() {
    local host="$1"
    local port="$2"
    local name="{3:-service}"

    if nc -zv -w 3 "$host" "$port" 2>/dev/null; then
        echo "[OK] $name ($host:$port)"
        return 0
    else
        echo "[FAIL] $name ($host:$port)"
        return 1
    fi
}

#!/usr/bin/env bash
set -euo pipefail

# Restart a service on remote host
restart_service() {
    local host="$1"
    local service="$2"

    echo "Restarting $service on $host..."
    ssh "$host" "sudo systemctl restart $service"

    sleep 5

    if ssh "$host" "systemctl is-active $service" | grep -q "active"; then
        echo "Successfully restarted $service"
        return 0
    else
        echo "Failed to restart $service"
        return 1
    fi
}

# Clear disk space
clear_disk() {
    local host="$1"

    echo "Clearing disk space on $host..."

    # Remove old logs
    ssh "$host" "sudo find /var/log -name '*.gz' -mtime +30 -delete"

    # Clear package cache
    ssh "$host" "sudo apt-get clean 2>/dev/null || sudo yum clean all 2>/dev/null"

    # Remove temp files
    ssh "$host" "sudo find /tmp -type f -mtime +7 -delete"

    # Show result
    ssh "$host" "df -h / | tail -1"
}

# Restart application
restart_app() {
    local host="$1"

    echo "Performing zero-downtime restart on $host..."

    # Graceful stop
    ssh "$host" "sudo systemctl stop myapp"

    # Wait for connections to drain
    sleep 10

    # Start
    ssh "$host" "sudo systemctl start myapp"

    # Verify
    sleep 5
    check_health "$host" 8080
}

#!/usr/bin/env bash

# Send Slack notification
send_slack() {
    local webhook="$SLACK_WEBHOOK_URL"
    local message="$1"
    local channel="{2:-#alerts}"

    curl -sS -X POST "$webhook" \
      -H "Content-Type: application/json" \
      -d "{
        "channel": "$channel",
        "text": "$message",
        "icon_emoji": ":warning:"
      }" > /dev/null
}

# Send PagerDuty alert
send_pagerduty() {
    local severity="$1"
    local message="$2"

    curl -sS -X POST "https://events.pagerduty.com/v2/enqueue" \
      -H "Content-Type: application/json" \
      -d "{
        "routing_key": "$PAGERDUTY_KEY",
        "event_action": "trigger",
        "payload": {
          "severity": "$severity",
          "summary": "$message"
        }
      }" > /dev/null
}

# Escalation chain
escalate() {
    local issue="$1"

    # Level 1: Slack notification
    send_slack "WARNING: $issue"

    # Level 2: Page on-call after 5 minutes
    sleep 300
    send_pagerduty "warning" "$issue"

    # Level 3: Page manager after 15 minutes
    sleep 600
    send_pagerduty "critical" "$issue - ESCALATING"
}

#!/usr/bin/env bash
set -euo pipefail

INSTANCE="$1"
HOST=$(echo "$INSTANCE" | cut -d: -f1)
PORT=$(echo "$INSTANCE" | cut -d: -f2)

LOG="/var/log/runbook-$(date +%Y%m%d-%H%M%S).log"
exec > >(tee -a "$LOG") 2>&1

echo "=== Service Down Runbook ==="
echo "Instance: $INSTANCE"
echo "Time: $(date)"

# Step 1: Verify the issue
echo "Step 1: Verifying service is down..."
if nc -zv -w 5 "$HOST" "$PORT" 2>/dev/null; then
    echo "Service is responding. False alarm."
    exit 0
fi

# Step 2: Collect diagnostics
echo "Step 2: Collecting diagnostics..."
collect_diagnostics "$HOST"

# Step 3: Attempt restart
echo "Step 3: Attempting service restart..."
if restart_service "$HOST" myapp; then
    echo "Service recovered after restart"
    send_slack "Service recovered on $HOST after automatic restart"
    exit 0
fi

# Step 4: Escalate
echo "Step 4: Restart failed, escalating..."
send_pagerduty "critical" "Service down on $HOST - manual intervention required"

echo "=== Runbook complete ==="
echo "Logs: $LOG"