Capstone: Production Runbook

#!/usr/bin/env bash
set -euo pipefail

# ============================================================
# Incident Runbook: High Error Rate
# ============================================================

readonly SCRIPT_NAME="$(basename "{BASH_SOURCE[0]}")"
readonly LOG_DIR="/var/log/runbooks"
readonly TIMESTAMP=$(date '+%Y%m%d-%H%M%S')
readonly LOG_FILE="$LOG_DIR/$SCRIPT_NAME-$TIMESTAMP.log"

# Configuration
readonly API_URL="https://api.example.com"
readonly HEALTH_ENDPOINT="$API_URL/health"
readonly SSH_HOST="api-server-1"
readonly SERVICE_NAME="api"
readonly SLACK_WEBHOOK="{SLACK_WEBHOOK_URL:?}"

# Logging
mkdir -p "$LOG_DIR"
exec > >(tee -a "$LOG_FILE") 2>&1

log()    { echo "[$(date '+%H:%M:%S')] $*"; }
error()  { echo "[$(date '+%H:%M:%S')] ERROR: $*" >&2; }
die()    { error "$@"; exit 1; }

# Cleanup
cleanup() {
    log "Runbook complete. Log: $LOG_FILE"
}
trap cleanup EXIT

check_api_health() {
    log "Checking API health..."
    local status
    status=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 "$HEALTH_ENDPOINT" 2>/dev/null || echo "000")
    log "Health endpoint returned HTTP $status"
    [[ "$status" == "200" ]]
}

check_error_rate() {
    log "Checking error rate..."
    local errors
    errors=$(curl -sS "$API_URL/metrics" 2>/dev/null | jq -r '.error_rate // 0')
    log "Current error rate: $errors%"
    (( $(echo "$errors > 5" | bc -l) ))
}

check_server_resources() {
    log "Checking server resources..."
    local cpu mem disk
    cpu=$(ssh "$SSH_HOST" "top -bn1 | grep 'Cpu(s)' | awk '{print 100-$8}'" 2>/dev/null || echo "unknown")
    mem=$(ssh "$SSH_HOST" "free | awk '/Mem:/ {printf "%.0f", $3/$2*100}"" 2>/dev/null || echo "unknown")
    disk=$(ssh "$SSH_HOST" "df / | awk 'NR==2 {print $5}' | tr -d '%'" 2>/dev/null || echo "unknown")

    log "CPU: {cpu}%, Memory: {mem}%, Disk: {disk}%"

    if [[ "$cpu" != "unknown" ]] && (( cpu > 90 )); then
        log "WARNING: CPU usage is critical"
    fi
    if [[ "$disk" != "unknown" ]] && (( disk > 90 )); then
        log "WARNING: Disk usage is critical"
    fi
}

check_recent_errors() {
    log "Checking recent errors..."
    ssh "$SSH_HOST" "journalctl -u $SERVICE_NAME --since '10 minutes ago' -p err --no-pager | tail -20" 2>/dev/null || true
}

collect_diagnostics() {
    log "=== Collecting Diagnostics ==="
    check_api_health || log "API health check failed"
    check_error_rate || log "Error rate elevated"
    check_server_resources
    check_recent_errors
    log "=== Diagnostics Complete ==="
}

restart_service() {
    log "Restarting $SERVICE_NAME on $SSH_HOST..."
    ssh "$SSH_HOST" "sudo systemctl restart $SERVICE_NAME"

    log "Waiting for service to become ready..."
    sleep 10

    if check_api_health; then
        log "Service restarted successfully"
        return 0
    else
        log "Service still unhealthy after restart"
        return 1
    fi
}

clear_disk_space() {
    log "Clearing disk space on $SSH_HOST..."
    ssh "$SSH_HOST" "sudo find /var/log -name '*.gz' -mtime +30 -delete"
    ssh "$SSH_HOST" "sudo journalctl --vacuum-time=7d"
    log "Disk cleanup complete"
}

scale_service() {
    local replicas="$1"
    log "Scaling $SERVICE_NAME to $replicas replicas..."
    kubectl scale deployment/"$SERVICE_NAME" --replicas="$replicas" -n production
    log "Scale operation complete"
}

remediate() {
    log "=== Starting Remediation ==="

    # Strategy 1: Try restart
    if restart_service; then
        log "Remediation successful: service restart"
        return 0
    fi

    # Strategy 2: Clear disk space
    clear_disk_space
    if restart_service; then
        log "Remediation successful: disk cleanup + restart"
        return 0
    fi

    # Strategy 3: Scale up
    scale_service 4
    sleep 30
    if check_api_health; then
        log "Remediation successful: scale up"
        return 0
    fi

    log "All remediation attempts failed"
    return 1
}

verify_fix() {
    log "=== Verifying Fix ==="

    # Check health 3 times over 30 seconds
    local success=0
    for i in 1 2 3; do
        if check_api_health; then
            ((success++))
        fi
        sleep 10
    done

    if (( success == 3 )); then
        log "Verification passed: API healthy 3/3 checks"
        return 0
    else
        log "Verification failed: API healthy $success/3 checks"
        return 1
    fi
}

send_notification() {
    local status="$1"
    local message="$2"

    local color
    [[ "$status" == "resolved" ]] && color="#36a64f" || color="#ff0000"

    curl -sS -X POST "$SLACK_WEBHOOK" \
      -H "Content-Type: application/json" \
      -d "{
        "attachments": [{
          "color": "$color",
          "title": "Incident $status",
          "text": "$message",
          "fields": [
            {"title": "Environment", "value": "production", "short": true},
            {"title": "Time", "value": "$(date)", "short": true},
            {"title": "Runbook Log", "value": "$LOG_FILE", "short": false}
          ]
        }]
      }" > /dev/null
}

main() {
    log "=== Incident Runbook: High Error Rate ==="
    log "Triggered at: $(date)"

    # Phase 1: Diagnose
    collect_diagnostics

    # Phase 2: Remediate
    if remediate; then
        # Phase 3: Verify
        if verify_fix; then
            send_notification "resolved" "API error rate resolved. Service restarted successfully."
            log "=== Incident Resolved ==="
            exit 0
        fi
    fi

    # Escalation
    send_notification "unresolved" "Automated remediation failed. Manual intervention required."
    log "=== Escalation Required ==="
    exit 1
}

main "$@"