Safe Remediation Patterns

# Phase 1: Validate with check mode
- name: Validate remediation plan
  hosts: "{{ target_host }}"
  tasks:
    - name: Check what would change
      ansible.builtin.package:
        name: nginx
        state: restarted
      check_mode: yes
      register: result

    - name: Abort if unexpected changes
      ansible.builtin.fail:
        msg: "Unexpected changes detected: {{ result }}"
      when: result.changed and not result.expected_changes | default(false)

# Phase 2: Apply if check passed
- name: Apply remediation
  hosts: "{{ target_host }}"
  serial: 1
  max_fail_percentage: 0
  tasks:
    - name: Restart service
      ansible.builtin.service:
        name: nginx
        state: restarted

- name: Deploy config fix
  ansible.builtin.template:
    src: nginx.conf.j2
    dest: /etc/nginx/nginx.conf
  diff: yes  # Show before/after in output
  register: config_result

- name: Log changes
  ansible.builtin.debug:
    msg: |
      Config changes:
      {{ config_result.diff }}

---
- name: Safe rolling remediation
  hosts: "{{ affected_hosts }}"
  serial: 1                   # One host at a time
  max_fail_percentage: 0      # Zero tolerance for failure
  any_errors_fatal: yes       # Stop entire play on first failure

  pre_tasks:
    - name: Remove host from load balancer
      ansible.builtin.uri:
        url: "http://lb.example.com/api/backends/{{ inventory_hostname }}/disable"
        method: POST
      delegate_to: localhost

  tasks:
    - name: Apply remediation
      ansible.builtin.service:
        name: "{{ service_name }}"
        state: restarted

    - name: Wait for service
      ansible.builtin.wait_for:
        port: "{{ service_port }}"
        timeout: 30

    - name: Health check
      ansible.builtin.uri:
        url: "http://{{ inventory_hostname }}:{{ service_port }}/health"
        status_code: 200

  post_tasks:
    - name: Re-enable in load balancer
      ansible.builtin.uri:
        url: "http://lb.example.com/api/backends/{{ inventory_hostname }}/enable"
        method: POST
      delegate_to: localhost

---
# AWX Workflow:
# 1. Diagnose (auto)
# 2. Create remediation plan (auto)
# 3. Approval gate (manual)
# 4. Apply remediation (auto)
# 5. Verify (auto)
# 6. Notify (auto)

# Approval node in AWX workflow
- name: Add approval gate
  ansible.controller.workflow_approval:
    name: "Approve Remediation"
    description: |
      Remediation Plan:
      - Service: {{ service_name }}
      - Hosts: {{ affected_hosts }}
      - Action: {{ remediation_action }}
    timeout: 3600
    state: present

- name: Circuit breaker check
  ansible.builtin.uri:
    url: "http://monitoring.example.com/api/circuit-breaker/{{ service_name }}"
    return_content: yes
  register: cb_status
  delegate_to: localhost

- name: Abort if circuit is open
  ansible.builtin.fail:
    msg: "Circuit breaker is open for {{ service_name }}. Manual intervention required."
  when: cb_status.status == 503

- name: Apply remediation
  ansible.builtin.service:
    name: "{{ service_name }}"
    state: restarted

- name: Record success
  ansible.builtin.uri:
    url: "http://monitoring.example.com/api/circuit-breaker/{{ service_name }}/record"
    method: POST
    body: '{"status": "success"}'
  delegate_to: localhost