Reloads, Restarts & Watchdogs

# Reload (no downtime)
sudo systemctl reload nginx
sudo systemctl reload sshd

# Restart (brief downtime)
sudo systemctl restart nginx

# Try reload, fall back to restart
sudo systemctl try-reload-or-restart nginx

# Check if reload is supported
systemctl show nginx -p ExecReload
# ExecReload=/bin/kill -HUP $MAINPID

# Force restart
sudo systemctl kill nginx
sudo systemctl start nginx

# Signal-based reload (most common)
[Service]
ExecReload=/bin/kill -HUP $MAINPID

# Command-based reload
[Service]
ExecReload=/usr/bin/myapp --reload-config

# Multiple reload commands
[Service]
ExecReload=/bin/kill -HUP $MAINPID
ExecReload=/bin/kill -USR1 $MAINPID

# Reload with validation
[Service]
ExecReloadPre=/usr/bin/myapp --validate-config
ExecReload=/bin/kill -HUP $MAINPID

# Check reload behavior
systemctl show nginx -p ExecReload
# ExecReload=/bin/kill -HUP $MAINPID

systemctl show myapp -p ExecReload
# (not set)  # reload not supported

# WatchdogSec enables systemd watchdog
[Service]
WatchdogSec=30
# Service must notify systemd every 30 seconds
# If notification is missed, service is restarted

# The service must call sd_notify() with WATCHDOG=1
# This is typically done in the application code

# Watchdog with separate check
[Service]
WatchdogSec=30
NotifyAccess=main

# Restart on watchdog timeout
Restart=on-watchdog
RestartSec=5

# Check watchdog status
systemctl show myapp -p WatchdogUSec
# WatchdogUSec=30s

systemctl show myapp -p WatchdogTimestamp
# WatchdogTimestamp=Mon 2024-01-15 10:30:00 UTC

# sd_notify messages
# READY=1        — Service is ready (for Type=notify)
# RELOADING=1    — Service is reloading
# STOPPING=1     — Service is stopping
# WATCHDOG=1     — Watchdog ping
# WATCHDOG=1     — Watchdog ping
# STATUS=...     — Status text
# ERRNO=...      — Error number
# BUSERROR=...   — D-Bus error

# In application code (Python example):
# import sdnotify
# notifier = sdnotify.SystemdNotifier()
# notifier.notify("READY=1")
# notifier.notify("WATCHDOG=1")

# In shell script:
systemd-notify --ready
systemd-notify WATCHDOG=1
systemd-notify "STATUS=Processing request..."

# Notify access levels
[Service]
NotifyAccess=all    # Allow any process to notify
NotifyAccess=main   # Only main PID can notify
NotifyAccess=exec   # Only exec'd processes can notify

# Pattern 1: ExecReload with graceful reload
[Service]
ExecStart=/opt/myapp/bin/server
ExecReload=/bin/kill -HUP $MAINPID
Restart=on-failure
RestartSec=5

# Pattern 2: Pre-start with config validation
[Service]
ExecStartPre=/opt/myapp/bin/server --validate-config
ExecStart=/opt/myapp/bin/server
Restart=on-failure

# Pattern 3: Socket activation for connection draining
[Service]
Type=notify
Accept=no
# systemd holds connections until service is ready

# Pattern 4: Try-reload-or-restart
# systemd tries reload first, falls back to restart
sudo systemctl try-reload-or-restart myapp

# Pattern 5: Stop → Wait → Start (manual control)
sudo systemctl stop myapp
# Wait for connections to drain
sleep 10
sudo systemctl start myapp

# Restart policy
Restart=on-failure
RestartSec=5

# Rate limiting
StartLimitBurst=5
StartLimitIntervalSec=300

# Check restart count
systemctl show myapp -p NRestarts
# NRestarts=3

# Reset restart counter
systemctl reset-failed myapp

# View restart history
journalctl -u myapp | grep -i "started|stopped|failed"

# Override restart for specific cases
[Service]
Restart=on-failure
RestartSec=5
# Don't restart on specific exit codes
RestartPreventExitStatus=1 2 3

# Restart on specific exit codes
RestartForceExitStatus=SIGKILL