Bash Scripting for SysAdmins: Automation and Monitoring

Introduction

Bash scripting automates repetitive sysadmin tasks — from backups and log rotation to deployment pipelines and health checks. This guide covers practical Bash scripting patterns used in real production environments.

Script Structure Best Practices

BASH

#!/bin/bash
set -euo pipefail
# -e: exit on error
# -u: error on undefined variable
# -o pipefail: catch errors in pipes

# Script metadata
readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
readonly LOG_FILE="/var/log/myscript.log"
readonly TIMESTAMP=$(date +%Y%m%d_%H%M%S)

# Cleanup on exit
cleanup() {
    local exit_code=$?
    echo "Script exited with code: $exit_code" >> "$LOG_FILE"
    # Remove temp files
    rm -f /tmp/myscript_$$.*
}
trap cleanup EXIT

# Logging function
log() {
    echo "[$(date +'%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG_FILE"
}

log "Script started"

Functions and Error Handling

BASH

# Function with return value
check_disk_usage() {
    local mount_point="$1"
    local threshold="${2:-85}"  # default 85%
    
    local usage=$(df "$mount_point" | awk 'NR==2 {print $5}' | tr -d '%')
    
    if [[ "$usage" -gt "$threshold" ]]; then
        log "WARNING: $mount_point is ${usage}% full (threshold: ${threshold}%)"
        return 1
    fi
    
    log "OK: $mount_point is ${usage}% full"
    return 0
}

# Call the function
if ! check_disk_usage /var 90; then
    # Send alert
    echo "Disk alert on $(hostname)" | mail -s "Disk Alert" admin@company.com
fi

Practical Script: System Health Check

BASH

#!/bin/bash
set -euo pipefail

REPORT=""
ALERTS=""
HOSTNAME=$(hostname -f)

report() { REPORT+="$1
"; }
alert() { ALERTS+="ALERT: $1
"; }

# CPU Load
cpu_load=$(uptime | awk -F'load average:' '{print $2}' | awk -F',' '{print $1}' | tr -d ' ')
report "CPU Load (1min): $cpu_load"
if (( $(echo "$cpu_load > 4.0" | bc -l) )); then
    alert "High CPU load: $cpu_load"
fi

# Memory
mem_free_pct=$(free | awk 'NR==2{printf "%.0f", $4/$2*100}')
report "Memory free: ${mem_free_pct}%"
if [[ "$mem_free_pct" -lt 10 ]]; then
    alert "Low memory: only ${mem_free_pct}% free"
fi

# Disk usage on all mounts
while IFS= read -r line; do
    mount=$(echo "$line" | awk '{print $6}')
    usage=$(echo "$line" | awk '{print $5}' | tr -d '%')
    report "Disk $mount: ${usage}%"
    if [[ "$usage" -gt 85 ]]; then
        alert "Disk $mount is ${usage}% full"
    fi
done < <(df -h --output=source,size,used,avail,pcent,target | tail -n +2)

# Services check
for service in nginx mysql sshd; do
    if systemctl is-active --quiet "$service"; then
        report "Service $service: running"
    else
        alert "Service $service is NOT running"
    fi
done

# Send report
echo -e "$REPORT" | mail -s "Health Report: $HOSTNAME" admin@company.com

if [[ -n "$ALERTS" ]]; then
    echo -e "$ALERTS" | mail -s "ALERT: $HOSTNAME Health Issues" oncall@company.com
fi

String Manipulation

BASH

# Trim whitespace
trim() {
    local str="$1"
    str="${str#"${str%%[![:space:]]*}"}"   # ltrim
    str="${str%"${str##*[![:space:]]}"}"   # rtrim
    echo "$str"
}

# Extract part of string
filename="backup_2024-01-15_prod.tar.gz"
date_part="${filename#backup_}"           # Remove prefix: 2024-01-15_prod.tar.gz
date_only="${date_part%%_*}"              # Up to first _: 2024-01-15
echo "$date_only"

# String contains
if [[ "$hostname" == *"prod"* ]]; then
    echo "This is a production server"
fi

File Processing

BASH

# Process CSV file
while IFS=',' read -r hostname ip role; do
    echo "Host: $hostname | IP: $ip | Role: $role"
    # Do something with each host
    ping -c 1 -W 1 "$ip" > /dev/null 2>&1 && echo "$hostname: UP" || echo "$hostname: DOWN"
done < hosts.csv

# Find and process files
find /var/log -name "*.log" -mtime +30 -print0 | while IFS= read -r -d '' file; do
    gzip "$file"
    log "Compressed: $file"
done

Deployment Script Example

BASH

#!/bin/bash
set -euo pipefail

APP_DIR="/var/www/myapp"
BACKUP_DIR="/var/backups/myapp"
DEPLOY_USER="www-data"
GIT_REPO="https://github.com/company/myapp.git"
BRANCH="${1:-main}"

log() { echo "[$(date +'%H:%M:%S')] $*"; }

log "Deploying branch: $BRANCH"

# Backup current version
if [[ -d "$APP_DIR" ]]; then
    log "Backing up current version..."
    tar -czf "$BACKUP_DIR/backup_$(date +%Y%m%d_%H%M%S).tar.gz" "$APP_DIR"
fi

# Pull new code
log "Fetching latest code..."
cd "$APP_DIR"
git fetch origin
git checkout "$BRANCH"
git pull origin "$BRANCH"

# Install dependencies
log "Installing dependencies..."
npm ci --production

# Run database migrations
log "Running migrations..."
npm run migrate

# Reload application (zero downtime)
log "Reloading application..."
systemctl reload nginx
pm2 reload myapp --update-env

log "Deployment complete!"

Cron Job Best Practices

BASH

# Good cron entry: lock file prevents overlapping runs
cat > /etc/cron.d/health-check << 'EOF'
*/5 * * * * root flock -n /var/lock/health-check.lock /usr/local/bin/health-check.sh >> /var/log/health-check.log 2>&1
EOF

# Test the script before adding to cron
bash -n /usr/local/bin/health-check.sh  # Check syntax
bash -x /usr/local/bin/health-check.sh  # Debug trace