日志监控与告警
# 第 7 章 日志监控与告警
# 目录介绍
# 7.1 日志分析
# 7.1.1 常见日志格式解析
能看懂日志格式是分析的基础。以下是运维中最常见的几种：
#!/bin/bash
# ===== 1. Nginx / Apache 访问日志（Combined 格式）=====
# 示例行：
# 192.168.1.100 - - [10/Jun/2025:14:30:00 +0800] "GET /api/users HTTP/1.1" 200 1234 "https://example.com" "Mozilla/5.0"
# $1 $2 $3 $4 $5 $6 $7 $8 $9 $10
# 字段解析：
# $1 = 客户端 IP
# $2 = 远程登录名（通常为 -）
# $3 = 远程用户名（通常为 -）
# $4 = 时间戳 [日/月/年:时:分:秒 时区]
# $5 = 请求行 = 方法 URL 协议
# $6 = HTTP 方法（GET/POST...）
# $7 = 请求 URL 路径
# $8 = HTTP 协议版本
# $9 = HTTP 状态码（200/301/404/500...）
# $10= 响应字节数
# $11= Referer
# $12= User-Agent
# ===== Nginx access.log 一行解析脚本 =====
cat > parse_nginx_log.sh << 'EOF'
#!/bin/bash
LOG_FILE="${1:-/var/log/nginx/access.log}"
awk '{
# 提取时间戳
gsub(/[\[\]]/, "", $4) # 去掉方括号 [ ]
timestamp = $4 " " $5 # 时间 + 时区
# 提取请求信息
gsub(/"/, "", $6) # 去掉引号
method = $6
url = $7
# 提取关键指标
ip = $1
status = $9
bytes = $10
# 输出解析结果
printf "[%s] %s %-7s %s → HTTP %d, %d bytes\n",
timestamp, ip, method, url, status, bytes
}' "$LOG_FILE"
EOF
# ===== 2. Apache 错误日志格式 =====
# [Mon Jun 10 14:30:00.123456 2025] [core:error] [pid 12345] [client 1.2.3.4:56789] AH00124: Request exceeded the limit
# ===== 3. Syslog / 系统日志格式 =====
# Jun 10 14:30:00 server01 sshd[12345]: Failed password for root from 1.2.3.4 port 22 ssh2
# ===== 4. 应用日志（常见自定义格式）=====
# 2025-06-10 14:30:00.123 [ERROR] [main-thread] com.example.Service - Connection timeout after 30000ms
# $1 $2 $3 $4 $5 $6
# ===== 实战：通用日志字段提取函数 =====
extract_log_fields() {
local format="$1" # nginx / syslog / app
local line="$2"
case "$format" in
nginx)
echo "$line" | awk '{print "IP:",$1," 状态:",$9," URL:",$7," 字节:",$10}'
;;
syslog)
echo "$line" | awk '{print "时间:",$1,$2,$3," 主机:",$4," 进程:",$5," 内容:",substr($0, index($0,$6))}'
;;
app)
echo "$line" | awk -F'[][]' '{
split($1, t, " ")
print "时间:", t[1], t[2], " 级别:", $2, " 消息:", $4
}'
;;
esac
}
# 7.1.2 访问量统计——PV/UV/QPS/带宽
#!/bin/bash
LOG="/var/log/nginx/access.log"
# ===== PV (Page View) —— 总请求量 =====
echo "=== PV 统计 ==="
wc -l "$LOG"
# 输出：1234567 /var/log/nginx/access.log
# 按小时统计 PV（看当天流量高峰）
awk '{split($4, t, ":"); hour=t[2]; count[hour]++}
END {
for (h=0; h<=23; h++) printf "%02d:00 %6d\n", h, count[h]+0
}' "$LOG" | sort -t':' -k1n
# ===== UV (Unique Visitor) —— 独立 IP 数 =====
echo "=== UV 统计 ==="
awk '{print $1}' "$LOG" | sort -u | wc -l
# 按小时 UV（更精细）
awk '{split($4, t, ":"); hour=t[2]; uv[hour][$1]=1}
END {
for (h=0; h<=23; h++) {
count=0; for (ip in uv[h]) count++
printf "%02d:00 %6d\n", h, count
}
}' "$LOG" | sort -t':' -k1n
# ===== QPS (Query Per Second) —— 每秒请求数 =====
echo "=== QPS 统计 ==="
# 统计最近 1 分钟的 QPS
tail -1000 "$LOG" | awk -v interval=60 '{
split($4, t, ":")
timestamp = t[2]":"t[3]":"t[4]
count[timestamp]++
}
END {
total = 0; n = 0
for (ts in count) { total += count[ts]; n++ }
printf "最近 %d 条日志，平均 QPS: %.2f\n", 1000, total/(n*1.0)
}'
# 更精确的 QPS（按秒统计）
awk '{
split($4, t, ":")
sec = sprintf("%s:%s:%s:%s", t[2], t[3], t[4], substr(t[5],1,2))
qps[sec]++
}
END {
for (s in qps) if (qps[s] > 100) print s, qps[s]
}' "$LOG" | sort -k2 -rn | head -10
# 输出 QPS 超过 100 的时刻（用于定位瞬时高峰）
# ===== 带宽 / 流量统计 =====
echo "=== 流量统计 ==="
awk '{
split($4, t, ":")
hour = t[2]+0
bytes_total[hour] += $10
count[hour]++
}
END {
printf "%-8s %12s %12s %12s\n", "小时", "请求数", "流量(MB)", "平均(KB/请求)"
for (h=0; h<=23; h++) {
if (count[h] > 0) {
printf "%02d:00 %10d %10.2f %10.2f\n",
h, count[h], bytes_total[h]/1048576,
(bytes_total[h]/count[h])/1024
}
}
}' "$LOG" | sort -t':' -k1n
# ===== 一行汇总：今日核心指标 =====
awk 'BEGIN { split("Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec", m)
for (i in m) month_num[m[i]] = sprintf("%02d", i) }
{
ip_count[$1]++
if ($9 >= 500) err5xx++
else if ($9 >= 400) err4xx++
success++
total_bytes += $10
# 提取当前日期
split($4, t, "/"); split(t[1], date, " ")
today = date[2] "-" month_num[date[1]] "-" strftime("%Y")
}
END {
printf "===== %s 日志日报 =====\n", today
printf "总请求: %d\n", NR
printf "独立IP: %d\n", length(ip_count)
printf "成功: %d (%.1f%%)\n", success, success/NR*100
printf "4xx: %d (%.1f%%)\n", err4xx+0, err4xx/NR*100
printf "5xx: %d (%.1f%%)\n", err5xx+0, err5xx/NR*100
printf "总流量: %.2f MB\n", total_bytes/1048576
}' "$LOG"
# 7.1.3 Top IP 分析与异常流量检测
#!/bin/bash
LOG="/var/log/nginx/access.log"
# ===== Top 10 访问 IP =====
awk '{print $1}' "$LOG" | sort | uniq -c | sort -rn | head -10
# ===== 异常 IP 检测：请求量远超平均的 IP =====
awk '{ip[$1]++; total++}
END {
avg = total / length(ip)
printf "总独立IP: %d, 平均每IP请求: %.1f\n\n", length(ip), avg
printf "%-20s %8s %8s %8s\n", "IP", "请求数", "占比%", "倍数"
for (i in ip) {
if (ip[i] > avg * 3) { # 超过平均 3 倍 = 异常
printf "%-20s %8d %7.1f%% %7.1fx\n", i, ip[i], ip[i]/total*100, ip[i]/avg
}
}
}' "$LOG"
# ===== 检测恶意请求特征 =====
# 1. 单一 IP 大量 404（目录扫描）
awk '$9 == 404 {ip[$1]++}
END {
for (i in ip) if (ip[i] > 50) printf "疑似扫描 IP: %s (404=%d次)\n", i, ip[i]
}' "$LOG"
# 2. 单一 IP 大量 5xx（攻击导致服务异常）
awk '$9 >= 500 {ip[$1]++; err[$1][$9]++}
END {
for (i in ip) if (ip[i] > 20) {
printf "异常 IP: %s (5xx=%d次)", i, ip[i]
for (code in err[i]) printf " HTTP %d=%d", code, err[i][code]
printf "\n"
}
}' "$LOG"
# 3. User-Agent 异常（爬虫/工具扫描）
awk '{
# 提取 User-Agent（Combined 格式的第 12 个字段）
match($0, /"[^"]*"$/)
ua = substr($0, RSTART+1, RLENGTH-2)
ua_count[ua]++
}
END {
for (u in ua_count) {
if (ua_count[u] > 100) printf "%d 次 - %s\n", ua_count[u], u
}
}' "$LOG" | sort -rn | head -10
# 7.1.4 状态码分布与异常监控
#!/bin/bash
LOG="/var/log/nginx/access.log"
# ===== 状态码分布统计 =====
awk '{code[$9]++; total++}
END {
printf "%-10s %8s %8s\n", "状态码", "数量", "占比"
printf "%-10s %8s %8s\n", "------", "----", "----"
for (c in code) {
printf "HTTP %-5s %8d %7.1f%%\n", c, code[c], code[c]/total*100
}
}' "$LOG" | sort
# ===== 按小时看状态码变化（找故障时间点）=====
awk '{
split($4, t, ":")
hour = t[2]+0
codes[hour][$9]++
}
END {
printf "%-7s", "Hour"
for (c=200; c<=504; c++) if (c==200||c==301||c==302||c==304||c==400||c==403||c==404||c==500||c==502||c==503||c==504) printf "%7d", c
printf "\n"
for (h=0; h<=23; h++) {
printf "%02d:00 ", h
for (c=200; c<=504; c++)
if (c==200||c==301||c==302||c==304||c==400||c==403||c==404||c==500||c==502||c==503||c==504)
printf "%7d", codes[h][c]+0
printf "\n"
}
}' "$LOG"
# ===== 5xx 错误实时监控（配合 tail -f）=====
# 使用方式：tail -f access.log | ./monitor_5xx.sh
cat > monitor_5xx.sh << 'EOF'
#!/bin/bash
THRESHOLD=10 # 每分钟超过 10 个 5xx 就告警
INTERVAL=60 # 统计周期（秒）
COUNT_FILE="/tmp/5xx_count_$$"
while read -r line; do
status=$(echo "$line" | awk '{print $9}')
if [[ "$status" -ge 500 ]] 2>/dev/null; then
echo "$(date +%s)" >> "$COUNT_FILE"
fi
done < <(tail -f /var/log/nginx/access.log) &
MONITOR_PID=$!
# 定时检查
while true; do
sleep "$INTERVAL"
now=$(date +%s)
cutoff=$((now - INTERVAL))
count=$(awk -v c="$cutoff" '$1 > c' "$COUNT_FILE" 2>/dev/null | wc -l)
if [[ "$count" -ge "$THRESHOLD" ]]; then
echo "[$(date)] ⚠️ 告警：过去 ${INTERVAL}s 内出现 $count 个 5xx 错误！"
# 重置计数文件
: > "$COUNT_FILE"
fi
done
trap "kill $MONITOR_PID 2>/dev/null; rm -f $COUNT_FILE" EXIT
EOF
# 7.1.5 响应时间分析——揪出慢请求
Nginx 可配置 $request_time（处理时间）和 $upstream_response_time（上游响应时间），需先在 log_format 中添加：
log_format timed '$remote_addr - [$time_local] "$request" $status $body_bytes_sent '
'"$http_referer" "$http_user_agent" $request_time $upstream_response_time';
#!/bin/bash
LOG="/var/log/nginx/timed_access.log"
# 假设最后两列是 $request_time 和 $upstream_response_time
# ===== Top 10 慢请求 =====
awk '{print $(NF-1), $7}' "$LOG" | sort -rn | head -10
# ===== 响应时间分桶统计 =====
awk '{
rt = $(NF-1) + 0
if (rt < 0.1) bucket["0~100ms"]++
else if (rt < 0.5) bucket["100~500ms"]++
else if (rt < 1) bucket["0.5~1s"]++
else if (rt < 3) bucket["1~3s"]++
else if (rt < 5) bucket["3~5s"]++
else bucket["5s+"]++
total++
}
END {
for (b in bucket) printf "%-12s %8d %6.1f%%\n", b, bucket[b], bucket[b]/total*100
}' "$LOG" | sort -k2 -rn
# ===== 按 URL 统计平均响应时间 =====
awk '{
url = $7
rt = $(NF-1) + 0
url_count[url]++
url_rt_total[url] += rt
if (rt > url_rt_max[url]) url_rt_max[url] = rt
}
END {
printf "%-50s %8s %8s %10s\n", "URL", "次数", "平均(秒)", "最大(秒)"
for (u in url_count) {
if (url_count[u] >= 10) { # 至少被请求 10 次才统计
avg = url_rt_total[u] / url_count[u]
printf "%-50s %8d %8.3f %10.3f\n", u, url_count[u], avg, url_rt_max[u]
}
}
}' "$LOG" | sort -k3 -rn | head -20
# ===== 检测上游响应慢（$upstream_response_time > 阈值）=====
awk '$(NF) > 3 { # 上游响应超过 3 秒
print $1, $4, $7, "上游:", $(NF), "总耗时:", $(NF-1)
}' "$LOG"
# 7.1.6 错误日志提取、分类与告警
#!/bin/bash
ERROR_LOG="/var/log/nginx/error.log"
# ===== 提取所有错误并分类 =====
# 按错误类型分组统计
awk '{
# 提取错误级别（第一个方括号中的内容）
match($0, /\[([a-z]+)\]/, arr)
level = arr[1]
# 提取简要信息
msg = $0
gsub(/^.*\] /, "", msg) # 去掉前缀
msg = substr(msg, 1, 80) # 截断
errors[level]++
if (errors[level] <= 3) samples[level] = samples[level] "\n " msg
}
END {
for (l in errors) {
printf "[%s] %d 次\n%s\n\n", toupper(l), errors[l], samples[l]
}
}' "$ERROR_LOG"
# ===== 错误趋势：每分钟错误数 =====
awk '{
match($0, /[0-9]{4}\/[0-9]{2}\/[0-9]{2} [0-9]{2}:[0-9]{2}/)
minute = substr($0, RSTART, 16)
count[minute]++
}
END {
for (m in count) print m, count[m]
}' "$ERROR_LOG" | sort | tail -30
# ===== 应用中 OOM / 超时 / 连接失败 分类 =====
cat > classify_errors.sh << 'EOF'
#!/bin/bash
LOG="${1:-/var/log/app/error.log}"
declare -A PATTERNS=(
["OOM/内存溢出"]="OutOfMemory|java\.lang\.OutOfMemoryError|OOM killer"
["连接超时"]="timeout|timed out|Connection refused"
["数据库错误"]="SQLException|Connection pool|too many connections"
["磁盘空间"]="No space left|Disk quota"
["权限错误"]="Permission denied|Access denied"
["第三方API失败"]="api.*error|HTTP \d{3}.*error"
["空指针/非法参数"]="NullPointerException|IllegalArgument"
["死锁"]="Deadlock|Lock wait timeout"
)
echo "===== 错误分类统计 ====="
for category in "${!PATTERNS[@]}"; do
count=$(grep -cP "${PATTERNS[$category]}" "$LOG" 2>/dev/null)
if [[ "$count" -gt 0 ]]; then
printf "%-20s %d 次\n" "$category" "$count"
fi
done
EOF
# 7.2 系统监控
# 7.2.1 CPU 监控——使用率/负载/进程排行
#!/bin/bash
# ===== CPU 使用率（实时）=====
cpu_usage() {
# 读取 /proc/stat 计算 CPU 使用率（用户态 + 系统态 + IO 等待）
local idle1 idle2 total1 total2
read -r _ user1 nice1 system1 idle1 iowait1 irq1 softirq1 steal1 _ < /proc/stat
idle1=$((idle1 + iowait1))
total1=$((user1 + nice1 + system1 + idle1 + iowait1 + irq1 + softirq1 + steal1))
sleep 1
read -r _ user2 nice2 system2 idle2 iowait2 irq2 softirq2 steal2 _ < /proc/stat
idle2=$((idle2 + iowait2))
total2=$((user2 + nice2 + system2 + idle2 + iowait2 + irq2 + softirq2 + steal2))
local diff_idle=$((idle2 - idle1))
local diff_total=$((total2 - total1))
echo "scale=2; (1 - $diff_idle / $diff_total) * 100" | bc
}
echo "CPU 使用率: $(cpu_usage)%"
# ===== 系统负载（Load Average）=====
check_load() {
local load1 load5 load15 cores
read -r load1 load5 load15 _ < /proc/loadavg
cores=$(nproc)
echo "Load: 1min=${load1} 5min=${load5} 15min=${load15} (CPU核数: ${cores})"
# 负载告警：1min 负载 > CPU 核数
if (( $(echo "$load1 > $cores * 1.5" | bc -l) )); then
echo "⚠️ 1 分钟负载 $load1 超过核心数 $cores 的 1.5 倍！"
fi
}
check_load
# ===== Top 5 CPU 进程 =====
ps aux --sort=-%cpu | head -6 | awk '{
if (NR == 1) { printf "%-8s %-20s %6s %6s %s\n", "PID", "进程", "CPU%", "MEM%", "命令" }
else { printf "%-8s %-20s %5.1f%% %5.1f%% %s\n", $2, substr($11,1,20), $3, $4, substr($0, index($0,$11)) }
}'
# ===== 持续 CPU 监控脚本 =====
cat > cpu_monitor.sh << 'SCRIPT'
#!/bin/bash
THRESHOLD=80 # CPU 使用率阈值（%）
COOLDOWN=300 # 冷却时间（秒），避免重复告警
LAST_ALERT_FILE="/tmp/last_cpu_alert"
while true; do
# 获取 CPU 使用率（取 top 输出）
cpu=$(top -bn1 | grep "Cpu(s)" | awk '{print $2}' | cut -d'%' -f1)
cpu=${cpu%.*} # 去掉小数
if [[ "$cpu" -ge "$THRESHOLD" ]]; then
now=$(date +%s)
last_alert=$(cat "$LAST_ALERT_FILE" 2>/dev/null || echo 0)
if (( now - last_alert > COOLDOWN )); then
echo "[$(date)] ⚠️ CPU 使用率 ${cpu}%，超过阈值 ${THRESHOLD}%"
echo "$now" > "$LAST_ALERT_FILE"
# 记录 Top 进程快照
ps aux --sort=-%cpu | head -6 >> /var/log/cpu_alert.log
fi
fi
sleep 10
done
SCRIPT
# 7.2.2 内存监控——使用率/OOM 风险/Swap
#!/bin/bash
# ===== 内存概览 =====
free -h | awk '{
if (NR==1) print "===== 内存概览 ====="
if (NR==2) printf "总内存: %8s 已用: %8s 可用: %8s\n", $2, $3, $7
if (NR==3) printf "Swap: %8s 已用: %8s 空闲: %8s\n", $2, $3, $4
}'
# ===== 详细内存使用 =====
check_memory() {
local total used avail percent
# 从 /proc/meminfo 精确读取
total=$(awk '/^MemTotal:/ {print $2}' /proc/meminfo)
avail=$(awk '/^MemAvailable:/ {print $2}' /proc/meminfo)
used=$((total - avail))
percent=$((used * 100 / total))
echo "内存使用: ${percent}% (已用: $((used/1024))MB / 总共: $((total/1024))MB)"
# OOM 风险告警：可用内存 < 10%
if [[ "$percent" -ge 90 ]]; then
echo "⚠️ 严重：可用内存不足 10%，存在 OOM 风险！"
# 列出 OOM Score 最高的进程（最有可能被 kill 的）
echo ""
echo "OOM Killer 候选进程（Score 最高 Top 5）："
for pid in $(ps aux --sort=-%mem | awk 'NR>1{print $2}' | head -5); do
if [[ -f /proc/$pid/oom_score ]]; then
score=$(cat /proc/$pid/oom_score)
name=$(cat /proc/$pid/comm 2>/dev/null)
echo " PID:$pid Score:$score Name:$name"
fi
done
fi
}
check_memory
# ===== Swap 使用监控 =====
check_swap() {
local total used percent
total=$(awk '/^SwapTotal:/ {print $2}' /proc/meminfo)
used=$(awk '/^SwapFree:/ {print $2}' /proc/meminfo)
used=$((total - used))
percent=$((used * 100 / total))
echo "Swap 使用: ${percent}%"
# Swap 使用 > 50% 意味着物理内存不足
if [[ "$percent" -ge 50 ]]; then
echo "⚠️ Swap 使用率 ${percent}%，物理内存可能不足，建议排查"
fi
}
check_swap
# ===== Top 5 内存进程 =====
ps aux --sort=-%mem | head -6 | awk '{
if (NR==1) printf "%-8s %6s %-20s %s\n", "PID", "MEM%", "进程", "RSS(MB)"
else printf "%-8s %5.1f%% %-20s %6d\n", $2, $4, substr($11,1,20), $6/1024
}'
# 7.2.3 磁盘监控——空间/inode/IO
#!/bin/bash
# ===== 磁盘空间概览 =====
df -h | awk 'NR==1 || /^\/dev\//' # 只显示物理磁盘
# ===== 磁盘使用率告警 =====
check_disk() {
local threshold=80
df -h | awk -v th="$threshold" 'NR>1 && /^\/dev\// {
gsub(/%/, "", $5)
if ($5+0 >= th) {
printf "⚠️ 磁盘 %-20s 使用率: %d%% (已用:%s / 总共:%s)\n",
$6, $5, $3, $2
}
}'
}
check_disk
# ===== Inode 使用率检测 =====
# Inode 耗尽也会导致"磁盘满"的错误（No space left on device）
df -i | awk 'NR>1 && /^\/dev\// {
gsub(/%/, "", $5)
if ($5+0 >= 80) {
printf "⚠️ Inode 使用率过高：%s %d%%\n", $6, $5
}
}'
# ===== 磁盘 I/O 监控 =====
# 使用 iostat 查看 IO 负载
iostat -x 1 2 | awk 'END {
printf "%-10s %8s %8s %8s %8s\n", "设备", "r/s", "w/s", "await", "util%"
}
NR > 6 && $1 ~ /^sd|^nvme/ {
printf "%-10s %8.1f %8.1f %8.1f %7.1f%%\n", $1, $4, $5, $10, $14
}'
# await > 50ms 或 util% > 80% 说明磁盘 IO 瓶颈
# ===== 查找大文件/大目录 =====
find_large() {
local dir="${1:-/}"
echo "=== Top 10 大目录（${dir}）==="
du -sh "$dir"* 2>/dev/null | sort -hr | head -10
echo ""
echo "=== Top 10 大文件（${dir}）==="
find "$dir" -type f -size +100M -exec ls -lh {} \; 2>/dev/null \
| awk '{print $5, $NF}' | sort -hr | head -10
}
# find_large /var
# ===== 磁盘增长趋势 =====
# 每小时记录一次，观察增长速率
cat > disk_trend.sh << 'SCRIPT'
#!/bin/bash
LOG="/var/log/disk_usage.log"
echo "$(date '+%Y-%m-%d %H:%M:%S') $(df -h / | awk 'NR==2{print $3, $5}')" >> "$LOG"
# 分析最近 24 小时增长率
if [[ $(wc -l < "$LOG") -ge 2 ]]; then
tail -24 "$LOG" | awk '{
gsub(/%/, "", $4)
if (NR == 1) first = $4
last = $4
}
END {
growth = last - first
printf "24小时增长: %.1f%% (从 %.1f%% 到 %.1f%%)\n", growth, first, last
if (growth > 5) print "⚠️ 磁盘增长过快！"
}'
fi
SCRIPT
# 7.2.4 网络监控——流量/连接/端口
#!/bin/bash
# ===== 网络接口流量 =====
# 读取 /proc/net/dev 统计收发流量
network_traffic() {
local iface="${1:-eth0}"
awk -v iface="$iface" '$1 ~ iface":" {
printf "接收: %.2f MB | 发送: %.2f MB\n", $2/1048576, $10/1048576
}' /proc/net/dev
}
network_traffic eth0
# 注意：这是累计值，需两次采样相减算速率
# ===== 连接状态统计 =====
echo "=== TCP 连接状态 ==="
ss -tan | awk 'NR>1 {state[$1]++}
END {
for (s in state) printf "%-12s %d\n", s, state[s]
}' | sort -k2 -rn
# 重点关注 TIME_WAIT（过多说明短连接多）和 CLOSE_WAIT（应用未正确关闭连接）
echo ""
ss -tan state time-wait | wc -l | xargs echo "TIME_WAIT:"
ss -tan state close-wait | wc -l | xargs echo "CLOSE_WAIT:"
ss -tan state established | wc -l | xargs echo "ESTABLISHED:"
# ===== 端口监听检查 =====
check_ports() {
local required_ports=("80" "443" "22" "3306" "6379")
echo "=== 端口检查 ==="
for port in "${required_ports[@]}"; do
if ss -tlnp | grep -q ":$port "; then
echo "✅ 端口 $port 正常监听"
else
echo "❌ 端口 $port 未监听！"
fi
done
}
check_ports
# ===== 对外连接数 Top IP =====
ss -tnp | awk 'NR>1 {
split($5, dst, ":")
ip = dst[1]
if (ip !~ /^127|^::1|^\*$/) count[ip]++
}
END {
for (i in count) printf "%-20s %d\n", i, count[i]
}' | sort -k2 -rn | head -10
# ===== 网络连通性检测 =====
check_connectivity() {
local targets=("8.8.8.8" "1.1.1.1" "baidu.com")
for target in "${targets[@]}"; do
if ping -c 1 -W 2 "$target" > /dev/null 2>&1; then
echo "✅ $target 可达"
else
echo "❌ $target 不通"
fi
done
}
# check_connectivity
# 7.2.5 进程存活检测与自动重启
#!/bin/bash
# ===== 单个进程检测 =====
check_and_restart() {
local process_name="$1"
local start_cmd="$2"
if pgrep -x "$process_name" > /dev/null; then
echo "✅ $process_name 运行中 (PID: $(pgrep -x "$process_name" | tr '\n' ' '))"
return 0
else
echo "❌ $process_name 已停止，尝试重启..."
eval "$start_cmd"
sleep 3
if pgrep -x "$process_name" > /dev/null; then
echo "✅ $process_name 重启成功"
return 0
else
echo "💀 $process_name 重启失败！发送告警..."
return 1
fi
fi
}
# check_and_restart "nginx" "systemctl start nginx"
# check_and_restart "redis-server" "systemctl start redis"
# ===== 进程守护脚本（while+sleep 模式）=====
cat > process_guard.sh << 'SCRIPT'
#!/bin/bash
# 用法：./process_guard.sh nginx "systemctl start nginx" 10
# 每隔 10 秒检测 nginx，挂了就重启
PROCESS="$1"
START_CMD="$2"
INTERVAL="${3:-30}"
while true; do
if ! pgrep -x "$PROCESS" > /dev/null; then
echo "[$(date)] $PROCESS 挂了，正在重启..." >> /var/log/guard.log
eval "$START_CMD"
sleep 3
if pgrep -x "$PROCESS" > /dev/null; then
echo "[$(date)] $PROCESS 重启成功" >> /var/log/guard.log
else
echo "[$(date)] 💀 $PROCESS 重启失败！" >> /var/log/guard.log
fi
fi
sleep "$INTERVAL"
done
SCRIPT
# ===== 批量检测关键服务 =====
cat > service_health_check.sh << 'SCRIPT'
#!/bin/bash
# 拷贝为 /etc/cron.d/health_check 实现每 5 分钟检测
declare -A SERVICES=(
["nginx"]="systemctl restart nginx"
["php-fpm"]="systemctl restart php-fpm"
["mysqld"]="systemctl restart mysqld"
["redis-server"]="systemctl restart redis"
)
for svc in "${!SERVICES[@]}"; do
if systemctl is-active --quiet "$svc"; then
:
else
echo "[$(date)] $svc 服务异常，尝试重启" >> /var/log/service_guard.log
${SERVICES[$svc]}
fi
done
SCRIPT
# ===== 检测进程是否假死（端口监听但无响应）=====
check_port_response() {
local port="$1"
local timeout=3
local url="${2:-http://localhost:$port/}"
if timeout "$timeout" curl -sf "$url" > /dev/null 2>&1; then
echo "✅ 端口 $port 响应正常"
return 0
else
echo "❌ 端口 $port 无响应（进程可能假死）"
return 1
fi
}
# 7.2.6 定时采集与资源趋势分析
#!/bin/bash
# ===== 资源采集脚本（cron 每 1 分钟执行一次）=====
# crontab: * * * * * /path/to/collect_metrics.sh
cat > collect_metrics.sh << 'SCRIPT'
#!/bin/bash
DATA_DIR="/var/log/metrics"
mkdir -p "$DATA_DIR"
TS=$(date '+%Y-%m-%d %H:%M:%S')
# CPU 使用率
cpu=$(top -bn1 | grep "Cpu(s)" | awk '{print 100 - $8}')
# 内存使用率
mem_total=$(awk '/^MemTotal:/ {print $2}' /proc/meminfo)
mem_avail=$(awk '/^MemAvailable:/ {print $2}' /proc/meminfo)
mem_used=$(( (mem_total - mem_avail) * 100 / mem_total ))
# 磁盘使用率
disk=$(df / | awk 'NR==2 {gsub(/%/,"",$5); print $5}')
# 负载
load=$(awk '{print $1}' /proc/loadavg)
# 写入 CSV
echo "$TS,$cpu,${mem_used},$disk,$load" >> "$DATA_DIR/system.csv"
# 只保留最近 7 天的数据
find "$DATA_DIR" -name "*.csv" -mtime +7 -delete
SCRIPT
# ===== 趋势分析脚本 =====
cat > trend_analysis.sh << 'SCRIPT'
#!/bin/bash
DATA="$1"
echo "=== 过去 24 小时趋势 ==="
awk -F',' '{
cpu += $2; mem += $3; disk += $4; load += $5; n++
}
END {
printf "CPU 平均: %.1f%%\n", cpu/n
printf "内存平均: %.1f%%\n", mem/n
printf "磁盘平均: %.1f%%\n", disk/n
printf "负载平均: %.2f\n", load/n
}' "$DATA"
echo ""
echo "=== 峰值 ==="
awk -F',' '{
if ($2 > max_cpu) { max_cpu = $2; cpu_ts = $1 }
if ($3 > max_mem) { max_mem = $3; mem_ts = $1 }
if ($4 > max_disk) { max_disk = $4; disk_ts = $1 }
if ($5 > max_load) { max_load = $5; load_ts = $1 }
}
END {
printf "CPU 峰值: %.1f%% (%s)\n", max_cpu, cpu_ts
printf "内存峰值: %.1f%% (%s)\n", max_mem, mem_ts
printf "磁盘峰值: %.1f%% (%s)\n", max_disk, disk_ts
printf "负载峰值: %.2f (%s)\n", max_load, load_ts
}' "$DATA"
SCRIPT
# 7.3 综合告警系统实战
# 7.3.1 告警策略设计——多级阈值与冷却机制
一个成熟的告警系统需要三个核心机制：
#!/bin/bash
# ===== 告警级别定义 =====
# WARN = 预警，关注但不紧急（如 CPU 使用 70%）
# ERROR = 需要立即处理（如 CPU 使用 90%）
# FATAL = 致命，可能影响业务（如磁盘满、服务宕机）
# ===== 冷却机制 =====
# 同一个告警在冷却期内不重复发送（避免告警风暴）
COOLDOWN_WARN=600 # 10 分钟
COOLDOWN_ERROR=300 # 5 分钟
COOLDOWN_FATAL=60 # 1 分钟（致命告警需要持续提醒）
ALERT_STATE_DIR="/tmp/alert_state"
mkdir -p "$ALERT_STATE_DIR"
# ===== 通用告警函数 =====
send_alert() {
local level="$1" # WARN / ERROR / FATAL
local target="$2" # 告警目标（CPU/MEM/DISK...）
local value="$3" # 当前值
local threshold="$4" # 阈值
local message="$5" # 详细消息
local now=$(date +%s)
local state_file="$ALERT_STATE_DIR/${target}_${level}"
# 检查冷却
local cooldown_var="COOLDOWN_${level}"
local cooldown=${!cooldown_var}
local last_alert=$(cat "$state_file" 2>/dev/null || echo 0)
if (( now - last_alert < cooldown )); then
return # 冷却中，不发送
fi
# 更新状态文件
echo "$now" > "$state_file"
# 输出告警
echo "[$(date)] ${level} [${target}] 当前:${value} 阈值:${threshold} - ${message}"
# 这里接入通知通道（见下一节）
}
# ===== 综合检查 =====
comprehensive_check() {
# CPU
local cpu=$(top -bn1 | grep "Cpu(s)" | awk '{print 100 - $8}')
cpu=${cpu%.*}
if [[ "$cpu" -ge 95 ]]; then
send_alert "FATAL" "CPU" "$cpu" "95" "CPU 使用率过高，服务可能卡死"
elif [[ "$cpu" -ge 85 ]]; then
send_alert "ERROR" "CPU" "$cpu" "85" "CPU 持续高负载"
elif [[ "$cpu" -ge 70 ]]; then
send_alert "WARN" "CPU" "$cpu" "70" "CPU 使用率偏高"
fi
# 内存
local mem_total=$(awk '/^MemTotal:/ {print $2}' /proc/meminfo)
local mem_avail=$(awk '/^MemAvailable:/ {print $2}' /proc/meminfo)
local mem_percent=$(( (mem_total - mem_avail) * 100 / mem_total ))
if [[ "$mem_percent" -ge 95 ]]; then
send_alert "FATAL" "MEM" "${mem_percent}%" "95" "可用内存严重不足，OOM 风险"
elif [[ "$mem_percent" -ge 85 ]]; then
send_alert "ERROR" "MEM" "${mem_percent}%" "85" "内存使用率过高"
fi
# 磁盘
local disk=$(df / | awk 'NR==2 {gsub(/%/,"",$5); print $5}')
if [[ "$disk" -ge 95 ]]; then
send_alert "FATAL" "DISK" "${disk}%" "95" "磁盘即将写满！"
elif [[ "$disk" -ge 85 ]]; then
send_alert "ERROR" "DISK" "${disk}%" "85" "磁盘空间不足"
elif [[ "$disk" -ge 75 ]]; then
send_alert "WARN" "DISK" "${disk}%" "75" "磁盘使用率偏高"
fi
# 负载
local load=$(awk '{print $1}' /proc/loadavg)
local cores=$(nproc)
if (( $(echo "$load > $cores * 2" | bc -l) )); then
send_alert "ERROR" "LOAD" "$load" "$((cores*2))" "系统负载过高"
elif (( $(echo "$load > $cores" | bc -l) )); then
send_alert "WARN" "LOAD" "$load" "$cores" "负载超过核心数"
fi
}
# comprehensive_check
# 7.3.2 告警通知——邮件/钉钉/企业微信
#!/bin/bash
# ===== 1. 邮件告警（通过 mail/mailx 命令）=====
send_email_alert() {
local subject="$1"
local body="$2"
local recipients="${3:-admin@example.com}"
echo "$body" | mail -s "$subject" "$recipients"
# 或使用 sendmail / msmtp 等 MTA
}
# sendmail 方式（更可靠，特别是服务器上）
send_email_sendmail() {
local to="$1"
local subject="$2"
local body="$3"
/usr/sbin/sendmail -t << EMAIL_EOF
To: $to
Subject: $subject
Content-Type: text/plain; charset=utf-8
$body
EMAIL_EOF
}
# ===== 2. 钉钉机器人告警 =====
DINGTALK_WEBHOOK="https://oapi.dingtalk.com/robot/send?access_token=YOUR_TOKEN"
send_dingtalk() {
local title="$1"
local content="$2"
local level="${3:-WARN}"
# 根据级别选择颜色
case "$level" in
FATAL) color="#FF0000" ;;
ERROR) color="#FF6600" ;;
WARN) color="#FFAA00" ;;
*) color="#0088CC" ;;
esac
local json=$(cat << JSON
{
"msgtype": "markdown",
"markdown": {
"title": "$title",
"text": "## ${level} <font color=${color}>${title}</font>\n\n${content}\n\n> 发送时间: $(date '+%Y-%m-%d %H:%M:%S')"
}
}
JSON
)
curl -s -X POST -H "Content-Type: application/json" -d "$json" "$DINGTALK_WEBHOOK" > /dev/null
}
# ===== 3. 企业微信机器人告警 =====
WECOM_WEBHOOK="https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=YOUR_KEY"
send_wecom() {
local content="$1"
local level="${2:-WARN}"
# 根据级别造前缀 emoji
case "$level" in
FATAL) prefix="💀【致命】" ;;
ERROR) prefix="🚨【错误】" ;;
WARN) prefix="⚠️ 【预警】" ;;
*) prefix="ℹ️ 【信息】" ;;
esac
local json="{\"msgtype\":\"text\",\"text\":{\"content\":\"${prefix}\n${content}\n\n时间: $(date '+%Y-%m-%d %H:%M:%S')\"}}"
curl -s -X POST -H "Content-Type: application/json" -d "$json" "$WECOM_WEBHOOK" > /dev/null
}
# ===== 通用通知路由器（自动选渠道）=====
notify() {
local title="$1"
local body="$2"
local level="$3"
# 致命告警：所有渠道都发
# 错误告警：钉钉 + 邮件
# 预警：仅钉钉
case "$level" in
FATAL)
send_dingtalk "$title" "$body" "FATAL"
send_wecom "$title" "$body" "FATAL"
send_email_sendmail "admin@example.com" "[FATAL] $title" "$body"
;;
ERROR)
send_dingtalk "$title" "$body" "ERROR"
send_email_sendmail "admin@example.com" "[ERROR] $title" "$body"
;;
WARN|*)
send_dingtalk "$title" "$body" "WARN"
;;
esac
}
# 7.3.3 一体化监控告警脚本——从采集到通知
#!/bin/bash
# ============================================
# 一体化监控告警脚本
# 用法：crontab 中配置 */5 * * * * /path/to/monitor.sh
# 推荐频率：每 5 分钟一次
# ============================================
set -euo pipefail
# ---- 配置区（修改为你自己的值）----
CONFIG_FILE="${HOME}/.monitor.conf"
if [[ -f "$CONFIG_FILE" ]]; then
source "$CONFIG_FILE"
fi
# 阈值设置
CPU_WARN=80
CPU_ERROR=90
MEM_WARN=85
MEM_ERROR=95
DISK_WARN=80
DISK_ERROR=90
LOAD_RATIO=1.5 # 负载超过核心数 * 此值的倍数
# 告警冷却（秒）
COOLDOWN=300 # 5 分钟
# Webhook
DING_WEBHOOK="${DING_WEBHOOK:-}"
WECOM_WEBHOOK="${WECOM_WEBHOOK:-}"
EMAIL_TO="${EMAIL_TO:-root@localhost}"
# 日志
LOG_FILE="/var/log/system_monitor.log"
ALERT_DIR="/tmp/monitor_alerts"
mkdir -p "$ALERT_DIR"
# ---- 工具函数 ----
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >> "$LOG_FILE"; echo "$*"; }
get_metric() { cat "$ALERT_DIR/${1}.last" 2>/dev/null || echo ""; }
set_metric() { echo "$2" > "$ALERT_DIR/${1}.last"; }
is_cooldown() {
local key="$1"
local ts_file="$ALERT_DIR/${key}.ts"
local now=$(date +%s)
local last=$(cat "$ts_file" 2>/dev/null || echo 0)
if (( now - last < COOLDOWN )); then return 0; fi
echo "$now" > "$ts_file"
return 1
}
# ---- 通知通道 ----
ding_notify() {
[[ -z "$DING_WEBHOOK" ]] && return
curl -s -X POST -H "Content-Type: application/json" \
-d "{\"msgtype\":\"text\",\"text\":{\"content\":\"[$(hostname)] $1\"}}" \
"$DING_WEBHOOK" > /dev/null
}
wecom_notify() {
[[ -z "$WECOM_WEBHOOK" ]] && return
curl -s -X POST -H "Content-Type: application/json" \
-d "{\"msgtype\":\"text\",\"text\":{\"content\":\"[$(hostname)] $1\"}}" \
"$WECOM_WEBHOOK" > /dev/null
}
alert() {
local level="$1"; shift
local msg="$*"
log "[${level}] $msg"
# 冷却检查（FATAL 不停发）
if [[ "$level" != "FATAL" ]]; then
local key=$(echo "$msg" | md5sum | cut -c1-8)
is_cooldown "$key" && return
fi
# 发送通知
case "$level" in
FATAL) ding_notify "💀 [致命] $msg"; wecom_notify "💀 [致命] $msg" ;;
ERROR) ding_notify "🚨 [错误] $msg" ;;
WARN) ding_notify "⚠️ [预警] $msg" ;;
esac
}
# ---- 采集函数 ----
collect_cpu() {
local cpu=$(top -bn1 2>/dev/null | grep "Cpu(s)" | awk '{print 100 - $8}' 2>/dev/null || echo "0")
cpu=${cpu%.*}
local prev=$(get_metric "cpu")
set_metric "cpu" "$cpu"
if [[ "$cpu" -ge "$CPU_ERROR" ]]; then
alert "ERROR" "CPU 使用率 ${cpu}%，阈值 ${CPU_ERROR}%"
elif [[ "$cpu" -ge "$CPU_WARN" ]]; then
alert "WARN" "CPU 使用率 ${cpu}%，阈值 ${CPU_WARN}%"
fi
log " CPU: ${cpu}%"
}
collect_memory() {
local total=$(awk '/^MemTotal:/ {print $2}' /proc/meminfo)
local avail=$(awk '/^MemAvailable:/ {print $2}' /proc/meminfo)
local percent=$(( (total - avail) * 100 / total ))
set_metric "mem" "$percent"
if [[ "$percent" -ge "$MEM_ERROR" ]]; then
alert "ERROR" "内存使用 ${percent}%，阈值 ${MEM_ERROR}%"
elif [[ "$percent" -ge "$MEM_WARN" ]]; then
alert "WARN" "内存使用 ${percent}%，阈值 ${MEM_WARN}%"
fi
log " MEM: ${percent}%"
}
collect_disk() {
while IFS= read -r line; do
local device=$(echo "$line" | awk '{print $1}')
local percent=$(echo "$line" | awk '{gsub(/%/,"",$5); print $5}')
local mount=$(echo "$line" | awk '{print $6}')
if [[ "$percent" -ge "$DISK_ERROR" ]]; then
alert "ERROR" "磁盘 ${mount} 使用 ${percent}%，阈值 ${DISK_ERROR}%"
elif [[ "$percent" -ge "$DISK_WARN" ]]; then
alert "WARN" "磁盘 ${mount} 使用 ${percent}%，阈值 ${DISK_WARN}%"
fi
log " DISK ${mount}: ${percent}%"
done < <(df -h | awk 'NR>1 && /^\/dev\//')
}
collect_load() {
local load=$(awk '{print $1}' /proc/loadavg)
local cores=$(nproc)
set_metric "load" "$load"
local threshold=$(echo "$cores * $LOAD_RATIO" | bc)
if (( $(echo "$load > $threshold" | bc -l) )); then
alert "WARN" "负载 ${load}，核心数 ${cores}，阈值 ${threshold}"
fi
log " LOAD: ${load} (cores: ${cores})"
}
collect_services() {
local services=("nginx" "php-fpm" "mysqld" "redis-server" "sshd" "docker")
for svc in "${services[@]}"; do
if systemctl is-active --quiet "$svc" 2>/dev/null; then
:
else
if systemctl list-unit-files "$svc.service" &>/dev/null; then
alert "ERROR" "服务 $svc 未运行！尝试重启..."
systemctl restart "$svc" 2>/dev/null || \
alert "FATAL" "服务 $svc 重启失败！"
fi
fi
done
}
# ---- 主流程 ----
main() {
log "===== 开始巡检 ====="
collect_load
collect_cpu
collect_memory
collect_disk
collect_services
log "===== 巡检结束 ====="
}
main
# 7.3.4 新手陷阱 Top 5
陷阱 1：top -bn1 第一次不准
# ❌ top 第一次采样可能不准确
top -bn1 | grep "Cpu(s)"
# ✅ 取第 2 次迭代或者用 /proc/stat 直接算
top -bn2 | grep "Cpu(s)" | tail -1
# 或用 mpstat：mpstat 1 1 | awk 'END{print 100-$NF}'
陷阱 2：内存监控用错指标
# ❌ 用 free 的 "available" 行看可用内存
# MemFree 不等于真实可用（Linux 会用空闲内存做缓存）
# ❌ 错误：认为 buff/cache 是"已用内存"
free -m | awk 'NR==2{printf "%d%%\n", $3*100/$2}'
# ✅ 用 MemAvailable（精确反映可用于新进程的内存）
awk '/^MemAvailable:/ {avail=$2} /^MemTotal:/ {total=$2}
END {printf "%d%%\n", (total-avail)*100/total}' /proc/meminfo
陷阱 3：df 和 du 统计不一致
# df 显示磁盘已满，但 du 统计文件远远没到
# 原因：已删除但仍被进程持有的文件（deleted but open）
# 检查是否有大量被删除但仍占用的文件
lsof | grep deleted | wc -l
lsof | grep deleted | awk '{sum+=$7} END {printf "%.2f MB\n", sum/1048576}'
# 重启持有这些文件的进程即可释放空间
陷阱 4：crontab 环境变量不同
# ❌ crontab 执行脚本时 PATH 极短，很多命令找不到
# 脚本里用绝对路径或先 source profile
# ✅ 在脚本开头明确设置
#!/bin/bash
export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
source /etc/profile 2>/dev/null || true
陷阱 5：告警脚本本身占用资源
# ❌ 监控脚本内部有死循环、递归、未限制频率的日志写入
# 结果：监控脚本变成系统最大负担
# ✅ 设置资源限制
ulimit -t 30 # CPU 时间限制 30 秒
ulimit -v 262144 # 虚拟内存限制 256MB
# 或用 timeout 包裹：timeout 30 monitor.sh
# 7.3.5 综合思考题
日志关联分析：如何从 Nginx 访问日志 + 错误日志中，关联找出导致 502 错误的真正元凶（比如上游 PHP-FPM 进程在那一秒正好超时）？
OOM 预防：写一个脚本，当可用内存低于 10% 时，主动找出占用内存最多的进程，发送告警并记录 fast 快照（/proc/
/smaps 摘要），以便事后分析。 磁盘增长率预警：如何在磁盘使用率 70% 时，根据最近 7 天的增长率预测"磁盘将在多少天后满"，提前发出预警？
告警抑制链：设计一个逻辑——当"网络不通"告警触发时，自动抑制"服务不可达"类型的下游告警，避免告警风暴。
综合脚本：编写一个
health_check.sh，检测以下项并生成 JSON 格式的健康报告：- 磁盘使用率 / Inode 使用率
- 内存使用率 / Swap 使用率
- CPU 使用率 / 负载
- 关键端口（80/443/3306/6379）监听状态
- 关键进程（nginx/mysql/redis/php-fpm）存活状态
- 最近的 5xx 错误数（过去 5 分钟）
上次更新: 2026/06/28, 17:55:19
