日志监控与告警

# 第 7 章日志监控与告警

# 目录介绍

7.1 日志分析
7.2 系统监控
7.3 综合告警系统实战

# 7.1 日志分析

# 7.1.1 常见日志格式解析

能看懂日志格式是分析的基础。以下是运维中最常见的几种：

#!/bin/bash

# ===== 1. Nginx / Apache 访问日志（Combined 格式）=====
# 示例行：
# 192.168.1.100 - - [10/Jun/2025:14:30:00 +0800] "GET /api/users HTTP/1.1" 200 1234 "https://example.com" "Mozilla/5.0"
#  $1             $2 $3  $4                     $5            $6      $7   $8  $9                 $10

# 字段解析：
#  $1 = 客户端 IP
#  $2 = 远程登录名（通常为 -）
#  $3 = 远程用户名（通常为 -）
#  $4 = 时间戳 [日/月/年:时:分:秒 时区]
#  $5 = 请求行 = 方法 URL 协议
#  $6 = HTTP 方法（GET/POST...）
#  $7 = 请求 URL 路径
#  $8 = HTTP 协议版本
#  $9 = HTTP 状态码（200/301/404/500...）
#  $10= 响应字节数
#  $11= Referer
#  $12= User-Agent

# ===== Nginx access.log 一行解析脚本 =====
cat > parse_nginx_log.sh << 'EOF'
#!/bin/bash
LOG_FILE="${1:-/var/log/nginx/access.log}"

awk '{
    # 提取时间戳
    gsub(/[\[\]]/, "", $4)     # 去掉方括号 [ ]
    timestamp = $4 " " $5      # 时间 + 时区

    # 提取请求信息
    gsub(/"/, "", $6)          # 去掉引号
    method = $6
    url = $7

    # 提取关键指标
    ip = $1
    status = $9
    bytes = $10

    # 输出解析结果
    printf "[%s] %s %-7s %s → HTTP %d, %d bytes\n",
        timestamp, ip, method, url, status, bytes
}' "$LOG_FILE"
EOF

# ===== 2. Apache 错误日志格式 =====
# [Mon Jun 10 14:30:00.123456 2025] [core:error] [pid 12345] [client 1.2.3.4:56789] AH00124: Request exceeded the limit

# ===== 3. Syslog / 系统日志格式 =====
# Jun 10 14:30:00 server01 sshd[12345]: Failed password for root from 1.2.3.4 port 22 ssh2

# ===== 4. 应用日志（常见自定义格式）=====
# 2025-06-10 14:30:00.123 [ERROR] [main-thread] com.example.Service - Connection timeout after 30000ms
#   $1          $2       $3     $4            $5                 $6

# ===== 实战：通用日志字段提取函数 =====
extract_log_fields() {
    local format="$1"      # nginx / syslog / app
    local line="$2"

    case "$format" in
        nginx)
            echo "$line" | awk '{print "IP:",$1," 状态:",$9," URL:",$7," 字节:",$10}'
            ;;
        syslog)
            echo "$line" | awk '{print "时间:",$1,$2,$3," 主机:",$4," 进程:",$5," 内容:",substr($0, index($0,$6))}'
            ;;
        app)
            echo "$line" | awk -F'[][]' '{
                split($1, t, " ")
                print "时间:", t[1], t[2], " 级别:", $2, " 消息:", $4
            }'
            ;;
    esac
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77

# 7.1.2 访问量统计——PV/UV/QPS/带宽

#!/bin/bash

LOG="/var/log/nginx/access.log"

# ===== PV (Page View) —— 总请求量 =====
echo "=== PV 统计 ==="
wc -l "$LOG"
# 输出：1234567 /var/log/nginx/access.log

# 按小时统计 PV（看当天流量高峰）
awk '{split($4, t, ":"); hour=t[2]; count[hour]++}
END {
    for (h=0; h<=23; h++) printf "%02d:00 %6d\n", h, count[h]+0
}' "$LOG" | sort -t':' -k1n

# ===== UV (Unique Visitor) —— 独立 IP 数 =====
echo "=== UV 统计 ==="
awk '{print $1}' "$LOG" | sort -u | wc -l

# 按小时 UV（更精细）
awk '{split($4, t, ":"); hour=t[2]; uv[hour][$1]=1}
END {
    for (h=0; h<=23; h++) {
        count=0; for (ip in uv[h]) count++
        printf "%02d:00 %6d\n", h, count
    }
}' "$LOG" | sort -t':' -k1n

# ===== QPS (Query Per Second) —— 每秒请求数 =====
echo "=== QPS 统计 ==="
# 统计最近 1 分钟的 QPS
tail -1000 "$LOG" | awk -v interval=60 '{
    split($4, t, ":")
    timestamp = t[2]":"t[3]":"t[4]
    count[timestamp]++
}
END {
    total = 0; n = 0
    for (ts in count) { total += count[ts]; n++ }
    printf "最近 %d 条日志，平均 QPS: %.2f\n", 1000, total/(n*1.0)
}'

# 更精确的 QPS（按秒统计）
awk '{
    split($4, t, ":")
    sec = sprintf("%s:%s:%s:%s", t[2], t[3], t[4], substr(t[5],1,2))
    qps[sec]++
}
END {
    for (s in qps) if (qps[s] > 100) print s, qps[s]
}' "$LOG" | sort -k2 -rn | head -10
# 输出 QPS 超过 100 的时刻（用于定位瞬时高峰）

# ===== 带宽 / 流量统计 =====
echo "=== 流量统计 ==="
awk '{
    split($4, t, ":")
    hour = t[2]+0
    bytes_total[hour] += $10
    count[hour]++
}
END {
    printf "%-8s %12s %12s %12s\n", "小时", "请求数", "流量(MB)", "平均(KB/请求)"
    for (h=0; h<=23; h++) {
        if (count[h] > 0) {
            printf "%02d:00   %10d %10.2f %10.2f\n",
                h, count[h], bytes_total[h]/1048576,
                (bytes_total[h]/count[h])/1024
        }
    }
}' "$LOG" | sort -t':' -k1n

# ===== 一行汇总：今日核心指标 =====
awk 'BEGIN { split("Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec", m)
    for (i in m) month_num[m[i]] = sprintf("%02d", i) }
{
    ip_count[$1]++
    if ($9 >= 500) err5xx++
    else if ($9 >= 400) err4xx++
    success++
    total_bytes += $10
    # 提取当前日期
    split($4, t, "/"); split(t[1], date, " ")
    today = date[2] "-" month_num[date[1]] "-" strftime("%Y")
}
END {
    printf "===== %s 日志日报 =====\n", today
    printf "总请求: %d\n", NR
    printf "独立IP: %d\n", length(ip_count)
    printf "成功: %d (%.1f%%)\n", success, success/NR*100
    printf "4xx: %d (%.1f%%)\n", err4xx+0, err4xx/NR*100
    printf "5xx: %d (%.1f%%)\n", err5xx+0, err5xx/NR*100
    printf "总流量: %.2f MB\n", total_bytes/1048576
}' "$LOG"

# 7.1.3 Top IP 分析与异常流量检测

#!/bin/bash

LOG="/var/log/nginx/access.log"

# ===== Top 10 访问 IP =====
awk '{print $1}' "$LOG" | sort | uniq -c | sort -rn | head -10

# ===== 异常 IP 检测：请求量远超平均的 IP =====
awk '{ip[$1]++; total++}
END {
    avg = total / length(ip)
    printf "总独立IP: %d, 平均每IP请求: %.1f\n\n", length(ip), avg
    printf "%-20s %8s %8s %8s\n", "IP", "请求数", "占比%", "倍数"
    for (i in ip) {
        if (ip[i] > avg * 3) {     # 超过平均 3 倍 = 异常
            printf "%-20s %8d %7.1f%% %7.1fx\n", i, ip[i], ip[i]/total*100, ip[i]/avg
        }
    }
}' "$LOG"

# ===== 检测恶意请求特征 =====
# 1. 单一 IP 大量 404（目录扫描）
awk '$9 == 404 {ip[$1]++}
END {
    for (i in ip) if (ip[i] > 50) printf "疑似扫描 IP: %s (404=%d次)\n", i, ip[i]
}' "$LOG"

# 2. 单一 IP 大量 5xx（攻击导致服务异常）
awk '$9 >= 500 {ip[$1]++; err[$1][$9]++}
END {
    for (i in ip) if (ip[i] > 20) {
        printf "异常 IP: %s (5xx=%d次)", i, ip[i]
        for (code in err[i]) printf " HTTP %d=%d", code, err[i][code]
        printf "\n"
    }
}' "$LOG"

# 3. User-Agent 异常（爬虫/工具扫描）
awk '{
    # 提取 User-Agent（Combined 格式的第 12 个字段）
    match($0, /"[^"]*"$/)
    ua = substr($0, RSTART+1, RLENGTH-2)
    ua_count[ua]++
}
END {
    for (u in ua_count) {
        if (ua_count[u] > 100) printf "%d 次 - %s\n", ua_count[u], u
    }
}' "$LOG" | sort -rn | head -10

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

# 7.1.4 状态码分布与异常监控

#!/bin/bash

LOG="/var/log/nginx/access.log"

# ===== 状态码分布统计 =====
awk '{code[$9]++; total++}
END {
    printf "%-10s %8s %8s\n", "状态码", "数量", "占比"
    printf "%-10s %8s %8s\n", "------", "----", "----"
    for (c in code) {
        printf "HTTP %-5s %8d %7.1f%%\n", c, code[c], code[c]/total*100
    }
}' "$LOG" | sort

# ===== 按小时看状态码变化（找故障时间点）=====
awk '{
    split($4, t, ":")
    hour = t[2]+0
    codes[hour][$9]++
}
END {
    printf "%-7s", "Hour"
    for (c=200; c<=504; c++) if (c==200||c==301||c==302||c==304||c==400||c==403||c==404||c==500||c==502||c==503||c==504) printf "%7d", c
    printf "\n"
    for (h=0; h<=23; h++) {
        printf "%02d:00 ", h
        for (c=200; c<=504; c++)
            if (c==200||c==301||c==302||c==304||c==400||c==403||c==404||c==500||c==502||c==503||c==504)
                printf "%7d", codes[h][c]+0
        printf "\n"
    }
}' "$LOG"

# ===== 5xx 错误实时监控（配合 tail -f）=====
# 使用方式：tail -f access.log | ./monitor_5xx.sh
cat > monitor_5xx.sh << 'EOF'
#!/bin/bash
THRESHOLD=10           # 每分钟超过 10 个 5xx 就告警
INTERVAL=60            # 统计周期（秒）
COUNT_FILE="/tmp/5xx_count_$$"

while read -r line; do
    status=$(echo "$line" | awk '{print $9}')
    if [[ "$status" -ge 500 ]] 2>/dev/null; then
        echo "$(date +%s)" >> "$COUNT_FILE"
    fi
done < <(tail -f /var/log/nginx/access.log) &
MONITOR_PID=$!

# 定时检查
while true; do
    sleep "$INTERVAL"
    now=$(date +%s)
    cutoff=$((now - INTERVAL))
    count=$(awk -v c="$cutoff" '$1 > c' "$COUNT_FILE" 2>/dev/null | wc -l)

    if [[ "$count" -ge "$THRESHOLD" ]]; then
        echo "[$(date)] ⚠️ 告警：过去 ${INTERVAL}s 内出现 $count 个 5xx 错误！"
        # 重置计数文件
        : > "$COUNT_FILE"
    fi
done

trap "kill $MONITOR_PID 2>/dev/null; rm -f $COUNT_FILE" EXIT
EOF

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65

# 7.1.5 响应时间分析——揪出慢请求

Nginx 可配置 $request_time（处理时间）和 $upstream_response_time（上游响应时间），需先在 log_format 中添加：

log_format timed '$remote_addr - [$time_local] "$request" $status $body_bytes_sent '
                  '"$http_referer" "$http_user_agent" $request_time $upstream_response_time';

1
2

#!/bin/bash

LOG="/var/log/nginx/timed_access.log"
# 假设最后两列是 $request_time 和 $upstream_response_time

# ===== Top 10 慢请求 =====
awk '{print $(NF-1), $7}' "$LOG" | sort -rn | head -10

# ===== 响应时间分桶统计 =====
awk '{
    rt = $(NF-1) + 0
    if (rt < 0.1)     bucket["0~100ms"]++
    else if (rt < 0.5) bucket["100~500ms"]++
    else if (rt < 1)   bucket["0.5~1s"]++
    else if (rt < 3)   bucket["1~3s"]++
    else if (rt < 5)   bucket["3~5s"]++
    else               bucket["5s+"]++
    total++
}
END {
    for (b in bucket) printf "%-12s %8d %6.1f%%\n", b, bucket[b], bucket[b]/total*100
}' "$LOG" | sort -k2 -rn

# ===== 按 URL 统计平均响应时间 =====
awk '{
    url = $7
    rt = $(NF-1) + 0
    url_count[url]++
    url_rt_total[url] += rt
    if (rt > url_rt_max[url]) url_rt_max[url] = rt
}
END {
    printf "%-50s %8s %8s %10s\n", "URL", "次数", "平均(秒)", "最大(秒)"
    for (u in url_count) {
        if (url_count[u] >= 10) {   # 至少被请求 10 次才统计
            avg = url_rt_total[u] / url_count[u]
            printf "%-50s %8d %8.3f %10.3f\n", u, url_count[u], avg, url_rt_max[u]
        }
    }
}' "$LOG" | sort -k3 -rn | head -20

# ===== 检测上游响应慢（$upstream_response_time > 阈值）=====
awk '$(NF) > 3 {    # 上游响应超过 3 秒
    print $1, $4, $7, "上游:", $(NF), "总耗时:", $(NF-1)
}' "$LOG"

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45

# 7.1.6 错误日志提取、分类与告警

#!/bin/bash

ERROR_LOG="/var/log/nginx/error.log"

# ===== 提取所有错误并分类 =====
# 按错误类型分组统计
awk '{
    # 提取错误级别（第一个方括号中的内容）
    match($0, /\[([a-z]+)\]/, arr)
    level = arr[1]

    # 提取简要信息
    msg = $0
    gsub(/^.*\] /, "", msg)    # 去掉前缀
    msg = substr(msg, 1, 80)   # 截断

    errors[level]++
    if (errors[level] <= 3) samples[level] = samples[level] "\n  " msg
}
END {
    for (l in errors) {
        printf "[%s] %d 次\n%s\n\n", toupper(l), errors[l], samples[l]
    }
}' "$ERROR_LOG"

# ===== 错误趋势：每分钟错误数 =====
awk '{
    match($0, /[0-9]{4}\/[0-9]{2}\/[0-9]{2} [0-9]{2}:[0-9]{2}/)
    minute = substr($0, RSTART, 16)
    count[minute]++
}
END {
    for (m in count) print m, count[m]
}' "$ERROR_LOG" | sort | tail -30

# ===== 应用中 OOM / 超时 / 连接失败 分类 =====
cat > classify_errors.sh << 'EOF'
#!/bin/bash
LOG="${1:-/var/log/app/error.log}"

declare -A PATTERNS=(
    ["OOM/内存溢出"]="OutOfMemory|java\.lang\.OutOfMemoryError|OOM killer"
    ["连接超时"]="timeout|timed out|Connection refused"
    ["数据库错误"]="SQLException|Connection pool|too many connections"
    ["磁盘空间"]="No space left|Disk quota"
    ["权限错误"]="Permission denied|Access denied"
    ["第三方API失败"]="api.*error|HTTP \d{3}.*error"
    ["空指针/非法参数"]="NullPointerException|IllegalArgument"
    ["死锁"]="Deadlock|Lock wait timeout"
)

echo "===== 错误分类统计 ====="
for category in "${!PATTERNS[@]}"; do
    count=$(grep -cP "${PATTERNS[$category]}" "$LOG" 2>/dev/null)
    if [[ "$count" -gt 0 ]]; then
        printf "%-20s %d 次\n" "$category" "$count"
    fi
done
EOF

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59

# 7.2 系统监控

# 7.2.1 CPU 监控——使用率/负载/进程排行

#!/bin/bash

# ===== CPU 使用率（实时）=====
cpu_usage() {
    # 读取 /proc/stat 计算 CPU 使用率（用户态 + 系统态 + IO 等待）
    local idle1 idle2 total1 total2

    read -r _ user1 nice1 system1 idle1 iowait1 irq1 softirq1 steal1 _ < /proc/stat
    idle1=$((idle1 + iowait1))
    total1=$((user1 + nice1 + system1 + idle1 + iowait1 + irq1 + softirq1 + steal1))

    sleep 1

    read -r _ user2 nice2 system2 idle2 iowait2 irq2 softirq2 steal2 _ < /proc/stat
    idle2=$((idle2 + iowait2))
    total2=$((user2 + nice2 + system2 + idle2 + iowait2 + irq2 + softirq2 + steal2))

    local diff_idle=$((idle2 - idle1))
    local diff_total=$((total2 - total1))

    echo "scale=2; (1 - $diff_idle / $diff_total) * 100" | bc
}

echo "CPU 使用率: $(cpu_usage)%"

# ===== 系统负载（Load Average）=====
check_load() {
    local load1 load5 load15 cores
    read -r load1 load5 load15 _ < /proc/loadavg
    cores=$(nproc)

    echo "Load: 1min=${load1}  5min=${load5}  15min=${load15}  (CPU核数: ${cores})"

    # 负载告警：1min 负载 > CPU 核数
    if (( $(echo "$load1 > $cores * 1.5" | bc -l) )); then
        echo "⚠️ 1 分钟负载 $load1 超过核心数 $cores 的 1.5 倍！"
    fi
}
check_load

# ===== Top 5 CPU 进程 =====
ps aux --sort=-%cpu | head -6 | awk '{
    if (NR == 1) { printf "%-8s %-20s %6s %6s %s\n", "PID", "进程", "CPU%", "MEM%", "命令" }
    else { printf "%-8s %-20s %5.1f%% %5.1f%% %s\n", $2, substr($11,1,20), $3, $4, substr($0, index($0,$11)) }
}'

# ===== 持续 CPU 监控脚本 =====
cat > cpu_monitor.sh << 'SCRIPT'
#!/bin/bash
THRESHOLD=80                    # CPU 使用率阈值（%）
COOLDOWN=300                    # 冷却时间（秒），避免重复告警
LAST_ALERT_FILE="/tmp/last_cpu_alert"

while true; do
    # 获取 CPU 使用率（取 top 输出）
    cpu=$(top -bn1 | grep "Cpu(s)" | awk '{print $2}' | cut -d'%' -f1)
    cpu=${cpu%.*}                # 去掉小数

    if [[ "$cpu" -ge "$THRESHOLD" ]]; then
        now=$(date +%s)
        last_alert=$(cat "$LAST_ALERT_FILE" 2>/dev/null || echo 0)

        if (( now - last_alert > COOLDOWN )); then
            echo "[$(date)] ⚠️ CPU 使用率 ${cpu}%，超过阈值 ${THRESHOLD}%"
            echo "$now" > "$LAST_ALERT_FILE"
            # 记录 Top 进程快照
            ps aux --sort=-%cpu | head -6 >> /var/log/cpu_alert.log
        fi
    fi
    sleep 10
done
SCRIPT

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72

# 7.2.2 内存监控——使用率/OOM 风险/Swap

#!/bin/bash

# ===== 内存概览 =====
free -h | awk '{
    if (NR==1) print "===== 内存概览 ====="
    if (NR==2) printf "总内存: %8s  已用: %8s  可用: %8s\n", $2, $3, $7
    if (NR==3) printf "Swap:    %8s  已用: %8s  空闲: %8s\n", $2, $3, $4
}'

# ===== 详细内存使用 =====
check_memory() {
    local total used avail percent

    # 从 /proc/meminfo 精确读取
    total=$(awk '/^MemTotal:/ {print $2}' /proc/meminfo)
    avail=$(awk '/^MemAvailable:/ {print $2}' /proc/meminfo)
    used=$((total - avail))
    percent=$((used * 100 / total))

    echo "内存使用: ${percent}% (已用: $((used/1024))MB / 总共: $((total/1024))MB)"

    # OOM 风险告警：可用内存 < 10%
    if [[ "$percent" -ge 90 ]]; then
        echo "⚠️ 严重：可用内存不足 10%，存在 OOM 风险！"

        # 列出 OOM Score 最高的进程（最有可能被 kill 的）
        echo ""
        echo "OOM Killer 候选进程（Score 最高 Top 5）："
        for pid in $(ps aux --sort=-%mem | awk 'NR>1{print $2}' | head -5); do
            if [[ -f /proc/$pid/oom_score ]]; then
                score=$(cat /proc/$pid/oom_score)
                name=$(cat /proc/$pid/comm 2>/dev/null)
                echo "  PID:$pid  Score:$score  Name:$name"
            fi
        done
    fi
}
check_memory

# ===== Swap 使用监控 =====
check_swap() {
    local total used percent
    total=$(awk '/^SwapTotal:/ {print $2}' /proc/meminfo)
    used=$(awk '/^SwapFree:/ {print $2}' /proc/meminfo)
    used=$((total - used))
    percent=$((used * 100 / total))

    echo "Swap 使用: ${percent}%"

    # Swap 使用 > 50% 意味着物理内存不足
    if [[ "$percent" -ge 50 ]]; then
        echo "⚠️ Swap 使用率 ${percent}%，物理内存可能不足，建议排查"
    fi
}
check_swap

# ===== Top 5 内存进程 =====
ps aux --sort=-%mem | head -6 | awk '{
    if (NR==1) printf "%-8s %6s %-20s %s\n", "PID", "MEM%", "进程", "RSS(MB)"
    else printf "%-8s %5.1f%% %-20s %6d\n", $2, $4, substr($11,1,20), $6/1024
}'

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61

# 7.2.3 磁盘监控——空间/inode/IO

#!/bin/bash

# ===== 磁盘空间概览 =====
df -h | awk 'NR==1 || /^\/dev\//'            # 只显示物理磁盘

# ===== 磁盘使用率告警 =====
check_disk() {
    local threshold=80
    df -h | awk -v th="$threshold" 'NR>1 && /^\/dev\// {
        gsub(/%/, "", $5)
        if ($5+0 >= th) {
            printf "⚠️ 磁盘 %-20s 使用率: %d%%  (已用:%s / 总共:%s)\n",
                $6, $5, $3, $2
        }
    }'
}
check_disk

# ===== Inode 使用率检测 =====
# Inode 耗尽也会导致"磁盘满"的错误（No space left on device）
df -i | awk 'NR>1 && /^\/dev\// {
    gsub(/%/, "", $5)
    if ($5+0 >= 80) {
        printf "⚠️ Inode 使用率过高：%s %d%%\n", $6, $5
    }
}'

# ===== 磁盘 I/O 监控 =====
# 使用 iostat 查看 IO 负载
iostat -x 1 2 | awk 'END {
    printf "%-10s %8s %8s %8s %8s\n", "设备", "r/s", "w/s", "await", "util%"
}
NR > 6 && $1 ~ /^sd|^nvme/ {
    printf "%-10s %8.1f %8.1f %8.1f %7.1f%%\n", $1, $4, $5, $10, $14
}'
# await > 50ms 或 util% > 80% 说明磁盘 IO 瓶颈

# ===== 查找大文件/大目录 =====
find_large() {
    local dir="${1:-/}"
    echo "=== Top 10 大目录（${dir}）==="
    du -sh "$dir"* 2>/dev/null | sort -hr | head -10

    echo ""
    echo "=== Top 10 大文件（${dir}）==="
    find "$dir" -type f -size +100M -exec ls -lh {} \; 2>/dev/null \
        | awk '{print $5, $NF}' | sort -hr | head -10
}
# find_large /var

# ===== 磁盘增长趋势 =====
# 每小时记录一次，观察增长速率
cat > disk_trend.sh << 'SCRIPT'
#!/bin/bash
LOG="/var/log/disk_usage.log"

echo "$(date '+%Y-%m-%d %H:%M:%S') $(df -h / | awk 'NR==2{print $3, $5}')" >> "$LOG"

# 分析最近 24 小时增长率
if [[ $(wc -l < "$LOG") -ge 2 ]]; then
    tail -24 "$LOG" | awk '{
        gsub(/%/, "", $4)
        if (NR == 1) first = $4
        last = $4
    }
    END {
        growth = last - first
        printf "24小时增长: %.1f%% (从 %.1f%% 到 %.1f%%)\n", growth, first, last
        if (growth > 5) print "⚠️ 磁盘增长过快！"
    }'
fi
SCRIPT

# 7.2.4 网络监控——流量/连接/端口

#!/bin/bash

# ===== 网络接口流量 =====
# 读取 /proc/net/dev 统计收发流量
network_traffic() {
    local iface="${1:-eth0}"
    awk -v iface="$iface" '$1 ~ iface":" {
        printf "接收: %.2f MB  |  发送: %.2f MB\n", $2/1048576, $10/1048576
    }' /proc/net/dev
}
network_traffic eth0
# 注意：这是累计值，需两次采样相减算速率

# ===== 连接状态统计 =====
echo "=== TCP 连接状态 ==="
ss -tan | awk 'NR>1 {state[$1]++}
END {
    for (s in state) printf "%-12s %d\n", s, state[s]
}' | sort -k2 -rn

# 重点关注 TIME_WAIT（过多说明短连接多）和 CLOSE_WAIT（应用未正确关闭连接）
echo ""
ss -tan state time-wait | wc -l | xargs echo "TIME_WAIT:"
ss -tan state close-wait | wc -l | xargs echo "CLOSE_WAIT:"
ss -tan state established | wc -l | xargs echo "ESTABLISHED:"

# ===== 端口监听检查 =====
check_ports() {
    local required_ports=("80" "443" "22" "3306" "6379")
    echo "=== 端口检查 ==="
    for port in "${required_ports[@]}"; do
        if ss -tlnp | grep -q ":$port "; then
            echo "✅ 端口 $port 正常监听"
        else
            echo "❌ 端口 $port 未监听！"
        fi
    done
}
check_ports

# ===== 对外连接数 Top IP =====
ss -tnp | awk 'NR>1 {
    split($5, dst, ":")
    ip = dst[1]
    if (ip !~ /^127|^::1|^\*$/) count[ip]++
}
END {
    for (i in count) printf "%-20s %d\n", i, count[i]
}' | sort -k2 -rn | head -10

# ===== 网络连通性检测 =====
check_connectivity() {
    local targets=("8.8.8.8" "1.1.1.1" "baidu.com")
    for target in "${targets[@]}"; do
        if ping -c 1 -W 2 "$target" > /dev/null 2>&1; then
            echo "✅ $target 可达"
        else
            echo "❌ $target 不通"
        fi
    done
}
# check_connectivity

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62

# 7.2.5 进程存活检测与自动重启

#!/bin/bash

# ===== 单个进程检测 =====
check_and_restart() {
    local process_name="$1"
    local start_cmd="$2"

    if pgrep -x "$process_name" > /dev/null; then
        echo "✅ $process_name 运行中 (PID: $(pgrep -x "$process_name" | tr '\n' ' '))"
        return 0
    else
        echo "❌ $process_name 已停止，尝试重启..."
        eval "$start_cmd"
        sleep 3
        if pgrep -x "$process_name" > /dev/null; then
            echo "✅ $process_name 重启成功"
            return 0
        else
            echo "💀 $process_name 重启失败！发送告警..."
            return 1
        fi
    fi
}

# check_and_restart "nginx" "systemctl start nginx"
# check_and_restart "redis-server" "systemctl start redis"

# ===== 进程守护脚本（while+sleep 模式）=====
cat > process_guard.sh << 'SCRIPT'
#!/bin/bash
# 用法：./process_guard.sh nginx "systemctl start nginx" 10
#      每隔 10 秒检测 nginx，挂了就重启

PROCESS="$1"
START_CMD="$2"
INTERVAL="${3:-30}"

while true; do
    if ! pgrep -x "$PROCESS" > /dev/null; then
        echo "[$(date)] $PROCESS 挂了，正在重启..." >> /var/log/guard.log
        eval "$START_CMD"
        sleep 3
        if pgrep -x "$PROCESS" > /dev/null; then
            echo "[$(date)] $PROCESS 重启成功" >> /var/log/guard.log
        else
            echo "[$(date)] 💀 $PROCESS 重启失败！" >> /var/log/guard.log
        fi
    fi
    sleep "$INTERVAL"
done
SCRIPT

# ===== 批量检测关键服务 =====
cat > service_health_check.sh << 'SCRIPT'
#!/bin/bash
# 拷贝为 /etc/cron.d/health_check 实现每 5 分钟检测

declare -A SERVICES=(
    ["nginx"]="systemctl restart nginx"
    ["php-fpm"]="systemctl restart php-fpm"
    ["mysqld"]="systemctl restart mysqld"
    ["redis-server"]="systemctl restart redis"
)

for svc in "${!SERVICES[@]}"; do
    if systemctl is-active --quiet "$svc"; then
        :
    else
        echo "[$(date)] $svc 服务异常，尝试重启" >> /var/log/service_guard.log
        ${SERVICES[$svc]}
    fi
done
SCRIPT

# ===== 检测进程是否假死（端口监听但无响应）=====
check_port_response() {
    local port="$1"
    local timeout=3
    local url="${2:-http://localhost:$port/}"

    if timeout "$timeout" curl -sf "$url" > /dev/null 2>&1; then
        echo "✅ 端口 $port 响应正常"
        return 0
    else
        echo "❌ 端口 $port 无响应（进程可能假死）"
        return 1
    fi
}

# 7.2.6 定时采集与资源趋势分析

#!/bin/bash

# ===== 资源采集脚本（cron 每 1 分钟执行一次）=====
# crontab: * * * * * /path/to/collect_metrics.sh

cat > collect_metrics.sh << 'SCRIPT'
#!/bin/bash
DATA_DIR="/var/log/metrics"
mkdir -p "$DATA_DIR"
TS=$(date '+%Y-%m-%d %H:%M:%S')

# CPU 使用率
cpu=$(top -bn1 | grep "Cpu(s)" | awk '{print 100 - $8}')

# 内存使用率
mem_total=$(awk '/^MemTotal:/ {print $2}' /proc/meminfo)
mem_avail=$(awk '/^MemAvailable:/ {print $2}' /proc/meminfo)
mem_used=$(( (mem_total - mem_avail) * 100 / mem_total ))

# 磁盘使用率
disk=$(df / | awk 'NR==2 {gsub(/%/,"",$5); print $5}')

# 负载
load=$(awk '{print $1}' /proc/loadavg)

# 写入 CSV
echo "$TS,$cpu,${mem_used},$disk,$load" >> "$DATA_DIR/system.csv"

# 只保留最近 7 天的数据
find "$DATA_DIR" -name "*.csv" -mtime +7 -delete
SCRIPT

# ===== 趋势分析脚本 =====
cat > trend_analysis.sh << 'SCRIPT'
#!/bin/bash
DATA="$1"

echo "=== 过去 24 小时趋势 ==="
awk -F',' '{
    cpu += $2; mem += $3; disk += $4; load += $5; n++
}
END {
    printf "CPU 平均:   %.1f%%\n", cpu/n
    printf "内存平均:   %.1f%%\n", mem/n
    printf "磁盘平均:   %.1f%%\n", disk/n
    printf "负载平均:   %.2f\n", load/n
}' "$DATA"

echo ""
echo "=== 峰值 ==="
awk -F',' '{
    if ($2 > max_cpu)  { max_cpu = $2;  cpu_ts = $1 }
    if ($3 > max_mem)  { max_mem = $3;  mem_ts = $1 }
    if ($4 > max_disk) { max_disk = $4; disk_ts = $1 }
    if ($5 > max_load) { max_load = $5; load_ts = $1 }
}
END {
    printf "CPU 峰值:   %.1f%%  (%s)\n", max_cpu, cpu_ts
    printf "内存峰值:   %.1f%%  (%s)\n", max_mem, mem_ts
    printf "磁盘峰值:   %.1f%%  (%s)\n", max_disk, disk_ts
    printf "负载峰值:   %.2f  (%s)\n", max_load, load_ts
}' "$DATA"
SCRIPT

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63

# 7.3 综合告警系统实战

# 7.3.1 告警策略设计——多级阈值与冷却机制

一个成熟的告警系统需要三个核心机制：

#!/bin/bash

# ===== 告警级别定义 =====
# WARN   = 预警，关注但不紧急（如 CPU 使用 70%）
# ERROR  = 需要立即处理（如 CPU 使用 90%）
# FATAL  = 致命，可能影响业务（如磁盘满、服务宕机）

# ===== 冷却机制 =====
# 同一个告警在冷却期内不重复发送（避免告警风暴）
COOLDOWN_WARN=600       # 10 分钟
COOLDOWN_ERROR=300      # 5 分钟
COOLDOWN_FATAL=60       # 1 分钟（致命告警需要持续提醒）

ALERT_STATE_DIR="/tmp/alert_state"
mkdir -p "$ALERT_STATE_DIR"

# ===== 通用告警函数 =====
send_alert() {
    local level="$1"       # WARN / ERROR / FATAL
    local target="$2"      # 告警目标（CPU/MEM/DISK...）
    local value="$3"       # 当前值
    local threshold="$4"   # 阈值
    local message="$5"     # 详细消息
    local now=$(date +%s)
    local state_file="$ALERT_STATE_DIR/${target}_${level}"

    # 检查冷却
    local cooldown_var="COOLDOWN_${level}"
    local cooldown=${!cooldown_var}
    local last_alert=$(cat "$state_file" 2>/dev/null || echo 0)

    if (( now - last_alert < cooldown )); then
        return  # 冷却中，不发送
    fi

    # 更新状态文件
    echo "$now" > "$state_file"

    # 输出告警
    echo "[$(date)] ${level} [${target}] 当前:${value} 阈值:${threshold} - ${message}"
    # 这里接入通知通道（见下一节）
}

# ===== 综合检查 =====
comprehensive_check() {
    # CPU
    local cpu=$(top -bn1 | grep "Cpu(s)" | awk '{print 100 - $8}')
    cpu=${cpu%.*}
    if [[ "$cpu" -ge 95 ]]; then
        send_alert "FATAL" "CPU" "$cpu" "95" "CPU 使用率过高，服务可能卡死"
    elif [[ "$cpu" -ge 85 ]]; then
        send_alert "ERROR" "CPU" "$cpu" "85" "CPU 持续高负载"
    elif [[ "$cpu" -ge 70 ]]; then
        send_alert "WARN" "CPU" "$cpu" "70" "CPU 使用率偏高"
    fi

    # 内存
    local mem_total=$(awk '/^MemTotal:/ {print $2}' /proc/meminfo)
    local mem_avail=$(awk '/^MemAvailable:/ {print $2}' /proc/meminfo)
    local mem_percent=$(( (mem_total - mem_avail) * 100 / mem_total ))
    if [[ "$mem_percent" -ge 95 ]]; then
        send_alert "FATAL" "MEM" "${mem_percent}%" "95" "可用内存严重不足，OOM 风险"
    elif [[ "$mem_percent" -ge 85 ]]; then
        send_alert "ERROR" "MEM" "${mem_percent}%" "85" "内存使用率过高"
    fi

    # 磁盘
    local disk=$(df / | awk 'NR==2 {gsub(/%/,"",$5); print $5}')
    if [[ "$disk" -ge 95 ]]; then
        send_alert "FATAL" "DISK" "${disk}%" "95" "磁盘即将写满！"
    elif [[ "$disk" -ge 85 ]]; then
        send_alert "ERROR" "DISK" "${disk}%" "85" "磁盘空间不足"
    elif [[ "$disk" -ge 75 ]]; then
        send_alert "WARN" "DISK" "${disk}%" "75" "磁盘使用率偏高"
    fi

    # 负载
    local load=$(awk '{print $1}' /proc/loadavg)
    local cores=$(nproc)
    if (( $(echo "$load > $cores * 2" | bc -l) )); then
        send_alert "ERROR" "LOAD" "$load" "$((cores*2))" "系统负载过高"
    elif (( $(echo "$load > $cores" | bc -l) )); then
        send_alert "WARN" "LOAD" "$load" "$cores" "负载超过核心数"
    fi
}

# comprehensive_check

# 7.3.2 告警通知——邮件/钉钉/企业微信

#!/bin/bash

# ===== 1. 邮件告警（通过 mail/mailx 命令）=====
send_email_alert() {
    local subject="$1"
    local body="$2"
    local recipients="${3:-admin@example.com}"

    echo "$body" | mail -s "$subject" "$recipients"
    # 或使用 sendmail / msmtp 等 MTA
}

# sendmail 方式（更可靠，特别是服务器上）
send_email_sendmail() {
    local to="$1"
    local subject="$2"
    local body="$3"

    /usr/sbin/sendmail -t << EMAIL_EOF
To: $to
Subject: $subject
Content-Type: text/plain; charset=utf-8

$body
EMAIL_EOF
}

# ===== 2. 钉钉机器人告警 =====
DINGTALK_WEBHOOK="https://oapi.dingtalk.com/robot/send?access_token=YOUR_TOKEN"

send_dingtalk() {
    local title="$1"
    local content="$2"
    local level="${3:-WARN}"

    # 根据级别选择颜色
    case "$level" in
        FATAL) color="#FF0000" ;;
        ERROR) color="#FF6600" ;;
        WARN)  color="#FFAA00" ;;
        *)     color="#0088CC" ;;
    esac

    local json=$(cat << JSON
{
    "msgtype": "markdown",
    "markdown": {
        "title": "$title",
        "text": "## ${level} <font color=${color}>${title}</font>\n\n${content}\n\n> 发送时间: $(date '+%Y-%m-%d %H:%M:%S')"
    }
}
JSON
)
    curl -s -X POST -H "Content-Type: application/json" -d "$json" "$DINGTALK_WEBHOOK" > /dev/null
}

# ===== 3. 企业微信机器人告警 =====
WECOM_WEBHOOK="https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=YOUR_KEY"

send_wecom() {
    local content="$1"
    local level="${2:-WARN}"

    # 根据级别造前缀 emoji
    case "$level" in
        FATAL) prefix="💀【致命】" ;;
        ERROR) prefix="🚨【错误】" ;;
        WARN)  prefix="⚠️ 【预警】" ;;
        *)     prefix="ℹ️ 【信息】" ;;
    esac

    local json="{\"msgtype\":\"text\",\"text\":{\"content\":\"${prefix}\n${content}\n\n时间: $(date '+%Y-%m-%d %H:%M:%S')\"}}"
    curl -s -X POST -H "Content-Type: application/json" -d "$json" "$WECOM_WEBHOOK" > /dev/null
}

# ===== 通用通知路由器（自动选渠道）=====
notify() {
    local title="$1"
    local body="$2"
    local level="$3"

    # 致命告警：所有渠道都发
    # 错误告警：钉钉 + 邮件
    # 预警：仅钉钉

    case "$level" in
        FATAL)
            send_dingtalk "$title" "$body" "FATAL"
            send_wecom "$title" "$body" "FATAL"
            send_email_sendmail "admin@example.com" "[FATAL] $title" "$body"
            ;;
        ERROR)
            send_dingtalk "$title" "$body" "ERROR"
            send_email_sendmail "admin@example.com" "[ERROR] $title" "$body"
            ;;
        WARN|*)
            send_dingtalk "$title" "$body" "WARN"
            ;;
    esac
}

# 7.3.3 一体化监控告警脚本——从采集到通知

#!/bin/bash
# ============================================
# 一体化监控告警脚本
# 用法：crontab 中配置 */5 * * * * /path/to/monitor.sh
# 推荐频率：每 5 分钟一次
# ============================================

set -euo pipefail

# ---- 配置区（修改为你自己的值）----
CONFIG_FILE="${HOME}/.monitor.conf"
if [[ -f "$CONFIG_FILE" ]]; then
    source "$CONFIG_FILE"
fi

# 阈值设置
CPU_WARN=80
CPU_ERROR=90
MEM_WARN=85
MEM_ERROR=95
DISK_WARN=80
DISK_ERROR=90
LOAD_RATIO=1.5       # 负载超过核心数 * 此值的倍数

# 告警冷却（秒）
COOLDOWN=300          # 5 分钟

# Webhook
DING_WEBHOOK="${DING_WEBHOOK:-}"
WECOM_WEBHOOK="${WECOM_WEBHOOK:-}"
EMAIL_TO="${EMAIL_TO:-root@localhost}"

# 日志
LOG_FILE="/var/log/system_monitor.log"
ALERT_DIR="/tmp/monitor_alerts"
mkdir -p "$ALERT_DIR"

# ---- 工具函数 ----
log()          { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >> "$LOG_FILE"; echo "$*"; }
get_metric()   { cat "$ALERT_DIR/${1}.last" 2>/dev/null || echo ""; }
set_metric()   { echo "$2" > "$ALERT_DIR/${1}.last"; }

is_cooldown() {
    local key="$1"
    local ts_file="$ALERT_DIR/${key}.ts"
    local now=$(date +%s)
    local last=$(cat "$ts_file" 2>/dev/null || echo 0)
    if (( now - last < COOLDOWN )); then return 0; fi
    echo "$now" > "$ts_file"
    return 1
}

# ---- 通知通道 ----
ding_notify() {
    [[ -z "$DING_WEBHOOK" ]] && return
    curl -s -X POST -H "Content-Type: application/json" \
        -d "{\"msgtype\":\"text\",\"text\":{\"content\":\"[$(hostname)] $1\"}}" \
        "$DING_WEBHOOK" > /dev/null
}

wecom_notify() {
    [[ -z "$WECOM_WEBHOOK" ]] && return
    curl -s -X POST -H "Content-Type: application/json" \
        -d "{\"msgtype\":\"text\",\"text\":{\"content\":\"[$(hostname)] $1\"}}" \
        "$WECOM_WEBHOOK" > /dev/null
}

alert() {
    local level="$1"; shift
    local msg="$*"

    log "[${level}] $msg"

    # 冷却检查（FATAL 不停发）
    if [[ "$level" != "FATAL" ]]; then
        local key=$(echo "$msg" | md5sum | cut -c1-8)
        is_cooldown "$key" && return
    fi

    # 发送通知
    case "$level" in
        FATAL) ding_notify "💀 [致命] $msg"; wecom_notify "💀 [致命] $msg" ;;
        ERROR) ding_notify "🚨 [错误] $msg" ;;
        WARN)  ding_notify "⚠️  [预警] $msg" ;;
    esac
}

# ---- 采集函数 ----
collect_cpu() {
    local cpu=$(top -bn1 2>/dev/null | grep "Cpu(s)" | awk '{print 100 - $8}' 2>/dev/null || echo "0")
    cpu=${cpu%.*}
    local prev=$(get_metric "cpu")

    set_metric "cpu" "$cpu"

    if [[ "$cpu" -ge "$CPU_ERROR" ]]; then
        alert "ERROR" "CPU 使用率 ${cpu}%，阈值 ${CPU_ERROR}%"
    elif [[ "$cpu" -ge "$CPU_WARN" ]]; then
        alert "WARN" "CPU 使用率 ${cpu}%，阈值 ${CPU_WARN}%"
    fi
    log "  CPU: ${cpu}%"
}

collect_memory() {
    local total=$(awk '/^MemTotal:/ {print $2}' /proc/meminfo)
    local avail=$(awk '/^MemAvailable:/ {print $2}' /proc/meminfo)
    local percent=$(( (total - avail) * 100 / total ))
    set_metric "mem" "$percent"

    if [[ "$percent" -ge "$MEM_ERROR" ]]; then
        alert "ERROR" "内存使用 ${percent}%，阈值 ${MEM_ERROR}%"
    elif [[ "$percent" -ge "$MEM_WARN" ]]; then
        alert "WARN" "内存使用 ${percent}%，阈值 ${MEM_WARN}%"
    fi
    log "  MEM: ${percent}%"
}

collect_disk() {
    while IFS= read -r line; do
        local device=$(echo "$line" | awk '{print $1}')
        local percent=$(echo "$line" | awk '{gsub(/%/,"",$5); print $5}')
        local mount=$(echo "$line" | awk '{print $6}')

        if [[ "$percent" -ge "$DISK_ERROR" ]]; then
            alert "ERROR" "磁盘 ${mount} 使用 ${percent}%，阈值 ${DISK_ERROR}%"
        elif [[ "$percent" -ge "$DISK_WARN" ]]; then
            alert "WARN" "磁盘 ${mount} 使用 ${percent}%，阈值 ${DISK_WARN}%"
        fi
        log "  DISK ${mount}: ${percent}%"
    done < <(df -h | awk 'NR>1 && /^\/dev\//')
}

collect_load() {
    local load=$(awk '{print $1}' /proc/loadavg)
    local cores=$(nproc)
    set_metric "load" "$load"

    local threshold=$(echo "$cores * $LOAD_RATIO" | bc)
    if (( $(echo "$load > $threshold" | bc -l) )); then
        alert "WARN" "负载 ${load}，核心数 ${cores}，阈值 ${threshold}"
    fi
    log "  LOAD: ${load} (cores: ${cores})"
}

collect_services() {
    local services=("nginx" "php-fpm" "mysqld" "redis-server" "sshd" "docker")
    for svc in "${services[@]}"; do
        if systemctl is-active --quiet "$svc" 2>/dev/null; then
            :
        else
            if systemctl list-unit-files "$svc.service" &>/dev/null; then
                alert "ERROR" "服务 $svc 未运行！尝试重启..."
                systemctl restart "$svc" 2>/dev/null || \
                    alert "FATAL" "服务 $svc 重启失败！"
            fi
        fi
    done
}

# ---- 主流程 ----
main() {
    log "===== 开始巡检 ====="
    collect_load
    collect_cpu
    collect_memory
    collect_disk
    collect_services
    log "===== 巡检结束 ====="
}

main

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171

# 7.3.4 新手陷阱 Top 5

陷阱 1：top -bn1 第一次不准

# ❌ top 第一次采样可能不准确
top -bn1 | grep "Cpu(s)"

# ✅ 取第 2 次迭代或者用 /proc/stat 直接算
top -bn2 | grep "Cpu(s)" | tail -1
# 或用 mpstat：mpstat 1 1 | awk 'END{print 100-$NF}'

1
2
3
4
5
6

陷阱 2：内存监控用错指标

# ❌ 用 free 的 "available" 行看可用内存
# MemFree 不等于真实可用（Linux 会用空闲内存做缓存）

# ❌ 错误：认为 buff/cache 是"已用内存"
free -m | awk 'NR==2{printf "%d%%\n", $3*100/$2}'

# ✅ 用 MemAvailable（精确反映可用于新进程的内存）
awk '/^MemAvailable:/ {avail=$2} /^MemTotal:/ {total=$2}
     END {printf "%d%%\n", (total-avail)*100/total}' /proc/meminfo

1
2
3
4
5
6
7
8
9

陷阱 3：df 和 du 统计不一致

# df 显示磁盘已满，但 du 统计文件远远没到
# 原因：已删除但仍被进程持有的文件（deleted but open）

# 检查是否有大量被删除但仍占用的文件
lsof | grep deleted | wc -l
lsof | grep deleted | awk '{sum+=$7} END {printf "%.2f MB\n", sum/1048576}'
# 重启持有这些文件的进程即可释放空间

1
2
3
4
5
6
7

陷阱 4：crontab 环境变量不同

# ❌ crontab 执行脚本时 PATH 极短，很多命令找不到
# 脚本里用绝对路径或先 source profile

# ✅ 在脚本开头明确设置
#!/bin/bash
export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
source /etc/profile 2>/dev/null || true

1
2
3
4
5
6
7

陷阱 5：告警脚本本身占用资源

# ❌ 监控脚本内部有死循环、递归、未限制频率的日志写入
# 结果：监控脚本变成系统最大负担

# ✅ 设置资源限制
ulimit -t 30         # CPU 时间限制 30 秒
ulimit -v 262144     # 虚拟内存限制 256MB
# 或用 timeout 包裹：timeout 30 monitor.sh

1
2
3
4
5
6
7

# 7.3.5 综合思考题

日志关联分析：如何从 Nginx 访问日志 + 错误日志中，关联找出导致 502 错误的真正元凶（比如上游 PHP-FPM 进程在那一秒正好超时）？
OOM 预防：写一个脚本，当可用内存低于 10% 时，主动找出占用内存最多的进程，发送告警并记录 fast 快照（/proc//smaps 摘要），以便事后分析。
磁盘增长率预警：如何在磁盘使用率 70% 时，根据最近 7 天的增长率预测"磁盘将在多少天后满"，提前发出预警？
告警抑制链：设计一个逻辑——当"网络不通"告警触发时，自动抑制"服务不可达"类型的下游告警，避免告警风暴。
综合脚本：编写一个 health_check.sh，检测以下项并生成 JSON 格式的健康报告：
- 磁盘使用率 / Inode 使用率
- 内存使用率 / Swap 使用率
- CPU 使用率 / 负载
- 关键端口（80/443/3306/6379）监听状态
- 关键进程（nginx/mysql/redis/php-fpm）存活状态
- 最近的 5xx 错误数（过去 5 分钟）

#Shell #运维

上次更新: 2026/06/17, 12:47:39

← 文件查找与统计备份进程与磁盘→

日志监控与告警

# 第 7 章 日志监控与告警

# 目录介绍

# 7.1 日志分析

# 7.1.1 常见日志格式解析

# 7.1.2 访问量统计——PV/UV/QPS/带宽

# 7.1.3 Top IP 分析与异常流量检测

# 7.1.4 状态码分布与异常监控

# 7.1.5 响应时间分析——揪出慢请求

# 7.1.6 错误日志提取、分类与告警

# 7.2 系统监控

# 7.2.1 CPU 监控——使用率/负载/进程排行

# 7.2.2 内存监控——使用率/OOM 风险/Swap

# 7.2.3 磁盘监控——空间/inode/IO

# 7.2.4 网络监控——流量/连接/端口

# 7.2.5 进程存活检测与自动重启

# 7.2.6 定时采集与资源趋势分析

# 7.3 综合告警系统实战

# 7.3.1 告警策略设计——多级阈值与冷却机制

# 7.3.2 告警通知——邮件/钉钉/企业微信

# 7.3.3 一体化监控告警脚本——从采集到通知

# 7.3.4 新手陷阱 Top 5

# 7.3.5 综合思考题

# 第 7 章日志监控与告警