系统资源监控：htop/iotop/nmon/du

1
#!/bin/bash
2
# pipeline_monitor.sh —— 跑流程的同时记录资源
3
set -euo pipefail
4

5
LOG_DIR="./monitor_logs"
6
mkdir -p "${LOG_DIR}"
7
PREFIX="${LOG_DIR}/$(date +%Y%m%d_%H%M%S)"
8

9
# 启动后台监控
10
echo "Starting monitoring..."
11

12
# 1. CPU + 内存（每30秒）
13
(
14
    echo "timestamp,cpu_user,cpu_sys,cpu_idle,cpu_wait,mem_total_kb,mem_avail_kb,swap_used_kb"
15
    while true; do
16
        ts=$(date +%s)
17
        cpu_line=$(top -bn1 | grep "Cpu(s)" | sed 's/[,%]/ /g')
18
        cpu_user=$(echo "$cpu_line" | awk '{print $2}')
19
        cpu_sys=$(echo "$cpu_line" | awk '{print $4}')
20
        cpu_idle=$(echo "$cpu_line" | awk '{print $8}')
21
        cpu_wait=$(echo "$cpu_line" | awk '{print $10}')
22

23
        mem_info=$(free -k | grep Mem:)
24
        mem_total=$(echo "$mem_info" | awk '{print $2}')
25
        mem_avail=$(echo "$mem_info" | awk '{print $7}')
26
        swap_used=$(free -k | grep Swap: | awk '{print $3}')
27

28
        echo "${ts},${cpu_user},${cpu_sys},${cpu_idle},${cpu_wait},${mem_total},${mem_avail},${swap_used}"
29
        sleep 30
30
    done
31
) > "${PREFIX}_cpu_mem.csv" &
32
MONITOR_PID=$!
33

34
# 2. 磁盘使用（每5分钟）
35
(
36
    echo "timestamp,path,used_percent,avail_gb"
37
    while true; do
38
        ts=$(date +%s)
39
        df -h /data /tmp / | tail -n +2 | while read -r fs size used avail pct mnt; do
40
            echo "${ts},${mnt},${pct%\%},${avail}"
41
        done
42
        sleep 300
43
    done
44
) > "${PREFIX}_disk.csv" &
45
DISK_PID=$!
46

47
# 3. 进程快照（每10分钟）
48
(
49
    while true; do
50
        echo "=== $(date) ==="
51
        ps aux --sort=-%mem | head -16
52
        echo ""
53
        sleep 600
54
    done
55
) > "${PREFIX}_processes.log" &
56
PROC_PID=$!
57

58
# ========== 主流程（替换成你的实际流程） ==========
59
echo "Starting main pipeline at $(date)" | tee "${PREFIX}_events.log"
60

61
# 这里放你的实际生信流程
62
# fastp -i ... -I ... -o ... -O ...
63
# bwa mem ... | samtools sort ...
64
# bcftools mpileup ...
65

66
echo "Pipeline finished at $(date)" | tee -a "${PREFIX}_events.log"
67

68
# ========== 停止监控 ==========
69
kill ${MONITOR_PID} ${DISK_PID} ${PROC_PID} 2>/dev/null || true
70
wait ${MONITOR_PID} ${DISK_PID} ${PROC_PID} 2>/dev/null || true
71

72
echo "Monitoring logs saved to: ${LOG_DIR}/"
73
echo "  CPU/Memory: ${PREFIX}_cpu_mem.csv"
74
echo "  Disk:       ${PREFIX}_disk.csv"
75
echo "  Processes:  ${PREFIX}_processes.log"
76
echo "  Events:     ${PREFIX}_events.log"
77

78
# 快速汇总
79
echo ""
80
echo "=== Resource Summary ==="
81
echo "Peak memory usage:"
82
grep -v timestamp "${PREFIX}_cpu_mem.csv" | \
83
    awk -F, '{avail=$7/1048576; if(max_used_mb=="") max_used_mb=($6-$7); if(($6-$7)>max_used_mb) max_used_mb=($6-$7)} END {printf "  Max used: %.1f GB\n", max_used_mb/1048576}'
84
echo "Min available memory:"
85
grep -v timestamp "${PREFIX}_cpu_mem.csv" | \
86
    awk -F, '{if(min_avail=="" || $7<min_avail) min_avail=$7} END {printf "  Min avail: %.1f GB\n", min_avail/1048576}'