-
Notifications
You must be signed in to change notification settings - Fork 3
/
stackMonitor.sh
62 lines (50 loc) · 1.88 KB
/
stackMonitor.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
#!/bin/bash
ENGINE=hthor
ENGINE_BIN="/opt/HPCCSystems/bin/hthor"
LOG_FILE_NAME="$ENGINE-$( date +%Y-%m-%d-%H-%M-%S ).log"
#LOG_FILE_NAME=/dev/stdout
TOP_LOG=1
AFFINITY_CHECK=1
CPU_INFO=1
echo "$0 start."
echo "Log file: '${LOG_FILE_NAME}'"
echo "$0 start." > $LOG_FILE_NAME
if [[ CPU_INFO -eq 1 ]]
then
echo "CPU info:" >> ${LOG_FILE_NAME}
cat /proc/cpuinfo >> ${LOG_FILE_NAME}
echo "================================" >> ${LOG_FILE_NAME}
fi
while [ true ]
do
pid=$( pidof ${ENGINE} )
#echo "pid: '${pid}' $( date +%Y-%m-%d-%H-%M-%S ):"
if [[ -n "${pid}" ]]
then
echo "${ENGINE} pid: '${pid}' @$( date +%Y-%m-%d-%H-%M-%S ):" >> ${LOG_FILE_NAME}
echo "----------------------------------------" >> ${LOG_FILE_NAME}
if [[ AFFINITY_CHECK -eq 1 ]]
then
# run affinity check script only once
AFFINITY_CHECK=0
./tidAffinityCheck.sh "hthor" >> ${LOG_FILE_NAME}
echo "............................." >> ${LOG_FILE_NAME}
fi
if [[ TOP_LOG -eq 1 ]]
then
echo "$( top -H -b -n 1 | head -n 20 )" >> ${LOG_FILE_NAME}
echo "............................." >> ${LOG_FILE_NAME}
top -H -b -n 1 -p ${pid} >> ${LOG_FILE_NAME}
echo "............................." >> ${LOG_FILE_NAME}
fi
echo "Stack traces of ${ENGINE}" >> ${LOG_FILE_NAME}
sudo gdb --batch --quiet -ex "set interactive-mode off" -ex "thread apply all bt" -ex "quit" ${ENGINE_BIN} ${pid} >> ${LOG_FILE_NAME} 2>&1
else
echo "pid: 'none' $( date +%Y-%m-%d-%H-%M-%S ):" >> ${LOG_FILE_NAME}
fi
echo "================================" >> ${LOG_FILE_NAME}
echo "" >> ${LOG_FILE_NAME}
sleep 10
done
echo "$0 End." >> $LOG_FILE_NAME
echo "End."