Output of top at intervals:
Code: Select all
[coadminsj@cawlkl21 ~]$ SECONDS=0; while [ ${SECONDS} -lt '301' ]; do top -bcn1| head -n20; sleep 30; done
top - 15:36:53 up 35 days, 22:37, 3 users, load average: 0.42, 0.54, 0.72
Tasks: 174 total, 2 running, 172 sleeping, 0 stopped, 0 zombie
Cpu(s): 20.8%us, 4.6%sy, 0.0%ni, 73.3%id, 0.9%wa, 0.0%hi, 0.3%si, 0.0%st
Mem: 8061348k total, 4838840k used, 3222508k free, 47812k buffers
Swap: 2064380k total, 53876k used, 2010504k free, 3376836k cached
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
22510 apache 20 0 168m 66m 30m R 94.6 0.8 0:04.42 /usr/local/nagios/s
24355 nagios 20 0 55084 4472 1036 S 3.9 0.1 3:58.39 /usr/local/nagios/b
1 root 20 0 19360 1072 908 S 0.0 0.0 0:01.84 /sbin/init
2 root 20 0 0 0 0 S 0.0 0.0 0:00.00 [kthreadd]
3 root RT 0 0 0 0 S 0.0 0.0 2:46.43 [migration/0]
4 root 20 0 0 0 0 S 0.0 0.0 2:12.76 [ksoftirqd/0]
5 root RT 0 0 0 0 S 0.0 0.0 0:00.00 [stopper/0]
6 root RT 0 0 0 0 S 0.0 0.0 0:07.21 [watchdog/0]
7 root RT 0 0 0 0 S 0.0 0.0 3:19.06 [migration/1]
8 root RT 0 0 0 0 S 0.0 0.0 0:00.00 [stopper/1]
9 root 20 0 0 0 0 S 0.0 0.0 1:25.42 [ksoftirqd/1]
10 root RT 0 0 0 0 S 0.0 0.0 0:10.49 [watchdog/1]
11 root 20 0 0 0 0 S 0.0 0.0 2:45.48 [events/0]
top - 15:37:23 up 35 days, 22:38, 3 users, load average: 1.10, 0.69, 0.77
Tasks: 186 total, 2 running, 184 sleeping, 0 stopped, 0 zombie
Cpu(s): 20.8%us, 4.6%sy, 0.0%ni, 73.3%id, 0.9%wa, 0.0%hi, 0.3%si, 0.0%st
Mem: 8061348k total, 4998884k used, 3062464k free, 48328k buffers
Swap: 2064380k total, 53876k used, 2010504k free, 3527136k cached
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
24379 apache 20 0 154m 23m 1664 R 91.7 0.3 0:01.44 /usr/local/nagios/sbin/avail.cgi
4539 root 20 0 195m 4164 3492 S 1.9 0.1 2:38.18 winbindd
10819 mysql 20 0 1223m 92m 4136 S 1.9 1.2 3984:09 /usr/libexec/mysqld --basedir=/usr --datadir=/var/lib/mysql --user=mysql --log
24355 nagios 20 0 55084 4472 1036 S 1.9 0.1 4:01.42 /usr/local/nagios/bin/ndo2db -c /usr/local/nagios/etc/ndo2db.cfg
1 root 20 0 19360 1072 908 S 0.0 0.0 0:01.84 /sbin/init
2 root 20 0 0 0 0 S 0.0 0.0 0:00.00 [kthreadd]
3 root RT 0 0 0 0 S 0.0 0.0 2:46.43 [migration/0]
4 root 20 0 0 0 0 S 0.0 0.0 2:12.76 [ksoftirqd/0]
5 root RT 0 0 0 0 S 0.0 0.0 0:00.00 [stopper/0]
6 root RT 0 0 0 0 S 0.0 0.0 0:07.21 [watchdog/0]
7 root RT 0 0 0 0 S 0.0 0.0 3:19.06 [migration/1]
8 root RT 0 0 0 0 S 0.0 0.0 0:00.00 [stopper/1]
9 root 20 0 0 0 0 S 0.0 0.0 1:25.42 [ksoftirqd/1]
top - 15:37:54 up 35 days, 22:38, 3 users, load average: 1.18, 0.75, 0.79
Tasks: 178 total, 2 running, 176 sleeping, 0 stopped, 0 zombie
Cpu(s): 20.8%us, 4.6%sy, 0.0%ni, 73.3%id, 0.9%wa, 0.0%hi, 0.3%si, 0.0%st
Mem: 8061348k total, 5182604k used, 2878744k free, 48404k buffers
Swap: 2064380k total, 53876k used, 2010504k free, 3527504k cached
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
24379 apache 20 0 350m 234m 30m R 94.8 3.0 0:29.89 /usr/local/nagios/sbin/avail.cgi
10819 mysql 20 0 1223m 92m 4136 S 2.0 1.2 3984:11 /usr/libexec/mysqld --basedir=/usr --datadir=/var/lib/mysql --user=mysql --log-error=/var/log/mysqld.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.
24346 nagios 20 0 39520 14m 1540 S 2.0 0.2 0:34.26 /usr/local/nagios/bin/nagios -d /usr/local/nagios/etc/nagios.cfg
24355 nagios 20 0 55084 4472 1036 S 2.0 0.1 4:04.08 /usr/local/nagios/bin/ndo2db -c /usr/local/nagios/etc/ndo2db.cfg
1 root 20 0 19360 1072 908 S 0.0 0.0 0:01.84 /sbin/init
2 root 20 0 0 0 0 S 0.0 0.0 0:00.00 [kthreadd]
3 root RT 0 0 0 0 S 0.0 0.0 2:46.43 [migration/0]
4 root 20 0 0 0 0 S 0.0 0.0 2:12.76 [ksoftirqd/0]
5 root RT 0 0 0 0 S 0.0 0.0 0:00.00 [stopper/0]
6 root RT 0 0 0 0 S 0.0 0.0 0:07.21 [watchdog/0]
7 root RT 0 0 0 0 S 0.0 0.0 3:19.06 [migration/1]
8 root RT 0 0 0 0 S 0.0 0.0 0:00.00 [stopper/1]
9 root 20 0 0 0 0 S 0.0 0.0 1:25.42 [ksoftirqd/1]
top - 15:38:24 up 35 days, 22:39, 3 users, load average: 1.24, 0.81, 0.80
Tasks: 183 total, 2 running, 181 sleeping, 0 stopped, 0 zombie
Cpu(s): 20.8%us, 4.6%sy, 0.0%ni, 73.3%id, 0.9%wa, 0.0%hi, 0.3%si, 0.0%st
Mem: 8061348k total, 5354340k used, 2707008k free, 48472k buffers
Swap: 2064380k total, 53876k used, 2010504k free, 3527916k cached
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
24379 apache 20 0 512m 409m 48m R 79.1 5.2 0:57.87 /usr/local/nagios/sbin/avail.cgi
7855 apache 20 0 443m 28m 6020 S 35.6 0.4 0:17.98 /usr/sbin/httpd
3163 apache 20 0 454m 39m 6164 S 25.7 0.5 0:46.58 /usr/sbin/httpd
10819 mysql 20 0 1223m 93m 4136 S 19.8 1.2 3984:12 /usr/libexec/mysqld --basedir=/usr --datadir=/var/lib/mysql --user=mysql --log-error=/var/log/mysqld.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.
1 root 20 0 19360 1072 908 S 0.0 0.0 0:01.84 /sbin/init
2 root 20 0 0 0 0 S 0.0 0.0 0:00.00 [kthreadd]
3 root RT 0 0 0 0 S 0.0 0.0 2:46.43 [migration/0]
4 root 20 0 0 0 0 S 0.0 0.0 2:12.76 [ksoftirqd/0]
5 root RT 0 0 0 0 S 0.0 0.0 0:00.00 [stopper/0]
6 root RT 0 0 0 0 S 0.0 0.0 0:07.21 [watchdog/0]
7 root RT 0 0 0 0 S 0.0 0.0 3:19.06 [migration/1]
8 root RT 0 0 0 0 S 0.0 0.0 0:00.00 [stopper/1]
9 root 20 0 0 0 0 S 0.0 0.0 1:25.42 [ksoftirqd/1]
top - 15:38:55 up 35 days, 22:39, 3 users, load average: 1.35, 0.87, 0.82
Tasks: 179 total, 2 running, 177 sleeping, 0 stopped, 0 zombie
Cpu(s): 20.8%us, 4.6%sy, 0.0%ni, 73.3%id, 0.9%wa, 0.0%hi, 0.3%si, 0.0%st
Mem: 8061348k total, 5511228k used, 2550120k free, 48548k buffers
Swap: 2064380k total, 53876k used, 2010504k free, 3528204k cached
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
24379 apache 20 0 652m 536m 24m R 98.9 6.8 1:26.43 /usr/local/nagios/sbin/avail.cgi
24346 nagios 20 0 39520 14m 1540 S 11.9 0.2 0:35.17 /usr/local/nagios/bin/nagios -d /usr/local/nagios/etc/nagios.cfg
24355 nagios 20 0 55084 4472 1036 S 2.0 0.1 4:09.74 /usr/local/nagios/bin/ndo2db -c /usr/local/nagios/etc/ndo2db.cfg
1 root 20 0 19360 1072 908 S 0.0 0.0 0:01.84 /sbin/init
2 root 20 0 0 0 0 S 0.0 0.0 0:00.00 [kthreadd]
3 root RT 0 0 0 0 S 0.0 0.0 2:46.43 [migration/0]
4 root 20 0 0 0 0 S 0.0 0.0 2:12.76 [ksoftirqd/0]
5 root RT 0 0 0 0 S 0.0 0.0 0:00.00 [stopper/0]
6 root RT 0 0 0 0 S 0.0 0.0 0:07.21 [watchdog/0]
7 root RT 0 0 0 0 S 0.0 0.0 3:19.06 [migration/1]
8 root RT 0 0 0 0 S 0.0 0.0 0:00.00 [stopper/1]
9 root 20 0 0 0 0 S 0.0 0.0 1:25.42 [ksoftirqd/1]
10 root RT 0 0 0 0 S 0.0 0.0 0:10.49 [watchdog/1]
top - 15:39:25 up 35 days, 22:40, 3 users, load average: 1.43, 0.95, 0.85
Tasks: 185 total, 2 running, 183 sleeping, 0 stopped, 0 zombie
Cpu(s): 20.8%us, 4.6%sy, 0.0%ni, 73.3%id, 0.9%wa, 0.0%hi, 0.3%si, 0.0%st
Mem: 8061348k total, 5020844k used, 3040504k free, 48612k buffers
Swap: 2064380k total, 53876k used, 2010504k free, 3528664k cached
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
2964 apache 20 0 509m 93m 6432 R 25.6 1.2 5:15.56 /usr/sbin/httpd
10819 mysql 20 0 1223m 94m 4136 S 2.0 1.2 3984:16 /usr/libexec/mysqld --basedir=/usr --datadir=/var/lib/mysql --user=mysql --log-error=/var/log/mysqld.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.
24353 nagios 20 0 51936 1236 912 S 2.0 0.0 0:04.45 /usr/local/nagios/bin/ndo2db -c /usr/local/nagios/etc/ndo2db.cfg
24355 nagios 20 0 55084 4472 1036 S 2.0 0.1 4:12.82 /usr/local/nagios/bin/ndo2db -c /usr/local/nagios/etc/ndo2db.cfg
29597 coadmins 20 0 19280 1356 960 R 2.0 0.0 0:00.01 top -bcn1
1 root 20 0 19360 1072 908 S 0.0 0.0 0:01.84 /sbin/init
2 root 20 0 0 0 0 S 0.0 0.0 0:00.00 [kthreadd]
3 root RT 0 0 0 0 S 0.0 0.0 2:46.43 [migration/0]
4 root 20 0 0 0 0 S 0.0 0.0 2:12.76 [ksoftirqd/0]
5 root RT 0 0 0 0 S 0.0 0.0 0:00.00 [stopper/0]
6 root RT 0 0 0 0 S 0.0 0.0 0:07.21 [watchdog/0]
7 root RT 0 0 0 0 S 0.0 0.0 3:19.06 [migration/1]
8 root RT 0 0 0 0 S 0.0 0.0 0:00.00 [stopper/1]
top - 15:39:56 up 35 days, 22:40, 3 users, load average: 1.49, 1.03, 0.88
Tasks: 175 total, 1 running, 174 sleeping, 0 stopped, 0 zombie
Cpu(s): 20.8%us, 4.6%sy, 0.0%ni, 73.3%id, 0.9%wa, 0.0%hi, 0.3%si, 0.0%st
Mem: 8061348k total, 4979388k used, 3081960k free, 48676k buffers
Swap: 2064380k total, 53876k used, 2010504k free, 3529032k cached
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
24355 nagios 20 0 55084 4472 1036 S 4.0 0.1 4:16.01 /usr/local/nagios/bin/ndo2db -c /usr/local/nagios/etc/ndo2db.cfg
10819 mysql 20 0 1223m 94m 4136 S 2.0 1.2 3984:19 /usr/libexec/mysqld --basedir=/usr --datadir=/var/lib/mysql --user=mysql --log-error=/var/log/mysqld.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.
24346 nagios 20 0 39520 14m 1540 S 2.0 0.2 0:36.06 /usr/local/nagios/bin/nagios -d /usr/local/nagios/etc/nagios.cfg
24348 nagios 20 0 10108 1016 684 S 2.0 0.0 0:03.21 /usr/local/nagios/bin/nagios --worker /usr/local/nagios/var/rw/nagios.qh
24350 nagios 20 0 10108 1016 684 S 2.0 0.0 0:03.20 /usr/local/nagios/bin/nagios --worker /usr/local/nagios/var/rw/nagios.qh
1 root 20 0 19360 1072 908 S 0.0 0.0 0:01.84 /sbin/init
2 root 20 0 0 0 0 S 0.0 0.0 0:00.00 [kthreadd]
3 root RT 0 0 0 0 S 0.0 0.0 2:46.43 [migration/0]
4 root 20 0 0 0 0 S 0.0 0.0 2:12.77 [ksoftirqd/0]
5 root RT 0 0 0 0 S 0.0 0.0 0:00.00 [stopper/0]
6 root RT 0 0 0 0 S 0.0 0.0 0:07.21 [watchdog/0]
7 root RT 0 0 0 0 S 0.0 0.0 3:19.06 [migration/1]
8 root RT 0 0 0 0 S 0.0 0.0 0:00.00 [stopper/1]
Code: Select all
mysqld 20608 kB
python 5764 kB
puppet 5728 kB
vmtoolsd 5636 kB
httpd 5280 kB
httpd 4800 kB
httpd 4572 kB
httpd 4532 kB
httpd 4488 kB
httpd 4368 kB
httpd 4272 kB
httpd 4244 kB
httpd 4224 kB
httpd 4220 kB
httpd 4220 kB
httpd 4220 kB
httpd 4220 kB
httpd 4220 kB
httpd 4220 kB
httpd 4184 kB
httpd 4180 kB
httpd 4132 kB
httpd 4132 kB
httpd 4124 kB
qmgr 868 kB
udevd 840 kB
udevd 836 kB
udevd 836 kB
winbindd 824 kB
master 808 kB
winbindd 764 kB
winbindd 764 kB
winbindd 752 kB
winbindd 620 kB
sshd 588 kB
ndo2db 512 kB
ntpd 492 kB
crond 480 kB
ndo2db 400 kB
ndo2db 396 kB
crond 392 kB
crond 392 kB
crond 392 kB
crond 392 kB
crond 392 kB
crond 392 kB
rsyslogd 332 kB
mysqld_safe 256 kB
xinetd 232 kB
oddjobd 216 kB
dbus-daemon 176 kB
init 132 kB
auditd 124 kB
tail 92 kB
tail 92 kB
mingetty 80 kB
mingetty 80 kB
mingetty 76 kB
mingetty 76 kB
mingetty 76 kB
mingetty 76 kB
npcd 28 kB
watchdog/1
watchdog/0
vmmemctl
usbhid_resumer
ttm_swap
sync_supers
su 0 kB
stopper/1
stopper/0
sshd 0 kB
sshd 0 kB
sshd 0 kB
sshd 0 kB
sshd 0 kB
sort 0 kB
sleep 0 kB
sh 0 kB
sh 0 kB
sh 0 kB
sh 0 kB
sh 0 kB
sh 0 kB
scsi_eh_2
scsi_eh_1
scsi_eh_0
pm
pickup 0 kB
php 0 kB
php 0 kB
php 0 kB
php 0 kB
php 0 kB
php 0 kB
pciehpd
netns
nagios 0 kB
nagios 0 kB
nagios 0 kB
nagios 0 kB
nagios 0 kB
nagios 0 kB
mpt_poll_0
mpt/0
migration/1
migration/0
md_misc/1
md_misc/0
md/1
md/0
local 0 kB
linkwatch
kthrotld/1
kthrotld/0
kthreadd
kswapd0
ksuspend_usbd
kstriped
ksoftirqd/1
ksoftirqd/0
ksmd
kseriod
kpsmoused
kintegrityd/1
kintegrityd/0
khungtaskd
khugepaged
khubd
khelper
kdmremove
kdmflush
kdmflush
kblockd/1
kblockd/0
kauditd
kacpi_notify
kacpi_hotplug
kacpid
jbd2/sda1-8
jbd2/dm-0-8
flush-253:0
ext4-dio-unwrit
ext4-dio-unwrit
events_power_ef
events_power_ef
events_long/1
events_long/0
events/1
events/1
events/0
events/0
deferwq
crypto/1
crypto/0
cleanup 0 kB
check_nrpe 0 kB
check_nrpe 0 kB
check_icmp 0 kB
check_icmp 0 kB
cgroup
bounce 0 kB
bounce 0 kB
bdi-default
bash 0 kB
bash 0 kB
bash 0 kB
bash 0 kB
bash 0 kB
awk 0 kB
avail.cgi 0 kB
ata_sff/1
ata_sff/0
ata_aux
async/mgr
aio/1
aio/0
Code: Select all
Filesystem Size Used Avail Use% Mounted on
/dev/mapper/VolGroup-lv_root
37G 27G 8.1G 77% /
tmpfs 3.9G 0 3.9G 0% /dev/shm
/dev/sda1 477M 69M 384M 16% /boot
Also, I traced down and fixed the apache error after I pulled the system logs, and the fix had no effect on the speed of the availability report.