cmerchant wrote:Noticed the redness on your monitoring engine status... I think in order to have the apply configuration work, we need to have nagios, ndo2db restarted.
But I would check the health of your database services as well.
Code: Select all
service nagios stop
service nagios status
service ndo2db stop
service ndo2db status
service mysqld stop
service mysqld status
service mysqld start
tail -100 /var/log/mysqld.log
service ndo2db start
tail -100 /var/log/messages
service nagios start
tail -100 /usr/local/nagios/var/nagios.log
after all of that, could you verify the status of the monitoring engine status.
Code: Select all
cat /usr/local/nagiosxi/var/sysstat.log
Did all that, and still Red. Applying config just clocks endlessly now.
This is the result of cat /usr/local/nagiosxi/var/sysstat.log:
Code: Select all
[min_execution_time] => 0.00127
[max_execution_time] => 60.017
[avg_execution_time] => 8.1480894702602
)
)
PHP Warning: exec(): Unable to fork [/usr/bin/uptime | sed s/,//g | awk -F'average: ' '{ print $2 }'] in /usr/local/nagiosxi/cron/sysstat.php on line 198
PHP Notice: Undefined offset: 0 in /usr/local/nagiosxi/cron/sysstat.php on line 201
PHP Notice: Undefined offset: 1 in /usr/local/nagiosxi/cron/sysstat.php on line 205
PHP Notice: Undefined offset: 2 in /usr/local/nagiosxi/cron/sysstat.php on line 206
LOAD:
Array
(
[load1] =>
[load5] =>
[load15] =>
)
PHP Warning: exec(): Unable to fork [/usr/bin/free -m | head --lines=2 | tail --lines=1 | awk '{ print $2,$3,$4,$5,$6,$7}'] in /usr/local/nagiosxi/cron/sysstat.php on line 220
PHP Notice: Undefined offset: 0 in /usr/local/nagiosxi/cron/sysstat.php on line 223
PHP Notice: Undefined offset: 1 in /usr/local/nagiosxi/cron/sysstat.php on line 227
PHP Notice: Undefined offset: 2 in /usr/local/nagiosxi/cron/sysstat.php on line 228
PHP Notice: Undefined offset: 3 in /usr/local/nagiosxi/cron/sysstat.php on line 229
PHP Notice: Undefined offset: 4 in /usr/local/nagiosxi/cron/sysstat.php on line 230
PHP Notice: Undefined offset: 5 in /usr/local/nagiosxi/cron/sysstat.php on line 231
MEMORY:
Array
(
[total] =>
[used] =>
[free] =>
[shared] =>
[buffers] =>
[cached] =>
)
PHP Warning: exec(): Unable to fork [/usr/bin/free -m | tail --lines=1 | awk '{ print $2,$3,$4}'] in /usr/local/nagiosxi/cron/sysstat.php on line 244
PHP Notice: Undefined offset: 0 in /usr/local/nagiosxi/cron/sysstat.php on line 247
PHP Notice: Undefined offset: 1 in /usr/local/nagiosxi/cron/sysstat.php on line 251
PHP Notice: Undefined offset: 2 in /usr/local/nagiosxi/cron/sysstat.php on line 252
SWAP:
Array
(
[total] =>
[used] =>
[free] =>
)
PHP Warning: exec(): Unable to fork [/usr/bin/iostat -c 5 2 | tail --lines=2 | head --lines=1 | awk '{ print $1,$2,$3,$4,$5,$6 }'] in /usr/local/nagiosxi/cron/sysstat.php on line 265
PHP Notice: Undefined offset: 0 in /usr/local/nagiosxi/cron/sysstat.php on line 268
PHP Notice: Undefined offset: 1 in /usr/local/nagiosxi/cron/sysstat.php on line 272
PHP Notice: Undefined offset: 2 in /usr/local/nagiosxi/cron/sysstat.php on line 273
PHP Notice: Undefined offset: 3 in /usr/local/nagiosxi/cron/sysstat.php on line 274
PHP Notice: Undefined offset: 4 in /usr/local/nagiosxi/cron/sysstat.php on line 275
PHP Notice: Undefined offset: 5 in /usr/local/nagiosxi/cron/sysstat.php on line 276
IOSTAT:
Array
(
[user] =>
[nice] =>
[system] =>
[iowait] =>
[steal] =>
[idle] =>
)
.DB BACKEND:
Array
(
[last_checkin] => 2015-04-08 14:40:59
[bytes_processed] => 18156270
[entries_processed] => 38119
[connect_time] => 2015-04-08 14:36:55
[disconnect_time] => 0000-00-00 00:00:00
)
CMDLINE=/etc/init.d/nagios status
PHP Warning: system(): Unable to fork [/etc/init.d/nagios status] in /usr/local/nagiosxi/cron/sysstat.php on line 159
OUTPUT=
RETURNCODE=-1
CMDLINE=/etc/init.d/npcd status
PHP Warning: system(): Unable to fork [/etc/init.d/npcd status] in /usr/local/nagiosxi/cron/sysstat.php on line 159
OUTPUT=
RETURNCODE=-1
CMDLINE=/etc/init.d/ndo2db status
PHP Warning: system(): Unable to fork [/etc/init.d/ndo2db status] in /usr/local/nagiosxi/cron/sysstat.php on line 159
OUTPUT=
RETURNCODE=-1
DAEMONS:
Array
(
[nagioscore] => Array
(
[daemon] => nagios
[output] =>
[return_code] => -1
[status] => 1
)
[pnp] => Array
(
[daemon] => npcd
[output] =>
[return_code] => -1
[status] => 1
)
[ndoutils] => Array
(
[daemon] => ndo2db
[output] =>
[return_code] => -1
[status] => 1
)
)
CORE STATS:
Array
(
[activehostchecks] => Array
(
[1min] => 235
[5min] => 1144
[15min] => 1144
)
[passivehostchecks] => Array
(
[1min] => 0
[5min] => 0
[15min] => 0
)
[activeservicechecks] => Array
(
[1min] => 341
[5min] => 1440
[15min] => 1440
)
[passiveservicechecks] => Array
(
[1min] => 0
[5min] => 0
[15min] => 0
)
[activehostcheckperf] => Array
(
[min_latency] => 0
[max_latency] => 1.72661
[avg_latency] => 0.03091142578125
[min_execution_time] => 0.00167
[max_execution_time] => 10.00291
[avg_execution_time] => 2.8069380517578
)
[activeservicecheckperf] => Array
(
[min_latency] => 0
[max_latency] => 7.754
[avg_latency] => 0.0664435724907063
[min_execution_time] => 0.00127
[max_execution_time] => 60.017
[avg_execution_time] => 8.1498299758364
)
)
PHP Warning: exec(): Unable to fork [/usr/bin/uptime | sed s/,//g | awk -F'average: ' '{ print $2 }'] in /usr/local/nagiosxi/cron/sysstat.php on line 198
PHP Notice: Undefined offset: 0 in /usr/local/nagiosxi/cron/sysstat.php on line 201
PHP Notice: Undefined offset: 1 in /usr/local/nagiosxi/cron/sysstat.php on line 205
PHP Notice: Undefined offset: 2 in /usr/local/nagiosxi/cron/sysstat.php on line 206
LOAD:
Array
(
[load1] =>
[load5] =>
[load15] =>
)
PHP Warning: exec(): Unable to fork [/usr/bin/free -m | head --lines=2 | tail --lines=1 | awk '{ print $2,$3,$4,$5,$6,$7}'] in /usr/local/nagiosxi/cron/sysstat.php on line 220
PHP Notice: Undefined offset: 0 in /usr/local/nagiosxi/cron/sysstat.php on line 223
PHP Notice: Undefined offset: 1 in /usr/local/nagiosxi/cron/sysstat.php on line 227
PHP Notice: Undefined offset: 2 in /usr/local/nagiosxi/cron/sysstat.php on line 228
PHP Notice: Undefined offset: 3 in /usr/local/nagiosxi/cron/sysstat.php on line 229
PHP Notice: Undefined offset: 4 in /usr/local/nagiosxi/cron/sysstat.php on line 230
PHP Notice: Undefined offset: 5 in /usr/local/nagiosxi/cron/sysstat.php on line 231
MEMORY:
Array
(
[total] =>
[used] =>
[free] =>
[shared] =>
[buffers] =>
[cached] =>
)
PHP Warning: exec(): Unable to fork [/usr/bin/free -m | tail --lines=1 | awk '{ print $2,$3,$4}'] in /usr/local/nagiosxi/cron/sysstat.php on line 244
PHP Notice: Undefined offset: 0 in /usr/local/nagiosxi/cron/sysstat.php on line 247
PHP Notice: Undefined offset: 1 in /usr/local/nagiosxi/cron/sysstat.php on line 251
PHP Notice: Undefined offset: 2 in /usr/local/nagiosxi/cron/sysstat.php on line 252
SWAP:
Array
(
[total] =>
[used] =>
[free] =>
)
PHP Warning: exec(): Unable to fork [/usr/bin/iostat -c 5 2 | tail --lines=2 | head --lines=1 | awk '{ print $1,$2,$3,$4,$5,$6 }'] in /usr/local/nagiosxi/cron/sysstat.php on line 265
PHP Notice: Undefined offset: 0 in /usr/local/nagiosxi/cron/sysstat.php on line 268
PHP Notice: Undefined offset: 1 in /usr/local/nagiosxi/cron/sysstat.php on line 272
PHP Notice: Undefined offset: 2 in /usr/local/nagiosxi/cron/sysstat.php on line 273
PHP Notice: Undefined offset: 3 in /usr/local/nagiosxi/cron/sysstat.php on line 274
PHP Notice: Undefined offset: 4 in /usr/local/nagiosxi/cron/sysstat.php on line 275
PHP Notice: Undefined offset: 5 in /usr/local/nagiosxi/cron/sysstat.php on line 276
IOSTAT:
Array
(
[user] =>
[nice] =>
[system] =>
[iowait] =>
[steal] =>
[idle] =>
)
.Done
[root@esu1l268 ~]#
tailing the ndo2db:
Code: Select all
141211 9:10:24 InnoDB: Completed initialization of buffer pool
141211 9:10:24 InnoDB: Started; log sequence number 0 44263
141211 9:10:24 [Note] Event Scheduler: Loaded 0 events
141211 9:10:24 [Note] /usr/libexec/mysqld: ready for connections.
Version: '5.1.67' socket: '/var/lib/mysql/mysql.sock' port: 3306 Source distribution
141212 15:33:10 [Note] /usr/libexec/mysqld: Normal shutdown
141212 15:33:10 [Note] Event Scheduler: Purging the queue. 0 events
141212 15:33:12 InnoDB: Starting shutdown...
141212 15:33:16 InnoDB: Shutdown completed; log sequence number 0 44273
141212 15:33:16 [Note] /usr/libexec/mysqld: Shutdown complete
141212 15:33:16 mysqld_safe mysqld from pid file /var/run/mysqld/mysqld.pid ended
141212 15:33:17 mysqld_safe Starting mysqld daemon with databases from /var/lib/mysql
141212 15:33:17 InnoDB: Initializing buffer pool, size = 8.0M
141212 15:33:17 InnoDB: Completed initialization of buffer pool
141212 15:33:17 InnoDB: Started; log sequence number 0 44273
141212 15:33:17 [Note] Event Scheduler: Loaded 0 events
141212 15:33:17 [Note] /usr/libexec/mysqld: ready for connections.
Version: '5.1.67' socket: '/var/lib/mysql/mysql.sock' port: 3306 Source distribution
141212 15:48:29 [Note] /usr/libexec/mysqld: Normal shutdown
141212 15:48:29 [Note] Event Scheduler: Purging the queue. 0 events
141212 15:48:31 InnoDB: Starting shutdown...
141212 15:48:33 InnoDB: Shutdown completed; log sequence number 0 44273
141212 15:48:33 [Note] /usr/libexec/mysqld: Shutdown complete
141212 15:48:33 mysqld_safe mysqld from pid file /var/run/mysqld/mysqld.pid ended
141212 15:54:33 mysqld_safe Starting mysqld daemon with databases from /var/lib/mysql
141212 15:54:33 InnoDB: Initializing buffer pool, size = 8.0M
141212 15:54:33 InnoDB: Completed initialization of buffer pool
141212 15:54:33 InnoDB: Started; log sequence number 0 44273
141212 15:54:33 [Note] Event Scheduler: Loaded 0 events
141212 15:54:33 [Note] /usr/libexec/mysqld: ready for connections.
Version: '5.1.67' socket: '/var/lib/mysql/mysql.sock' port: 3306 Source distribution
150119 8:06:38 [Note] /usr/libexec/mysqld: Normal shutdown
150119 8:06:38 [Note] Event Scheduler: Purging the queue. 0 events
150119 8:06:40 InnoDB: Starting shutdown...
150119 8:06:44 InnoDB: Shutdown completed; log sequence number 0 44283
150119 8:06:44 [Note] /usr/libexec/mysqld: Shutdown complete
150119 08:06:44 mysqld_safe mysqld from pid file /var/run/mysqld/mysqld.pid ended
150119 08:06:44 mysqld_safe Starting mysqld daemon with databases from /var/lib/mysql
150119 8:06:44 InnoDB: Initializing buffer pool, size = 8.0M
150119 8:06:44 InnoDB: Completed initialization of buffer pool
150119 8:06:44 InnoDB: Started; log sequence number 0 44283
150119 8:06:44 [Note] Event Scheduler: Loaded 0 events
150119 8:06:44 [Note] /usr/libexec/mysqld: ready for connections.
Version: '5.1.67' socket: '/var/lib/mysql/mysql.sock' port: 3306 Source distribution
150320 14:23:45 [Note] /usr/libexec/mysqld: Normal shutdown
150320 14:23:45 [Note] Event Scheduler: Purging the queue. 0 events
150320 14:23:47 InnoDB: Starting shutdown...
150320 14:23:51 InnoDB: Shutdown completed; log sequence number 0 44293
150320 14:23:51 [Note] /usr/libexec/mysqld: Shutdown complete
150320 14:23:51 mysqld_safe mysqld from pid file /var/run/mysqld/mysqld.pid ended
150320 14:23:52 mysqld_safe Starting mysqld daemon with databases from /var/lib/mysql
150320 14:23:52 InnoDB: Initializing buffer pool, size = 8.0M
150320 14:23:52 InnoDB: Completed initialization of buffer pool
150320 14:23:52 InnoDB: Started; log sequence number 0 44293
150320 14:23:52 [Note] Event Scheduler: Loaded 0 events
150320 14:23:52 [Note] /usr/libexec/mysqld: ready for connections.
Version: '5.1.67' socket: '/var/lib/mysql/mysql.sock' port: 3306 Source distribution
150408 14:36:44 [Note] /usr/libexec/mysqld: Normal shutdown
150408 14:36:44 [Note] Event Scheduler: Purging the queue. 0 events
150408 14:36:46 InnoDB: Starting shutdown...
150408 14:36:49 InnoDB: Shutdown completed; log sequence number 0 44303
150408 14:36:49 [Note] /usr/libexec/mysqld: Shutdown complete
150408 14:36:49 mysqld_safe mysqld from pid file /var/run/mysqld/mysqld.pid ended
150408 14:36:49 mysqld_safe Starting mysqld daemon with databases from /var/lib/mysql
150408 14:36:49 InnoDB: Initializing buffer pool, size = 8.0M
150408 14:36:49 InnoDB: Completed initialization of buffer pool
150408 14:36:49 InnoDB: Started; log sequence number 0 44303
150408 14:36:49 [Note] Event Scheduler: Loaded 0 events
150408 14:36:49 [Note] /usr/libexec/mysqld: ready for connections.
Version: '5.1.67' socket: '/var/lib/mysql/mysql.sock' port: 3306 Source distribution
150409 18:46:43 [Note] /usr/libexec/mysqld: Normal shutdown
150409 18:46:43 [Note] Event Scheduler: Purging the queue. 0 events
150409 18:46:45 InnoDB: Starting shutdown...
150409 18:46:46 InnoDB: Shutdown completed; log sequence number 0 44303
150409 18:46:46 [Note] /usr/libexec/mysqld: Shutdown complete
150409 18:46:46 mysqld_safe mysqld from pid file /var/run/mysqld/mysqld.pid ended
150409 18:47:00 mysqld_safe Starting mysqld daemon with databases from /var/lib/mysql
150409 18:47:00 InnoDB: Initializing buffer pool, size = 8.0M
150409 18:47:00 InnoDB: Completed initialization of buffer pool
150409 18:47:00 InnoDB: Started; log sequence number 0 44303
150409 18:47:00 [Note] Event Scheduler: Loaded 0 events
150409 18:47:00 [Note] /usr/libexec/mysqld: ready for connections.
Version: '5.1.67' socket: '/var/lib/mysql/mysql.sock' port: 3306 Source distribution
[root@esu1l268 scripts]# service ndo2db start
Starting ndo2db: done.
[root@esu1l268 scripts]# tail -100 /var/log/messages
Apr 9 18:45:02 esu1l268 nagios: wproc: Core Worker 22208: job 5131 (pid=834): Dormant child reaped
Apr 9 18:45:02 esu1l268 nagios: wproc: Core Worker 22211: job 5131 (pid=890) timed out. Killing it
Apr 9 18:45:02 esu1l268 nagios: wproc: CHECK job 5131 from worker Core Worker 22211 timed out after 60.01s
Apr 9 18:45:02 esu1l268 nagios: wproc: host=esu1l318; service=check_FS_space_linux_by_sshpass;
Apr 9 18:45:02 esu1l268 nagios: wproc: early_timeout=1; exited_ok=0; wait_status=0; error_code=62;
Apr 9 18:45:02 esu1l268 nagios: Warning: Check of service 'check_FS_space_linux_by_sshpass' on host 'esu1l 318' timed out after 60.007s!
Apr 9 18:45:02 esu1l268 nagios: wproc: Core Worker 22211: job 5131 (pid=890): Dormant child reaped
Apr 9 18:45:03 esu1l268 nagios: wproc: Core Worker 22245: job 5131 (pid=1150) timed out. Killing it
Apr 9 18:45:03 esu1l268 nagios: wproc: CHECK job 5131 from worker Core Worker 22245 timed out after 60.02s
Apr 9 18:45:03 esu1l268 nagios: wproc: host=esu1l321; service=check_FS_space_linux_by_sshpass;
Apr 9 18:45:03 esu1l268 nagios: wproc: early_timeout=1; exited_ok=0; wait_status=0; error_code=62;
Apr 9 18:45:03 esu1l268 nagios: Warning: Check of service 'check_FS_space_linux_by_sshpass' on host 'esu1l 321' timed out after 60.017s!
Apr 9 18:45:03 esu1l268 nagios: wproc: Core Worker 22245: job 5131 (pid=1150): Dormant child reaped
Apr 9 18:45:04 esu1l268 nagios: wproc: Core Worker 22200: job 5132 (pid=1245) timed out. Killing it
Apr 9 18:45:04 esu1l268 nagios: wproc: CHECK job 5132 from worker Core Worker 22200 timed out after 60.01s
Apr 9 18:45:04 esu1l268 nagios: wproc: host=esu1l325; service=check_FS_space_linux_by_sshpass;
Apr 9 18:45:04 esu1l268 nagios: wproc: early_timeout=1; exited_ok=0; wait_status=0; error_code=62;
Apr 9 18:45:04 esu1l268 nagios: Warning: Check of service 'check_FS_space_linux_by_sshpass' on host 'esu1l 325' timed out after 60.007s!
Apr 9 18:45:04 esu1l268 nagios: wproc: Core Worker 22200: job 5132 (pid=1245): Dormant child reaped
Apr 9 18:45:07 esu1l268 nagios: wproc: Core Worker 22207: job 5133 (pid=2386) timed out. Killing it
Apr 9 18:45:07 esu1l268 nagios: wproc: CHECK job 5133 from worker Core Worker 22207 timed out after 60.01s
Apr 9 18:45:07 esu1l268 nagios: wproc: host=esu1l327; service=check_FS_space_linux_by_sshpass;
Apr 9 18:45:07 esu1l268 nagios: wproc: early_timeout=1; exited_ok=0; wait_status=0; error_code=62;
Apr 9 18:45:07 esu1l268 nagios: Warning: Check of service 'check_FS_space_linux_by_sshpass' on host 'esu1l 327' timed out after 60.006s!
Apr 9 18:45:07 esu1l268 nagios: wproc: Core Worker 22207: job 5133 (pid=2386): Dormant child reaped
Apr 9 18:45:09 esu1l268 nagios: wproc: Core Worker 22244: job 5133 (pid=2900) timed out. Killing it
Apr 9 18:45:09 esu1l268 nagios: wproc: CHECK job 5133 from worker Core Worker 22244 timed out after 60.01s
Apr 9 18:45:09 esu1l268 nagios: wproc: host=esu1l326; service=check_FS_space_linux_by_sshpass;
Apr 9 18:45:09 esu1l268 nagios: wproc: early_timeout=1; exited_ok=0; wait_status=0; error_code=62;
Apr 9 18:45:09 esu1l268 nagios: Warning: Check of service 'check_FS_space_linux_by_sshpass' on host 'esu1l 326' timed out after 60.006s!
Apr 9 18:45:09 esu1l268 nagios: wproc: Core Worker 22244: job 5133 (pid=2900): Dormant child reaped
Apr 9 18:45:13 esu1l268 nagios: wproc: Core Worker 22208: job 5135 (pid=3757) timed out. Killing it
Apr 9 18:45:13 esu1l268 nagios: wproc: CHECK job 5135 from worker Core Worker 22208 timed out after 60.01s
Apr 9 18:45:13 esu1l268 nagios: wproc: host=esu2v262; service=Check for read only File system on linux;
Apr 9 18:45:13 esu1l268 nagios: wproc: early_timeout=1; exited_ok=0; wait_status=0; error_code=62;
Apr 9 18:45:13 esu1l268 nagios: Warning: Check of service 'Check for read only File system on linux' on ho st 'esu2v262' timed out after 60.005s!
Apr 9 18:45:13 esu1l268 nagios: wproc: Core Worker 22208: job 5135 (pid=3757): Dormant child reaped
Apr 9 18:45:16 esu1l268 vasd[2315]: _create_rule_from_local_file: Cannot locate <FEDERATED.FDS\UxRG_esu1l2 68>(UxRG_esu1l268) from file </etc/opt/quest/vas/users.allow>, error: <2>. Resolution will be attempted aga in in 30 seconds.
Apr 9 18:45:25 esu1l268 nagios: wproc: Core Worker 22248: job 5140 (pid=7581) timed out. Killing it
Apr 9 18:45:25 esu1l268 nagios: wproc: CHECK job 5140 from worker Core Worker 22248 timed out after 60.01s
Apr 9 18:45:25 esu1l268 nagios: wproc: host=scj3dbadm01; service=check_FS_space_Solaris_by_sshpass;
Apr 9 18:45:25 esu1l268 nagios: wproc: early_timeout=1; exited_ok=0; wait_status=0; error_code=62;
Apr 9 18:45:25 esu1l268 nagios: Warning: Check of service 'check_FS_space_Solaris_by_sshpass' on host 'scj 3dbadm01' timed out after 60.006s!
Apr 9 18:45:25 esu1l268 nagios: wproc: Core Worker 22248: job 5140 (pid=7581): Dormant child reaped
Apr 9 18:45:38 esu1l268 nagios: wproc: Core Worker 22219: job 5145 (pid=11115) timed out. Killing it
Apr 9 18:45:38 esu1l268 nagios: wproc: CHECK job 5145 from worker Core Worker 22219 timed out after 60.01s
Apr 9 18:45:38 esu1l268 nagios: wproc: host=esu2v660; service=Check for read only File system on linux;
Apr 9 18:45:38 esu1l268 nagios: wproc: early_timeout=1; exited_ok=0; wait_status=0; error_code=62;
Apr 9 18:45:38 esu1l268 nagios: Warning: Check of service 'Check for read only File system on linux' on ho st 'esu2v660' timed out after 60.006s!
Apr 9 18:45:38 esu1l268 nagios: wproc: Core Worker 22219: job 5145 (pid=11115): Dormant child reaped
Apr 9 18:45:39 esu1l268 nagios: wproc: Core Worker 22237: job 5145 (pid=11234) timed out. Killing it
Apr 9 18:45:39 esu1l268 nagios: wproc: CHECK job 5145 from worker Core Worker 22237 timed out after 60.01s
Apr 9 18:45:39 esu1l268 nagios: wproc: host=esu2v219; service=Check for read only File system on linux;
Apr 9 18:45:39 esu1l268 nagios: wproc: early_timeout=1; exited_ok=0; wait_status=0; error_code=62;
Apr 9 18:45:39 esu1l268 nagios: Warning: Check of service 'Check for read only File system on linux' on ho st 'esu2v219' timed out after 60.008s!
Apr 9 18:45:39 esu1l268 nagios: wproc: Core Worker 22237: job 5145 (pid=11234): Dormant child reaped
Apr 9 18:45:41 esu1l268 nagios: wproc: Core Worker 22205: job 5146 (pid=11753) timed out. Killing it
Apr 9 18:45:41 esu1l268 nagios: wproc: CHECK job 5146 from worker Core Worker 22205 timed out after 60.01s
Apr 9 18:45:41 esu1l268 nagios: wproc: host=sun21; service=check_FS_space_Solaris_by_sshpass;
Apr 9 18:45:41 esu1l268 nagios: wproc: early_timeout=1; exited_ok=0; wait_status=0; error_code=62;
Apr 9 18:45:41 esu1l268 nagios: Warning: Check of service 'check_FS_space_Solaris_by_sshpass' on host 'sun 21' timed out after 60.008s!
Apr 9 18:45:41 esu1l268 nagios: wproc: Core Worker 22205: job 5146 (pid=11753): Dormant child reaped
Apr 9 18:45:47 esu1l268 vasd[2315]: _create_rule_from_local_file: Cannot locate <FEDERATED.FDS\UxRG_esu1l2 68>(UxRG_esu1l268) from file </etc/opt/quest/vas/users.allow>, error: <2>. Resolution will be attempted aga in in 30 seconds.
Apr 9 18:45:51 esu1l268 nagios: wproc: Core Worker 22200: job 5150 (pid=13976) timed out. Killing it
Apr 9 18:45:51 esu1l268 nagios: wproc: CHECK job 5150 from worker Core Worker 22200 timed out after 60.01s
Apr 9 18:45:51 esu1l268 nagios: wproc: host=esu1l316; service=check_FS_space_linux_by_sshpass;
Apr 9 18:45:51 esu1l268 nagios: wproc: early_timeout=1; exited_ok=0; wait_status=0; error_code=62;
Apr 9 18:45:51 esu1l268 nagios: Warning: Check of service 'check_FS_space_linux_by_sshpass' on host 'esu1l 316' timed out after 60.006s!
Apr 9 18:45:51 esu1l268 nagios: wproc: Core Worker 22200: job 5150 (pid=13976): Dormant child reaped
Apr 9 18:45:53 esu1l268 nagios: wproc: Core Worker 22234: job 5150 (pid=14403) timed out. Killing it
Apr 9 18:45:53 esu1l268 nagios: wproc: CHECK job 5150 from worker Core Worker 22234 timed out after 60.01s
Apr 9 18:45:53 esu1l268 nagios: wproc: host=esu1l044; service=check_FS_space_linux_by_sshpass;
Apr 9 18:45:53 esu1l268 nagios: wproc: early_timeout=1; exited_ok=0; wait_status=0; error_code=62;
Apr 9 18:45:53 esu1l268 nagios: Warning: Check of service 'check_FS_space_linux_by_sshpass' on host 'esu1l 044' timed out after 60.007s!
Apr 9 18:45:53 esu1l268 nagios: wproc: Core Worker 22234: job 5150 (pid=14403): Dormant child reaped
Apr 9 18:45:55 esu1l268 nagios: wproc: Core Worker 22215: job 5151 (pid=14765) timed out. Killing it
Apr 9 18:45:55 esu1l268 nagios: wproc: CHECK job 5151 from worker Core Worker 22215 timed out after 60.01s
Apr 9 18:45:55 esu1l268 nagios: wproc: host=esu1l332; service=check_FS_space_linux_by_sshpass;
Apr 9 18:45:55 esu1l268 nagios: wproc: early_timeout=1; exited_ok=0; wait_status=0; error_code=62;
Apr 9 18:45:55 esu1l268 nagios: Warning: Check of service 'check_FS_space_linux_by_sshpass' on host 'esu1l 332' timed out after 60.007s!
Apr 9 18:45:55 esu1l268 nagios: wproc: Core Worker 22215: job 5151 (pid=14765): Dormant child reaped
Apr 9 18:45:59 esu1l268 nagios: wproc: Core Worker 22236: job 5152 (pid=15707) timed out. Killing it
Apr 9 18:45:59 esu1l268 nagios: wproc: CHECK job 5152 from worker Core Worker 22236 timed out after 60.01s
Apr 9 18:45:59 esu1l268 nagios: wproc: host=esu3v128; service=Check for read only File system on linux;
Apr 9 18:45:59 esu1l268 nagios: wproc: early_timeout=1; exited_ok=0; wait_status=0; error_code=62;
Apr 9 18:45:59 esu1l268 nagios: Warning: Check of service 'Check for read only File system on linux' on ho st 'esu3v128' timed out after 60.005s!
Apr 9 18:45:59 esu1l268 nagios: wproc: Core Worker 22236: job 5152 (pid=15707): Dormant child reaped
Apr 9 18:46:01 esu1l268 nagios: Caught SIGTERM, shutting down...
Apr 9 18:46:01 esu1l268 nagios: Successfully shutdown... (PID=22192)
Apr 9 18:46:01 esu1l268 nagios: Event broker module 'NERD' deinitialized successfully.
Apr 9 18:46:01 esu1l268 nagios: HP BSM Integration: Event Broker Module has been successfully unloaded.
Apr 9 18:46:01 esu1l268 nagios: Event broker module '/opt/OV/HPBsmIntNagios/lib64/libbsmintneb4.so' deinit ialized successfully.
Apr 9 18:46:01 esu1l268 nagios: ndomod: Shutdown complete.
Apr 9 18:46:01 esu1l268 nagios: Event broker module '/usr/local/nagios/bin/ndomod.o' deinitialized success fully.
Apr 9 18:46:17 esu1l268 vasd[2315]: _create_rule_from_local_file: Cannot locate <FEDERATED.FDS\UxRG_esu1l2 68>(UxRG_esu1l268) from file </etc/opt/quest/vas/users.allow>, error: <2>. Resolution will be attempted aga in in 30 seconds.
Apr 9 18:46:32 esu1l268 snmpd[2527]: Connection from UDP: [11.48.116.70]:48920->[11.48.4.85]
Apr 9 18:46:48 esu1l268 vasd[2315]: _create_rule_from_local_file: Cannot locate <FEDERATED.FDS\UxRG_esu1l2 68>(UxRG_esu1l268) from file </etc/opt/quest/vas/users.allow>, error: <2>. Resolution will be attempted aga in in 30 seconds.
Apr 9 18:47:19 esu1l268 vasd[2315]: _create_rule_from_local_file: Cannot locate <FEDERATED.FDS\UxRG_esu1l2 68>(UxRG_esu1l268) from file </etc/opt/quest/vas/users.allow>, error: <2>. Resolution will be attempted aga in in 30 seconds.
Apr 9 18:47:42 esu1l268 sshd[24527]: Accepted keyboard-interactive/pam for b161172 from 11.48.70.73 port 2 5197 ssh2
Apr 9 18:47:49 esu1l268 vasd[2315]: _create_rule_from_local_file: Cannot locate <FEDERATED.FDS\UxRG_esu1l2 68>(UxRG_esu1l268) from file </etc/opt/quest/vas/users.allow>, error: <2>. Resolution will be attempted aga in in 30 seconds.
[root@esu1l268 scripts]#
tail -100 /usr/local/nagios/var/nagios.log:
Code: Select all
[root@esu1l268 ~]# tail -100 /usr/local/nagios/var/nagios.log
[1428620223] wproc: host=esu2v805; service=Check for read only File system on linux;
[1428620223] wproc: early_timeout=1; exited_ok=0; wait_status=0; error_code=62;
[1428620223] Warning: Check of service 'Check for read only File system on linux' on host 'esu2v805' timed out after 60.011s!
[1428620223] wproc: Core Worker 24762: job 139 (pid=7841): Dormant child reaped
[1428620224] wproc: Core Worker 24753: job 139 (pid=7879) timed out. Killing it
[1428620224] wproc: CHECK job 139 from worker Core Worker 24753 timed out after 60.01s
[1428620224] wproc: host=esu2v823; service=Check for read only File system on linux;
[1428620224] wproc: early_timeout=1; exited_ok=0; wait_status=0; error_code=62;
[1428620224] Warning: Check of service 'Check for read only File system on linux' on host 'esu2v823' timed out after 60.010s!
[1428620224] wproc: Core Worker 24753: job 139 (pid=7879): Dormant child reaped
[1428620230] wproc: Core Worker 24752: job 142 (pid=8419) timed out. Killing it
[1428620230] wproc: CHECK job 142 from worker Core Worker 24752 timed out after 60.01s
[1428620230] wproc: host=esu2v974; service=Check for read only File system on linux;
[1428620230] wproc: early_timeout=1; exited_ok=0; wait_status=0; error_code=62;
[1428620230] Warning: Check of service 'Check for read only File system on linux' on host 'esu2v974' timed out after 60.006s!
[1428620230] wproc: Core Worker 24752: job 142 (pid=8419): Dormant child reaped
[1428620233] wproc: Core Worker 24748: job 143 (pid=8602) timed out. Killing it
[1428620233] wproc: CHECK job 143 from worker Core Worker 24748 timed out after 60.01s
[1428620233] wproc: host=esu1l321; service=Check for read only File system on linux;
[1428620233] wproc: early_timeout=1; exited_ok=0; wait_status=0; error_code=62;
[1428620233] Warning: Check of service 'Check for read only File system on linux' on host 'esu1l321' timed out after 60.013s!
[1428620233] wproc: Core Worker 24748: job 143 (pid=8602): Dormant child reaped
[1428620240] SERVICE ALERT: scj3-appadm01;Ping;CRITICAL;SOFT;1;CRITICAL - 11.48.37.140: rta nan, lost 100%
[1428620240] HP BSM Integration: Forwarding event for service 'Ping' (scj3-appadm01)...
[1428620242] wproc: Core Worker 24774: job 146 (pid=9496) timed out. Killing it
[1428620242] wproc: CHECK job 146 from worker Core Worker 24774 timed out after 60.01s
[1428620242] wproc: host=esu2v653; service=Check for read only File system on linux;
[1428620242] wproc: early_timeout=1; exited_ok=0; wait_status=0; error_code=62;
[1428620242] Warning: Check of service 'Check for read only File system on linux' on host 'esu2v653' timed out after 60.008s!
[1428620242] wproc: Core Worker 24774: job 146 (pid=9496): Dormant child reaped
[1428620242] wproc: Core Worker 24779: job 146 (pid=9621) timed out. Killing it
[1428620242] wproc: CHECK job 146 from worker Core Worker 24779 timed out after 60.01s
[1428620242] wproc: host=esu4v110; service=Check for read only File system on linux;
[1428620242] wproc: early_timeout=1; exited_ok=0; wait_status=0; error_code=62;
[1428620242] Warning: Check of service 'Check for read only File system on linux' on host 'esu4v110' timed out after 60.006s!
[1428620242] wproc: Core Worker 24779: job 146 (pid=9621): Dormant child reaped
[1428620242] wproc: Core Worker 24745: job 147 (pid=9743) timed out. Killing it
[1428620242] wproc: CHECK job 147 from worker Core Worker 24745 timed out after 60.01s
[1428620242] wproc: host=esu4v128; service=Check for read only File system on linux;
[1428620242] wproc: early_timeout=1; exited_ok=0; wait_status=0; error_code=62;
[1428620242] Warning: Check of service 'Check for read only File system on linux' on host 'esu4v128' timed out after 60.007s!
[1428620242] wproc: Core Worker 24745: job 147 (pid=9743): Dormant child reaped
[1428620243] wproc: Core Worker 24755: job 147 (pid=9769) timed out. Killing it
[1428620243] wproc: CHECK job 147 from worker Core Worker 24755 timed out after 60.01s
[1428620243] wproc: host=esu4v133; service=Check for read only File system on linux;
[1428620243] wproc: early_timeout=1; exited_ok=0; wait_status=0; error_code=62;
[1428620243] Warning: Check of service 'Check for read only File system on linux' on host 'esu4v133' timed out after 60.006s!
[1428620243] wproc: Core Worker 24755: job 147 (pid=9769): Dormant child reaped
[1428620246] wproc: Core Worker 24768: job 148 (pid=10422) timed out. Killing it
[1428620246] wproc: CHECK job 148 from worker Core Worker 24768 timed out after 60.01s
[1428620246] wproc: host=ibm104p01; service=check_FS_space_AIX_by_sshpass;
[1428620246] wproc: early_timeout=1; exited_ok=0; wait_status=0; error_code=62;
[1428620246] Warning: Check of service 'check_FS_space_AIX_by_sshpass' on host 'ibm104p01' timed out after 60.008s!
[1428620246] wproc: Core Worker 24768: job 148 (pid=10422): Dormant child reaped
[1428620247] wproc: Core Worker 24774: job 148 (pid=10558) timed out. Killing it
[1428620247] wproc: CHECK job 148 from worker Core Worker 24774 timed out after 60.01s
[1428620247] wproc: host=ibm104p04; service=check_FS_space_AIX_by_sshpass;
[1428620247] wproc: early_timeout=1; exited_ok=0; wait_status=0; error_code=62;
[1428620247] Warning: Check of service 'check_FS_space_AIX_by_sshpass' on host 'ibm104p04' timed out after 60.005s!
[1428620247] wproc: Core Worker 24774: job 148 (pid=10558): Dormant child reaped
[1428620250] HOST ALERT: scj3-appadm01;DOWN;SOFT;1;CRITICAL - 11.48.37.140: rta nan, lost 100%
[1428620250] HP BSM Integration: Forwarding event for host 'scj3-appadm01'...
[1428620250] wproc: Core Worker 24779: job 149 (pid=11077) timed out. Killing it
[1428620250] wproc: CHECK job 149 from worker Core Worker 24779 timed out after 60.01s
[1428620250] wproc: host=ibm70p7; service=check_FS_space_AIX_by_sshpass;
[1428620250] wproc: early_timeout=1; exited_ok=0; wait_status=0; error_code=62;
[1428620250] Warning: Check of service 'check_FS_space_AIX_by_sshpass' on host 'ibm70p7' timed out after 60.010s!
[1428620250] wproc: Core Worker 24779: job 149 (pid=11077): Dormant child reaped
[1428620251] wproc: Core Worker 24747: job 150 (pid=11197) timed out. Killing it
[1428620251] wproc: CHECK job 150 from worker Core Worker 24747 timed out after 60.01s
[1428620251] wproc: host=ibm78p02; service=check_FS_space_AIX_by_sshpass;
[1428620251] wproc: early_timeout=1; exited_ok=0; wait_status=0; error_code=62;
[1428620251] Warning: Check of service 'check_FS_space_AIX_by_sshpass' on host 'ibm78p02' timed out after 60.008s!
[1428620251] wproc: Core Worker 24747: job 150 (pid=11197): Dormant child reaped
[1428620251] wproc: Core Worker 24775: job 150 (pid=11204) timed out. Killing it
[1428620251] wproc: CHECK job 150 from worker Core Worker 24775 timed out after 60.01s
[1428620251] wproc: host=ibm78p11; service=check_FS_space_AIX_by_sshpass;
[1428620251] wproc: early_timeout=1; exited_ok=0; wait_status=0; error_code=62;
[1428620251] Warning: Check of service 'check_FS_space_AIX_by_sshpass' on host 'ibm78p11' timed out after 60.007s!
[1428620251] wproc: Core Worker 24775: job 150 (pid=11204): Dormant child reaped
[1428620297] wproc: Core Worker 24766: job 159 (pid=21904) timed out. Killing it
[1428620297] wproc: CHECK job 159 from worker Core Worker 24766 timed out after 60.01s
[1428620297] wproc: host=scr1db02; service=check_FS_space_Solaris_by_sshpass;
[1428620297] wproc: early_timeout=1; exited_ok=0; wait_status=0; error_code=62;
[1428620297] Warning: Check of service 'check_FS_space_Solaris_by_sshpass' on host 'scr1db02' timed out after 60.009s!
[1428620297] wproc: Core Worker 24766: job 159 (pid=21904): Dormant child reaped
[1428620297] SERVICE ALERT: scj3-appadm01;Ping;CRITICAL;SOFT;2;CRITICAL - 11.48.37.140: rta nan, lost 100%
[1428620307] HOST ALERT: scj3-appadm01;DOWN;SOFT;2;CRITICAL - 11.48.37.140: rta nan, lost 100%
[1428620316] wproc: Core Worker 24758: job 162 (pid=26873) timed out. Killing it
[1428620316] wproc: CHECK job 162 from worker Core Worker 24758 timed out after 60.01s
[1428620316] wproc: host=ibmpci00p02; service=check_FS_space_AIX_by_sshpass;
[1428620316] wproc: early_timeout=1; exited_ok=0; wait_status=0; error_code=62;
[1428620316] Warning: Check of service 'check_FS_space_AIX_by_sshpass' on host 'ibmpci00p02' timed out after 60.011s!
[1428620316] wproc: Core Worker 24758: job 162 (pid=26873): Dormant child reaped
[1428620321] wproc: Core Worker 24776: job 162 (pid=27858) timed out. Killing it
[1428620321] wproc: CHECK job 162 from worker Core Worker 24776 timed out after 60.01s
[1428620321] wproc: host=esu2v221; service=Check for read only File system on linux;
[1428620321] wproc: early_timeout=1; exited_ok=0; wait_status=0; error_code=62;
[1428620321] Warning: Check of service 'Check for read only File system on linux' on host 'esu2v221' timed out after 60.015s!
[1428620321] wproc: Core Worker 24776: job 162 (pid=27858): Dormant child reaped
[root@esu1l268 ~]#