Multiple (40+) poller cron jobs running

[GhostRider2110]

Is this normal? from

Code: Select all

ps -ef:
pache   44182 61676  0 08:36 ?        00:00:01 /usr/sbin/httpd
apache   44183 61676  0 08:36 ?        00:00:01 /usr/sbin/httpd
apache   44184 61676  0 08:36 ?        00:00:01 /usr/sbin/httpd
apache   44647 61676  0 08:41 ?        00:00:01 /usr/sbin/httpd
apache   44662 61676  0 08:41 ?        00:00:00 /usr/sbin/httpd
apache   44663 61676  0 08:41 ?        00:00:00 /usr/sbin/httpd
root     45494  1638  0 08:50 ?        00:00:00 CROND
nagios   45497 45494  0 08:50 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   45500 45497  0 08:50 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     45614  1638  0 08:51 ?        00:00:00 CROND
nagios   45617 45614  0 08:51 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   45618 45617  0 08:51 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     45714  1638  0 08:52 ?        00:00:00 CROND
nagios   45715 45714  0 08:52 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   45718 45715  0 08:52 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     45816  1638  0 08:53 ?        00:00:00 CROND
nagios   45818 45816  0 08:53 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   45819 45818  0 08:53 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     45894  1638  0 08:54 ?        00:00:00 CROND
nagios   45896 45894  0 08:54 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   45898 45896  0 08:54 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     45939  1638  0 08:55 ?        00:00:00 CROND
nagios   45940 45939  0 08:55 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   45942 45940  0 08:55 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     46067  1638  0 08:56 ?        00:00:00 CROND
nagios   46068 46067  0 08:56 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   46071 46068  0 08:56 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     46183  1638  0 08:57 ?        00:00:00 CROND
nagios   46184 46183  0 08:57 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   46186 46184  0 08:57 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     46283  1638  0 08:58 ?        00:00:00 CROND
nagios   46285 46283  0 08:58 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   46287 46285  0 08:58 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     46361  1638  0 08:59 ?        00:00:00 CROND
nagios   46362 46361  0 08:59 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   46363 46362  0 08:59 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     46426  1638  0 09:00 ?        00:00:00 CROND
nagios   46429 46426  0 09:00 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   46432 46429  0 09:00 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     46553  1638  0 09:01 ?        00:00:00 CROND
nagios   46555 46553  0 09:01 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   46557 46555  0 09:01 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     46656  1638  0 09:02 ?        00:00:00 CROND
nagios   46657 46656  0 09:02 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   46659 46657  0 09:02 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     46757  1638  0 09:03 ?        00:00:00 CROND
nagios   46758 46757  0 09:03 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   46760 46758  0 09:03 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     46836  1638  0 09:04 ?        00:00:00 CROND
nagios   46838 46836  0 09:04 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   46840 46838  0 09:04 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     46902  1638  0 09:05 ?        00:00:00 CROND
nagios   46903 46902  0 09:05 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   46905 46903  0 09:05 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     47003  1638  0 09:06 ?        00:00:00 CROND
nagios   47005 47003  0 09:06 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   47006 47005  0 09:06 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     47076  1638  0 09:07 ?        00:00:00 CROND
nagios   47078 47076  0 09:07 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   47080 47078  0 09:07 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     47176  1638  0 09:08 ?        00:00:00 CROND
nagios   47178 47176  0 09:08 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   47179 47178  0 09:08 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     47254  1638  0 09:09 ?        00:00:00 CROND
nagios   47256 47254  0 09:09 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   47258 47256  0 09:09 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     47264  1638  0 09:10 ?        00:00:00 CROND
nagios   47266 47264  0 09:10 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   47269 47266  0 09:10 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     47412  1638  0 09:11 ?        00:00:00 CROND
nagios   47413 47412  0 09:11 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   47415 47413  0 09:11 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     47581  1638  0 09:12 ?        00:00:00 CROND
nagios   47583 47581  0 09:12 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   47585 47583  0 09:12 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     47653  1638  0 09:13 ?        00:00:00 CROND
nagios   47655 47653  0 09:13 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   47656 47655  0 09:13 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     47765  1638  0 09:14 ?        00:00:00 CROND
nagios   47767 47765  0 09:14 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   47769 47767  0 09:14 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     47825  1638  0 09:15 ?        00:00:00 CROND
nagios   47827 47825  0 09:15 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   47828 47827  0 09:15 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     47941  1638  0 09:16 ?        00:00:00 CROND
nagios   47943 47941  0 09:16 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   47945 47943  0 09:16 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     48032  1638  0 09:17 ?        00:00:00 CROND
nagios   48033 48032  0 09:17 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   48035 48033  0 09:17 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     48131  1638  0 09:18 ?        00:00:00 CROND
nagios   48133 48131  0 09:18 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   48135 48133  0 09:18 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     48212  1638  0 09:19 ?        00:00:00 CROND
nagios   48213 48212  0 09:19 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   48215 48213  0 09:19 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     48277  1638  0 09:20 ?        00:00:00 CROND
nagios   48279 48277  0 09:20 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   48282 48279  0 09:20 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     48395  1638  0 09:21 ?        00:00:00 CROND
nagios   48397 48395  0 09:21 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   48399 48397  0 09:21 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     48487  1638  0 09:22 ?        00:00:00 CROND
nagios   48489 48487  0 09:22 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   48490 48489  0 09:22 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     48586  1638  0 09:23 ?        00:00:00 CROND
nagios   48588 48586  0 09:23 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   48590 48588  0 09:23 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     48668  1638  0 09:24 ?        00:00:00 CROND
nagios   48670 48668  0 09:24 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   48672 48670  0 09:24 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     48731  1638  0 09:25 ?        00:00:00 CROND
nagios   48732 48731  0 09:25 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   48735 48732  0 09:25 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     48849  1638  0 09:26 ?        00:00:00 CROND
nagios   48852 48849  0 09:26 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   48853 48852  0 09:26 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     48966  1638  0 09:27 ?        00:00:00 CROND
nagios   48967 48966  0 09:27 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   48969 48967  0 09:27 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     49043  1638  0 09:28 ?        00:00:00 CROND
nagios   49046 49043  0 09:28 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   49047 49046  0 09:28 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     49124  1638  0 09:29 ?        00:00:00 CROND
nagios   49125 49124  0 09:29 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   49128 49125  0 09:29 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     49187  1638  0 09:30 ?        00:00:00 CROND
nagios   49190 49187  0 09:30 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   49193 49190  0 09:30 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     49304  1638  0 09:31 ?        00:00:00 CROND
nagios   49305 49304  0 09:31 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   49307 49305  0 09:31 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     49417  1638  0 09:32 ?        00:00:00 CROND
nagios   49419 49417  0 09:32 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   49420 49419  0 09:32 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     49496  1638  0 09:33 ?        00:00:00 CROND
nagios   49497 49496  0 09:33 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   49499 49497  0 09:33 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     49519  1519  0 09:33 ?        00:00:00 sshd: root@pts/1 
root     49521 49519  0 09:33 pts/1    00:00:00 -bash
root     49596  1638  0 09:34 ?        00:00:00 CROND
nagios   49597 49596  0 09:34 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   49599 49597  0 09:34 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     49659  1638  0 09:35 ?        00:00:00 CROND
nagios   49660 49659  0 09:35 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   49663 49660  0 09:35 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     49773  1638  0 09:36 ?        00:00:00 CROND
root     49774  1638  0 09:36 ?        00:00:00 CROND
nagios   49775 49774  0 09:36 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   49776 49773  0 09:36 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php jobs > /usr/local/nagioslogserver/var/jobs.log 2>&1
nagios   49777 49776  0 09:36 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php jobs
nagios   49778 49775  0 09:36 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
root     49828 49521  1 09:36 pts/1    00:00:00 ps -ef
root     61060  1519  0 Apr07 ?        00:00:02 sshd: root@pts/0 
root     61062 61060  0 Apr07 pts/0    00:00:00 -bash
root     61137     1  0 Apr07 pts/0    00:00:00 dbus-launch --autolaunch 16ad65c92ea45bd0cc8dbebf00000018 --binary-syntax --close-stderr
root     61138     1  0 Apr07 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 5 --print-address 7 --session
root     61676     1  0 Apr09 ?        00:00:14 /usr/sbin/httpd
nagios   61785     1 16 Apr09 ?        23:49:05 /usr/bin/java -Xms2g -Xmx2g -Xss256k -Djava.awt.headless=true -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+
root     62285     1  0 Apr09 ?        00:00:00 runuser -s /bin/sh -c exec /usr/local/nagioslogserver/logstash/bin/logstash agent -f /usr/local/nagioslogserver/logstash/etc/conf.d -l /var/log/logstash/logstash.log  -w 4 nagios
nagios   62287 62285 24 Apr09 ?        1-10:17:47 /usr/bin/java -Djava.io.tmpdir=/usr/local/nagioslogserver/tmp -Xmx500m -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -Djava.awt.headless=true -XX:CMSInitiatingOccupancyFraction=75 -XX:
apache   63248 61676  0 Apr12 ?        00:00:16 /usr/sbin/httpd
apache   63249 61676  0 Apr12 ?        00:00:16 /usr/sbin/httpd
apache   63250 61676  0 Apr12 ?        00:00:16 /usr/sbin/httpd
apache   63251 61676  0 Apr12 ?        00:00:16 /usr/sbin/httpd
apache   63252 61676  0 Apr12 ?        00:00:15 /usr/sbin/httpd

The poller.log file is not showing much of anything. keeps truncating with:

Code: Select all

[root@IGAnagioslog ~]# tail -100f /usr/local/nagioslogserver/var/poller.log
Updating Cluster Hosts File
Updating Elasticsearch with instance...
Finished Polling.
tail: /usr/local/nagioslogserver/var/poller.log: file truncated
Updating Cluster Hosts File
Updating Elasticsearch with instance...
Finished Polling.

See-ya
Mitch

[jolson]

It could be normal - how many instances are in the cluster?

The poller subsystem updates the the elasticsearch database with the current running configuration. It also updates service state information for all of the instances in the Nagios Log Server cluster. In addition to this, the poller subsystem also updates each instance with the current list of all known hosts in the cluster.

Does the poller.log look unusual at all? It is normal that it truncates so often - but are any of the messages rapid or excessive?

[GhostRider2110]

System currently is configured with one system and one instance.

The poller.log is just what I listed. Nothing going into it but:

Code: Select all

[root@IGAnagioslog ~]# tail -100f /usr/local/nagioslogserver/var/poller.log
Updating Cluster Hosts File
Updating Elasticsearch with instance...
Finished Polling.
tail: /usr/local/nagioslogserver/var/poller.log: file truncated
Updating Cluster Hosts File
Updating Elasticsearch with instance...
Finished Polling.

Over and over.

[jolson]

Poller is an internal system used to keep track of cluster hosts and elasticsearch configuration - it's unlikely to be related to an rsyslog issue. It is interesting that your poller jobs seem to be hanging though.

On a lab system:

Code: Select all

# ps -ef |grep poller|grep -v grep
nagios    6909  6908  0 11:23 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios    6911  6909  0 11:23 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller

Poller executes once per minute on a cron job, along with jobs.log:

Code: Select all

# cat /etc/cron.d/nagioslogserver
* * * * * nagios /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
* * * * * nagios /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php jobs > /usr/local/nagioslogserver/var/jobs.log 2>&1

If you run a 'reset jobs' from the NLS GUI -> Command Subsystem, do the poller jobs all clear out?

[GhostRider2110]

I'll have to wait until the problem happens again. Right now all is running smooth after I restarted electricsearch. All the "hung" poller processes slowly went away over a 3-4min period. Looking back in the screen buffer, it seems both poller and jobs were hanging out there. But there were a lot more poller processes showing than jobs processes.

See-ya
Mitch

[jolson]

Sounds good - please let us know if this happens again.

[GhostRider2110]

Will do, this is the second or third time this has happened, I did check and am running the latest version and did rerun the apply configuration after the update. Now that I have a feeling of a relationship in the two problems I am having, I can keep a different eye out. --Mitch

[jolson]

Thank you Mitch.

[GhostRider2110]

Starting to see the multiple 'jobs' processes hanging around again.

Code: Select all

nagios    1640  1638 15 Apr16 ?        1-18:26:44 /usr/bin/java -Djava.io.tmpdir=/usr/local/nagioslogserver/tmp -Xmx500m -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -Djava.awt.headless=true
root      1690     1  0 Apr16 tty1     00:00:00 /sbin/mingetty /dev/tty1
root      1692     1  0 Apr16 tty2     00:00:00 /sbin/mingetty /dev/tty2
root      1694     1  0 Apr16 tty3     00:00:00 /sbin/mingetty /dev/tty3
root      1696     1  0 Apr16 tty4     00:00:00 /sbin/mingetty /dev/tty4
root      1698     1  0 Apr16 tty5     00:00:00 /sbin/mingetty /dev/tty5
root      1701   457  0 Apr16 ?        00:00:00 /sbin/udevd -d
root      1702   457  0 Apr16 ?        00:00:00 /sbin/udevd -d
root      1703     1  0 Apr16 tty6     00:00:00 /sbin/mingetty /dev/tty6
root      1716     2  0 Apr16 ?        00:04:24 [flush-253:0]
root      2733  1613  0 Apr21 ?        00:00:00 CROND
nagios    2735  2733  0 Apr21 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php jobs > /usr/local/nagioslogserver/var/jobs.log 2>&1
nagios    2738  2735  0 Apr21 ?        00:00:02 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php jobs
nagios    2799  2738  0 Apr21 ?        00:01:23 /bin/sh /usr/local/nagioslogserver/scripts/create_backup.sh
apache   11539  1602  0 Apr26 ?        00:00:02 /usr/sbin/httpd
apache   11540  1602  0 Apr26 ?        00:00:02 /usr/sbin/httpd
apache   11541  1602  0 Apr26 ?        00:00:02 /usr/sbin/httpd
apache   11542  1602  0 Apr26 ?        00:00:02 /usr/sbin/httpd
apache   11543  1602  0 Apr26 ?        00:00:02 /usr/sbin/httpd
apache   11544  1602  0 Apr26 ?        00:00:02 /usr/sbin/httpd
apache   11545  1602  0 Apr26 ?        00:00:02 /usr/sbin/httpd
apache   11546  1602  0 Apr26 ?        00:00:02 /usr/sbin/httpd
root     20146  1613  0 Apr26 ?        00:00:00 CROND
nagios   20149 20146  0 Apr26 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php jobs > /usr/local/nagioslogserver/var/jobs.log 2>&1
nagios   20150 20149  0 Apr26 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php jobs
nagios   20187 20150  0 Apr26 ?        00:00:11 /bin/sh /usr/local/nagioslogserver/scripts/create_backup.sh
root     26830  1613  0 Apr25 ?        00:00:00 CROND
nagios   26832 26830  0 Apr25 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php jobs > /usr/local/nagioslogserver/var/jobs.log 2>&1
nagios   26833 26832  0 Apr25 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php jobs
nagios   26852 26833  0 Apr25 ?        00:00:25 /bin/sh /usr/local/nagioslogserver/scripts/create_backup.sh
root     37130  1613  0 Apr24 ?        00:00:00 CROND
nagios   37133 37130  0 Apr24 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php jobs > /usr/local/nagioslogserver/var/jobs.log 2>&1
nagios   37135 37133  0 Apr24 ?        00:00:01 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php jobs
nagios   37393 37135  0 Apr24 ?        00:00:40 /bin/sh /usr/local/nagioslogserver/scripts/create_backup.sh
root     46403  1613  0 Apr23 ?        00:00:00 CROND
nagios   46406 46403  0 Apr23 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php jobs > /usr/local/nagioslogserver/var/jobs.log 2>&1
nagios   46408 46406  0 Apr23 ?        00:00:01 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php jobs
nagios   46589 46408  0 Apr23 ?        00:00:53 /bin/sh /usr/local/nagioslogserver/scripts/create_backup.sh
root     55316  1613  0 Apr22 ?        00:00:00 CROND
nagios   55318 55316  0 Apr22 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php jobs > /usr/local/nagioslogserver/var/jobs.log 2>&1
nagios   55320 55318  0 Apr22 ?        00:00:01 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php jobs
nagios   55509 55320  0 Apr22 ?        00:01:07 /bin/sh /usr/local/nagioslogserver/scripts/create_backup.sh
root     63605  1613  0 09:36 ?        00:00:00 CROND
root     63606  1613  0 09:36 ?        00:00:00 CROND
nagios   63607 63606  0 09:36 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller > /usr/local/nagioslogserver/var/poller.log 2>&1
nagios   63608 63607  0 09:36 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php poller
nagios   63609 63605  0 09:36 ?        00:00:00 /bin/sh -c /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php jobs > /usr/local/nagioslogserver/var/jobs.log 2>&1
nagios   63610 63609  0 09:36 ?        00:00:00 /usr/bin/php -q /var/www/html/nagioslogserver/www/index.php jobs
root     63778  1506  0 09:36 ?        00:00:00 sshd: root@pts/0 
root     63780 63778  0 09:36 pts/0    00:00:00 -bash
nagios   63882 26852  0 09:36 ?        00:00:00 sleep 5
nagios   63883 55509  0 09:36 ?        00:00:00 sleep 5
nagios   63893 37393  0 09:36 ?        00:00:00 sleep 5
nagios   63894 20187  0 09:36 ?        00:00:00 sleep 5
nagios   63895 46589  0 09:36 ?        00:00:00 sleep 5
nagios   63900  2799  0 09:36 ?        00:00:00 sleep 5
root     63901 63780  0 09:36 pts/0    00:00:00 ps -ef
[root@IGAnagioslog ~]# 

Looks like some back as far as the 21st. Before a ran the 'reset jobs' that jolson request I run when this happens again, I figured I would post something in case someone wanted me to look at something else before trying the reset. I'm not seeing any poller jobs hanging around nor am I seeing any indication on the client systems yet. But it normally many more poller/jobs processes hanging would be hanging around before I would see the client issue.

--Mitch

[jolson]

Mitch,

Could you take a state of your NLS Cluster and send it to me? I have a couple of hunches about why this might be happening:

Code: Select all

curl -XGET 'http://localhost:9200/_cluster/state?pretty'

Please PM the above to me as it may contain sensitive information.