Zookeeper script throwing an error

[shamrozkadiwal]

Hey All,

I have setup some services to monitor zookeeper and have been getting an error as "(Return code of 127 is out of bounds - plugin may be missing) " This error doesn't make sense because script is there for sure. In addition, nrpe client on that server is running because other top 4 services are green. Please let me to fix this issue.

FYI, I am following this link to setup zookeeper services

http://svn.apache.org/repos/asf/zookeep ... onitoring/
Thanks,

[tgriep]

Did you create a command definition in the nrpe.cfg file for the Zookeeper script?
Can you post the command that you defined in the nrpe.cfg file as well as the service check that is defined on the Nagios server so we can view the settings?

Thanks.

[shamrozkadiwal]

@tgriep, you mean command.cfg file where all the commands are defined.

/etc/nagios/commands/command.cfg

Code: Select all

define command
        command_name check_zookeeper
        command_line /usr/lib/nagios/plugins/check_zookeeper_apache.py -s $HOSTADDRESS$:4150 -o nagios -k '$ARG1$' -w '$ARG2$' -c '$ARG3$'
}

/etc/nagios/services/services.solr.cfg

Code: Select all

define service {
        hostgroup_name          zookeeper-servers
        use                     gen-solr-svc
        servicegroups           zookeeper-services
        service_description     ZK_Open_File_Descriptors_Count
        check_command           check_zookeeper!zk_open_file_descriptor_count!500!800
}

define service {
        hostgroup_name          zookeeper-servers
        use                     gen-solr-svc
        servicegroups           zookeeper-services
        service_description     ZK_Ephemerals_Count
        check_command           check_zookeeper!zk_ephemerals_count!10000!100000
}

define service {
        hostgroup_name          zookeeper-servers
        use                     gen-solr-svc
        servicegroups           zookeeper-services
        service_description     ZK_Avg_Latency
        check_command           check_zookeeper!zk_avg_latency!500!1000
}

define service {
        hostgroup_name          zookeeper-servers
        use                     gen-solr-svc
        servicegroups           zookeeper-services
        service_description     ZK_Max_Latency
        check_command           check_zookeeper!zk_max_latency!1000!2000
}

define service {
        hostgroup_name          zookeeper-servers
        use                     gen-solr-svc
        servicegroups           zookeeper-services
        service_description     ZK_Min_Latency
        check_command           check_zookeeper!zk_min_latency!500!1000
}

define service {
        hostgroup_name          zookeeper-servers
        use                     gen-solr-svc
        servicegroups           zookeeper-services
        service_description     ZK_Outstanding_Requests
        check_command           check_zookeeper!zk_outstanding_requests!20!50
}

define service {
        hostgroup_name          zookeeper-servers
        use                     gen-solr-svc
        servicegroups           zookeeper-services
        service_description     ZK_Watch_Count
        check_command           check_zookeeper!zk_watch_count!100!500
}

[tgriep]

Can you run the plugin on the Nagios server while you are logged in to it as the nagios user?

Run this to change to the nagios user

Code: Select all

su - nagios

Then run the plugin, replace xxx.xxx.xxx.xxx with the IP address of the Zookeeper server.

Code: Select all

/usr/lib/nagios/plugins/check_zookeeper.py -s xxx.xxx.xxx.xxx:80 -o nagios -k zk_open_file_descriptor_count -w 500 -c 800

Post the full output.

In your first post, you mentioned NRPE, is the plugin suppose to run on the remote server using NRPE?
I just want to clarify it.

[shamrozkadiwal]

@tgriep, I turned into nagios user and ran below commands

Code: Select all

nagios@camel / $ /usr/lib/nagios/plugins/check_zookeeper_apache.py -s 10.180.130.76:80 -o nagios -k zk_open_file_descriptor_count -w 500 -c 800
: No such file or directory
nagios@camel / $
nagios@camel / $ ls -al /usr/lib/nagios/plugins/check_zookeeper_apache.py
-rwxr-xr-x 1 nagios nagios 11352 Apr 17 07:58 /usr/lib/nagios/plugins/check_zookeeper_apache.py
nagios@camel / $

[tgriep]

In the top of the script, it calls the env application to create is own environment.
This is the line from the script.

Code: Select all

#! /usr/bin/env python

Is the env application in the /usr/bin folder?
If it is in a different folder, update the script to the correct path and see if the plugin runs.

If it still fails, run the script with python in verbose mode and post the output to see that the error is.

Code: Select all

python -v /usr/lib/nagios/plugins/check_zookeeper_apache.py

[shamrozkadiwal]

Code: Select all

camel bin # ls -al python
lrwxrwxrwx 1 root root 14 Dec 14  2016 python -> python-wrapper

camel bin #  pwd
/usr/bin

Code: Select all

camel bin #
camel bin # python -v /usr/lib/nagios/plugins/check_zookeeper_apache.py
# installing zipimport hook
import zipimport # builtin
# installed zipimport hook
# /usr/lib/python2.6/site.pyc matches /usr/lib/python2.6/site.py
import site # precompiled from /usr/lib/python2.6/site.pyc
# /usr/lib/python2.6/os.pyc matches /usr/lib/python2.6/os.py
import os # precompiled from /usr/lib/python2.6/os.pyc
import errno # builtin
import posix # builtin
# /usr/lib/python2.6/posixpath.pyc matches /usr/lib/python2.6/posixpath.py
import posixpath # precompiled from /usr/lib/python2.6/posixpath.pyc
# /usr/lib/python2.6/stat.pyc matches /usr/lib/python2.6/stat.py
import stat # precompiled from /usr/lib/python2.6/stat.pyc
# /usr/lib/python2.6/genericpath.pyc matches /usr/lib/python2.6/genericpath.py
import genericpath # precompiled from /usr/lib/python2.6/genericpath.pyc
# /usr/lib/python2.6/warnings.pyc matches /usr/lib/python2.6/warnings.py
import warnings # precompiled from /usr/lib/python2.6/warnings.pyc
# /usr/lib/python2.6/linecache.pyc matches /usr/lib/python2.6/linecache.py
import linecache # precompiled from /usr/lib/python2.6/linecache.pyc
# /usr/lib/python2.6/types.pyc matches /usr/lib/python2.6/types.py
import types # precompiled from /usr/lib/python2.6/types.pyc
# /usr/lib/python2.6/UserDict.pyc matches /usr/lib/python2.6/UserDict.py
import UserDict # precompiled from /usr/lib/python2.6/UserDict.pyc
# /usr/lib/python2.6/_abcoll.pyc matches /usr/lib/python2.6/_abcoll.py
import _abcoll # precompiled from /usr/lib/python2.6/_abcoll.pyc
# /usr/lib/python2.6/abc.pyc matches /usr/lib/python2.6/abc.py
import abc # precompiled from /usr/lib/python2.6/abc.pyc
# /usr/lib/python2.6/copy_reg.pyc matches /usr/lib/python2.6/copy_reg.py
import copy_reg # precompiled from /usr/lib/python2.6/copy_reg.pyc
import encodings # directory /usr/lib/python2.6/encodings
# /usr/lib/python2.6/encodings/__init__.pyc matches /usr/lib/python2.6/encodings/__init__.py
import encodings # precompiled from /usr/lib/python2.6/encodings/__init__.pyc
# /usr/lib/python2.6/codecs.pyc matches /usr/lib/python2.6/codecs.py
import codecs # precompiled from /usr/lib/python2.6/codecs.pyc
import _codecs # builtin
# /usr/lib/python2.6/encodings/aliases.pyc matches /usr/lib/python2.6/encodings/aliases.py
import encodings.aliases # precompiled from /usr/lib/python2.6/encodings/aliases.pyc
# /usr/lib/python2.6/encodings/ascii.pyc matches /usr/lib/python2.6/encodings/ascii.py
import encodings.ascii # precompiled from /usr/lib/python2.6/encodings/ascii.pyc
Python 2.6.4 (r264:75706, Nov  8 2010, 19:22:23)
[GCC 4.1.2 (Gentoo 4.1.2 p1.3)] on linux2
Type "help", "copyright", "credits" or "license" for more information.
# /usr/lib/python2.6/socket.pyc matches /usr/lib/python2.6/socket.py
import socket # precompiled from /usr/lib/python2.6/socket.pyc
dlopen("/usr/lib/python2.6/lib-dynload/_socket.so", 2);
import _socket # dynamically loaded from /usr/lib/python2.6/lib-dynload/_socket.so
dlopen("/usr/lib/python2.6/lib-dynload/_ssl.so", 2);
import _ssl # dynamically loaded from /usr/lib/python2.6/lib-dynload/_ssl.so
dlopen("/usr/lib/python2.6/lib-dynload/cStringIO.so", 2);
import cStringIO # dynamically loaded from /usr/lib/python2.6/lib-dynload/cStringIO.so
import logging # directory /usr/lib/python2.6/logging
# /usr/lib/python2.6/logging/__init__.pyc matches /usr/lib/python2.6/logging/__init__.py
import logging # precompiled from /usr/lib/python2.6/logging/__init__.pyc
dlopen("/usr/lib/python2.6/lib-dynload/time.so", 2);
import time # dynamically loaded from /usr/lib/python2.6/lib-dynload/time.so
# /usr/lib/python2.6/string.pyc matches /usr/lib/python2.6/string.py
import string # precompiled from /usr/lib/python2.6/string.pyc
# /usr/lib/python2.6/re.pyc matches /usr/lib/python2.6/re.py
import re # precompiled from /usr/lib/python2.6/re.pyc
# /usr/lib/python2.6/sre_compile.pyc matches /usr/lib/python2.6/sre_compile.py
import sre_compile # precompiled from /usr/lib/python2.6/sre_compile.pyc
import _sre # builtin
# /usr/lib/python2.6/sre_parse.pyc matches /usr/lib/python2.6/sre_parse.py
import sre_parse # precompiled from /usr/lib/python2.6/sre_parse.pyc
# /usr/lib/python2.6/sre_constants.pyc matches /usr/lib/python2.6/sre_constants.py
import sre_constants # precompiled from /usr/lib/python2.6/sre_constants.pyc
dlopen("/usr/lib/python2.6/lib-dynload/strop.so", 2);
import strop # dynamically loaded from /usr/lib/python2.6/lib-dynload/strop.so
# /usr/lib/python2.6/traceback.pyc matches /usr/lib/python2.6/traceback.py
import traceback # precompiled from /usr/lib/python2.6/traceback.pyc
import thread # builtin
# /usr/lib/python2.6/threading.pyc matches /usr/lib/python2.6/threading.py
import threading # precompiled from /usr/lib/python2.6/threading.pyc
# /usr/lib/python2.6/functools.pyc matches /usr/lib/python2.6/functools.py
import functools # precompiled from /usr/lib/python2.6/functools.pyc
dlopen("/usr/lib/python2.6/lib-dynload/_functools.so", 2);
import _functools # dynamically loaded from /usr/lib/python2.6/lib-dynload/_functools.so
# /usr/lib/python2.6/collections.pyc matches /usr/lib/python2.6/collections.py
import collections # precompiled from /usr/lib/python2.6/collections.pyc
dlopen("/usr/lib/python2.6/lib-dynload/_collections.so", 2);
import _collections # dynamically loaded from /usr/lib/python2.6/lib-dynload/_collections.so
dlopen("/usr/lib/python2.6/lib-dynload/operator.so", 2);
import operator # dynamically loaded from /usr/lib/python2.6/lib-dynload/operator.so
# /usr/lib/python2.6/keyword.pyc matches /usr/lib/python2.6/keyword.py
import keyword # precompiled from /usr/lib/python2.6/keyword.pyc
# /usr/lib/python2.6/atexit.pyc matches /usr/lib/python2.6/atexit.py
import atexit # precompiled from /usr/lib/python2.6/atexit.pyc
# /usr/lib/python2.6/subprocess.pyc matches /usr/lib/python2.6/subprocess.py
import subprocess # precompiled from /usr/lib/python2.6/subprocess.pyc
import gc # builtin
dlopen("/usr/lib/python2.6/lib-dynload/select.so", 2);
import select # dynamically loaded from /usr/lib/python2.6/lib-dynload/select.so
dlopen("/usr/lib/python2.6/lib-dynload/fcntl.so", 2);
import fcntl # dynamically loaded from /usr/lib/python2.6/lib-dynload/fcntl.so
# /usr/lib/python2.6/pickle.pyc matches /usr/lib/python2.6/pickle.py
import pickle # precompiled from /usr/lib/python2.6/pickle.pyc
import marshal # builtin
# /usr/lib/python2.6/struct.pyc matches /usr/lib/python2.6/struct.py
import struct # precompiled from /usr/lib/python2.6/struct.pyc
dlopen("/usr/lib/python2.6/lib-dynload/_struct.so", 2);
import _struct # dynamically loaded from /usr/lib/python2.6/lib-dynload/_struct.so
dlopen("/usr/lib/python2.6/lib-dynload/binascii.so", 2);
import binascii # dynamically loaded from /usr/lib/python2.6/lib-dynload/binascii.so
# /usr/lib/python2.6/StringIO.pyc matches /usr/lib/python2.6/StringIO.py
import StringIO # precompiled from /usr/lib/python2.6/StringIO.pyc
# /usr/lib/python2.6/optparse.pyc matches /usr/lib/python2.6/optparse.py
import optparse # precompiled from /usr/lib/python2.6/optparse.pyc
# /usr/lib/python2.6/textwrap.pyc matches /usr/lib/python2.6/textwrap.py
import textwrap # precompiled from /usr/lib/python2.6/textwrap.pyc
# /usr/lib/python2.6/gettext.pyc matches /usr/lib/python2.6/gettext.py
import gettext # precompiled from /usr/lib/python2.6/gettext.pyc
# /usr/lib/python2.6/locale.pyc matches /usr/lib/python2.6/locale.py
import locale # precompiled from /usr/lib/python2.6/locale.pyc
dlopen("/usr/lib/python2.6/lib-dynload/_locale.so", 2);
import _locale # dynamically loaded from /usr/lib/python2.6/lib-dynload/_locale.so
# /usr/lib/python2.6/copy.pyc matches /usr/lib/python2.6/copy.py
import copy # precompiled from /usr/lib/python2.6/copy.pyc
Usage: ./check_zookeeper.py <options>

check_zookeeper_apache.py: error: The list of servers is mandatory
# clear __builtin__._
# clear sys.path
# clear sys.argv
# clear sys.ps1
# clear sys.ps2
# clear sys.exitfunc
# clear sys.exc_type
# clear sys.exc_value
# clear sys.exc_traceback
# clear sys.last_type
# clear sys.last_value
# clear sys.last_traceback
# clear sys.path_hooks
# clear sys.path_importer_cache
# clear sys.meta_path
# clear sys.flags
# clear sys.float_info
# restore sys.stdin
# restore sys.stdout
# restore sys.stderr
# cleanup __main__
# cleanup[1] _collections
# cleanup[1] site
# cleanup[1] subprocess
# cleanup[1] select
# cleanup[1] gc
# cleanup[1] keyword
# cleanup[1] abc
# cleanup[1] gettext
# cleanup[1] sre_constants
# cleanup[1] collections
# cleanup[1] _codecs
# cleanup[1] _struct
# cleanup[1] _warnings
# cleanup[1] fcntl
# cleanup[1] zipimport
# cleanup[1] optparse
# cleanup[1] textwrap
# cleanup[1] copy
# cleanup[1] strop
# cleanup[1] _functools
# cleanup[1] _locale
# cleanup[1] logging
# cleanup[1] socket
# cleanup[1] thread
# cleanup[1] StringIO
# cleanup[1] pickle
# cleanup[1] signal
# cleanup[1] traceback
# cleanup[1] posix
# cleanup[1] exceptions
# cleanup[1] marshal
# cleanup[1] cStringIO
# cleanup[1] atexit
# cleanup[1] _ssl
# cleanup[1] locale
# cleanup[1] threading
# cleanup[1] _socket
# cleanup[1] binascii
# cleanup[1] string
# cleanup[1] struct
# cleanup[1] functools
# cleanup[1] encodings
# cleanup[1] operator
# cleanup[1] re
# cleanup[1] encodings.ascii
# cleanup[1] codecs
# cleanup[1] encodings.aliases
# cleanup[1] sre_compile
# cleanup[1] _sre
# cleanup[1] sre_parse
# cleanup[2] copy_reg
# cleanup[2] posixpath
# cleanup[2] errno
# cleanup[2] _abcoll
# cleanup[2] types
# cleanup[2] genericpath
# cleanup[2] stat
# cleanup[2] warnings
# cleanup[2] UserDict
# cleanup[2] os.path
# cleanup[2] linecache
# cleanup[2] time
# cleanup[2] os
# cleanup sys
# cleanup __builtin__
# cleanup ints: 26 unfreed ints
# cleanup floats: 2 unfreed floats
camel bin #

[tgriep]

So far, everything looks OK so I am going to ask some basic information about the Nagios system.
What version of Nagios Core are you running on the server?
What OS and release is the server running?

Login to the Nagios server as root, run all of the following commands and post the /tmp/info.txt file here so we can view it.

Code: Select all

chage -l nagios >/tmp/info.txt
grep nag /etc/group >>/tmp/info.txt
ls -alR /usr/lib/nagios >>/tmp/info.txt
ls -alR /etc/nagios >>/tmp/info.txt
ls -al /usr/bin/env >>/tmp/info.txt
which env >>/tmp/info.txt
ps -ef --cols=300 >>/tmp/info.txt
grep -R -i zoo /etc/nagios/* >>/tmp/info.txt

Thanks.

[shamrozkadiwal]

I have attached info.txt file

[tgriep]

Edit the following file

Code: Select all

/etc/group

change this from

Code: Select all

nagios:x:440:apache,rastew

to

Code: Select all

nagios:x:440:nagios,apache,rastew

Save it

The plugins folder is soft linked to another location so I will need you to run this and post the /tmp/info.txt file here so we can view it.

Code: Select all

ls -alR /usr/nagios/libexec/ >/tmp/info.txt

In the info.txt file, I see this command. Could this one be causing the error?

Code: Select all

/etc/nagios/objects/commands.cfg:	command_name    check_zookeeper
/etc/nagios/objects/commands.cfg:	command_line    $USER1$/check_plugin.pl $USER1$/check_zookeeper.pl -H $HOSTADDRESS$ -P 4150

Can you post this file from the server?

Code: Select all

/etc/nagios/nagios.cfg

Thanks.