Nagios Support Forum

Hi Please can you help,

I upgraded Nagios from 3.4.5 to 4.0.8 earlier in the week and we are having problems that not all email notifications are being sent.

Our configuration sends all notification emails to a DUMMY contact defined in a template, then we use escalations to send all Critical emails to a central helpdesk. Our escalation file is below, this worked fine under 3.4.5

The nagios.log shows notifications are being raised but nothing is actually sent out. I have stopped/restart sendmail and sent test emails from the command prompt ok.


# 'notify-host-by-email' command definition
define command{
command_name notify-host-by-email
command_line /usr/bin/printf "%b" "***** Nagios *****\n\nNotification Type: $NOTIFICATIONTYPE$\nSERVICE: $HOSTALIAS$\nHost: $HOSTNAME$\nState: $HOSTSTATE$\nDescription: $HOSTALIAS$\nAddres
s: $HOSTADDRESS$\nInfo: $HOSTOUTPUT$\n\nDate/Time: $LONGDATETIME$\nAcknowledged by: $HOSTACKAUTHORALIAS$\nHUB: $_HOSTHUB$\n" | /bin/mail -s "** $NOTIFICATIONTYPE$ Host Alert: $HOSTNAME$ is $HOSTSTATE
$ **" $CONTACTEMAIL$
}

# 'notify-service-by-email' command definition
define command{
command_name notify-service-by-email
command_line /usr/bin/printf "%b" "***** Nagios *****\n\nNotification Type: $NOTIFICATIONTYPE$\n\nService: $SERVICEDESC$\nHost: $HOSTNAME$\nSERVICE: $HOSTALIAS$\nAddress: $HOSTADDRESS$\nSt
ate: $SERVICESTATE$\n\nDate/Time: $LONGDATETIME$\nAcknowledged by: $SERVICEACKAUTHORALIAS$\nAdditional Info:\n\n$SERVICEOUTPUT$" | /bin/mail -s "** $NOTIFICATIONTYPE$ Service Alert: $HOSTNAME$/$SERVI
CEDESC$ is $SERVICESTATE$ **" $CONTACTEMAIL$
}


Contact template
define contact{
name central-hub-contact ; The name of this contact template
host_notifications_enabled 1 ; Contact will recieve host notifications
service_notifications_enabled 1 ; Contact will recieve service notifications
service_notification_period 24x7 ; service notifications can be sent anytime
host_notification_period 24x7 ; host notifications can be sent anytime
service_notification_options w,u,c,r,f,s ; send notifications for all service states
host_notification_options d,u,r,f,s ; send notifications for all host states,
service_notification_commands service-email-pnp4n-int-en-central ; send service notifications via email
host_notification_commands host-email-pnp4n-int-en-central ; send host notifications via email
register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL CONTACT, JUST A TEMPLATE!
}

define contact{
contact_name steve
use central-hub-contact
alias Steve
service_notification_period 24x7
host_notification_period 24x7
service_notification_options w,u,c,r
host_notification_options d,u,r
host_notifications_enabled 1 ; Contact will recieve host notifications
service_notifications_enabled 1 ; Contact will recieve service notifications
email [email protected]
}


#### Central HUB notifications ####
define command{
command_name service-email-pnp4n-int-en-central
command_line /usr/local/nagios/libexec/pnp4n_send_service_mail.central.hub.pl \
-p "Nagios" \
-f multi -u
}


define command{
command_name host-email-pnp4n-int-en-central
command_line /usr/local/nagios/libexec/pnp4n_send_host_mail.central.hub.pl \
-p "Nagios" \
-f graph -u
}


Escalations
###############################################################
# The two definitions below override the default DUMMY contact.

# ALL CRITICAL SERVICE NOTIFICATIONS - UNIX/LINUX & STORAGE.
define serviceescalation{
hostgroup_name all-servers,!citrix-servers,!lic-servers-WIN,!xen-servers, \
!site1-all-servers, !site2-all-servers, \
!site3-all-servers, !site4-all-servers
service_description *,!arc,!pro_listener,!arc_listener, \
!pro_database,!tns-site1, \
!site1-DB
last_notification 750
notification_interval 5
escalation_options c,r
escalation_period workhours
contact_groups helpdesk,UnixAdmins
}

# ALL CRITICAL SERVICE NOTIFICATIONS - WINDOWS & CITRIX
define serviceescalation{
hostgroup_name citrix-servers,lic-servers-WIN,windows-servers,\
xen-servers,vmware-servers
service_description *
last_notification 750
notification_interval 5
escalation_options c,r
escalation_period workhours
contact_groups helpdesk,WindowsAdmins
}

############################################################
# The following two definitions send the first ten WARNING msgs
# to third line, then stop

# WARNING NOTIFICATIONS - LINUX - SERVICE ESCALATIONS
define serviceescalation{
hostgroup_name all-servers,!citrix-servers,!lic-servers-WIN,!xen-servers, \
!site1-all-servers, !site2-all-servers, \
!site3-all-servers, !site4-all-servers
service_description *,!Proc_Aff!var
first_notification 1
last_notification 10
notification_interval 10
escalation_options w
escalation_period workhours
contact_groups UnixAdmins
}
define serviceescalation{
hostgroup_name all-servers,!citrix-servers,!lic-servers-WIN,!xen-servers, \
!site1-all-servers, !site2-all-servers, \
!site3-all-servers, !site4-all-servers
service_description *,!Proc_Aff,!var
first_notification 10
last_notification 11
notification_interval 0
escalation_options w
escalation_period workhours
contact_groups UnixAdmins
}
# WARNING NOTIFICATIONS - WINDOWS - SERVICE ESCALATIONS
define serviceescalation{
hostgroup_name citrix-servers,tvp-lic-servers-WIN,windows-servers,\
lic-servers, xen-servers,vmware-servers, xen-tdp-servers, \
!site1-all-servers, !site2-all-servers, \
!site3-all-servers, !site4-all-servers
host_name node1.co.uk
service_description *,!Proc_Aff
first_notification 1
last_notification 10
notification_interval 10
escalation_options w
escalation_period workhours
contact_groups WindowsAdmins
}
define serviceescalation{
hostgroup_name citrix-servers,tvp-lic-servers-WIN,windows-servers,\
lic-servers, xen-servers,vmware-servers, xen-tdp-servers, \
!site1-all-servers, !site2-all-servers, \
!site3-all-servers, !site4-all-servers
host_name node1.co.uk
service_description *,!Proc_Aff
first_notification 10
last_notification 11
notification_interval 0
escalation_options w
escalation_period workhours
contact_groups WindowsAdmins
}

###########################################################
# The three definitions below escalate CRITICAL SERVICE alterts every 5mins
# through first,second and third line and onto Lee.

# NOTIFICATION1 - SERVICE ESCALATIONS
define serviceescalation{
hostgroup_name all-servers, \
!site1-all-servers, !site2-all-servers, \
!site3-all-servers, !site4-all-servers
service_description *
first_notification 2
last_notification 10
notification_interval 5
escalation_options c,r
escalation_period workhours
contact_groups helpdesk,secondline
}

Logfile snippet
[1426861007] SERVICE NOTIFICATION: user;server;Daemon;CRITICAL;service-email-pnp4n-int-en-easten;CRITICAL:Error getting status: Cannot connect to license server system.
[1426861007] SERVICE NOTIFICATION: user2;server;Daemon;CRITICAL;service-email-pnp4n-int-en-easten;CRITICAL:Error getting status: Cannot connect to license server system.

Is there anything valuable in your maillog tail?

Is this upgrade on the same server you were running on before? If not what happens when you try to just E-mail yourself with the following command:

Hi, Thanks for your quick reply

The maillog looks ok before/after the upgrade I can't see any errors or failures example ...

Mar 20 15:11:09 host-name sendmail[3508]: t2KFB8GJ003285: to=<email address>, delay=00:00:01, xdelay=00:00:00, mailer=relay, pri=125357, relay=relay.bgep.co.uk. [10.160.96.8], dsn=2.0.0, stat=Sent ( <20150 3201511.t2KFB8GJ003285@host-name> Queued mail for delivery)
Mar 20 15:11:09 host-name sendmail[3513]: t2KFB8QK003270: to=<email address>, delay=00:00:01, xdelay=00:00:00, mailer=relay, pri=125357, relay=relay.bgep.co.uk. [10.16.133.7], dsn=2.0.0, stat=Sent ( <20150 3201511.t2KFB8QK003270@host-name> Queued mail for delivery)

The upgrade was on the same server we were using before and the command /usr/bin/printf "%b" Test E-mail | /bin/mail -s "Test Subject" [email protected] - works correctly

thanks
Steve.

Hi, I logged a call earlier in the week about problems after the 4.0.8 upgrade and it was resolved by removing a wild card from one of the escalation definitions, the definition worked OK under 3.4.5.

Is the issue resolved?

Hi, we still have the problem there are no notifications being sent for critical or warning messages please can you look at this as soon as possible

Thanks
Steve.

Could you post how the command "service-email-pnp4n-int-en-easten" is defined?
Could you post how a contact and a service definition that you are hiving issues with so we can review them?

Hi,
Please see the nagios configuration details below, let me know if you need more info.

cat notifications.cfg
#### Central HUB notifications ####
define command{
command_name service-email-pnp4n-int-en-central
command_line /usr/local/nagios/libexec/pnp4n_send_service_mail.central.hub.pl \
-p "Central hub Nagios" \
-f multi -u
}

define command{
command_name host-email-pnp4n-int-en-central
command_line /usr/local/nagios/libexec/pnp4n_send_host_mail.central.hub.pl \
-p "Central hub Nagios" \
-f graph -u
}

cat contacts.cfg
define contact{
contact_name steve
use central-hub-contact
alias steveB
email [email protected]
}

cat services.cfg
# CHECK ROOT F/S
define service{
use generic-service,srv-pnp
hostgroup cluster-servers
service_description /root
notification_options w,c,r
check_command check_disk!check_root
}

We are using frank4dd's (http://nagios.frank4dd.com/) perl script to email notifications. We have three copies of the script central-hub, eastern-hub and western-hub the only difference is the email address in the script.

Thanks
Steve

cat services.cfg
# CHECK ROOT F/S
define service{
use generic-service,srv-pnp
hostgroup cluster-servers
service_description /root
notification_options w,c,r
check_command check_disk!check_root
}

Steve, can you also post the service templates (generic-service,srv-pnp)? The contact "steve" is not defined in the "/root" service definition... I don't see contacts/contactgroups defined, i.e. Is "steve" inherited from a template?

Hi, all our notifications are sent to the admins contact_group which is a dummy email address. The user steve is part of the LinuxAdmins contact group. The idea of our configuration is to use escalations to send all critical alert to the helpdesk email address and warning emails to different contact groups.

Do you have an ftp address I could tar up and send our cfg files to you which might be easier?

Thanks
Steve.


define service{
name generic-service ; The 'name' of this service template
active_checks_enabled 1 ; Active service checks are enabled
#passive_checks_enabled 1 ; Passive service checks are enabled/accepted
parallelize_check 1 ; Active service checks should be parallelized (disabling this can lead to
major performance problems)
obsess_over_service 1 ; We should obsess over this service (if necessary)
check_freshness 0 ; Default is to NOT check service 'freshness'
notifications_enabled 1 ; Service notifications are enabled
event_handler_enabled 1 ; Service event handler is enabled
flap_detection_enabled 1 ; Flap detection is enabled
#failure_prediction_enabled 1 ; Failure prediction is enabled
process_perf_data 1 ; Process performance data
retain_status_information 1 ; Retain status information across program restarts
retain_nonstatus_information 1 ; Retain non-status information across program restarts
is_volatile 0 ; The service is not volatile
check_period 24x7 ; The service can be checked at any time of the day
max_check_attempts 2 ; Re-check the service up to 2 times in order to determine its final (hard)
state
check_interval 1 ; Check the service every 30 seconds under normal conditions
retry_check_interval 1 ; Re-check the service every 30 seconds until a hard state can be determine
d
notification_options w,u,c,r ; Send notifications about warning, unknown, critical, and recovery events
notification_interval 10 ; Re-notify about service problems 5 mins
notification_period 24x7 ; Notifications can be sent out at any time
contact_groups admins
register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL SERVICE, JUST A TEMPLATE!
process_perf_data 1
}

define service {
name srv-pnp
# action_url /pnp4nagios/share/index.php?host=$HOSTNAME$&srv=$SERVICEDESC$
action_url /pnp4nagios/graph?host=$HOSTNAME$&srv=$SERVICEDESC$
register 0
}


define contact{
name generic-contact ; The name of this contact template
host_notifications_enabled 1 ; Contact will recieve host notifications
service_notifications_enabled 1 ; Contact will recieve service notifications
service_notification_period 24x7 ; service notifications can be sent anytime
host_notification_period 24x7 ; host notifications can be sent anytime
service_notification_options w,u,c,r,f,s ; send notifications for all service states
host_notification_options d,u,r,f,s ; send notifications for all host states,
service_notification_commands notify-service-by-email ; send service notifications via email
host_notification_commands notify-host-by-email ; send host notifications via email
register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL CONTACT, JUST A TEMPLATE!
}

define contact{
name central-hub-contact ; The name of this contact template
host_notifications_enabled 1 ; Contact will recieve host notifications
service_notifications_enabled 1 ; Contact will recieve service notifications
service_notification_period 24x7 ; service notifications can be sent anytime
host_notification_period 24x7 ; host notifications can be sent anytime
service_notification_options w,u,c,r,f,s ; send notifications for all service states
host_notification_options d,u,r,f,s ; send notifications for all host states,
service_notification_commands service-email-pnp4n-int-en-central ; send service notifications via email
host_notification_commands host-email-pnp4n-int-en-central ; send host notifications via email
register 0 ; DONT REGISTER THIS DEFINITION - ITS NOT A REAL CONTACT, JUST A TEMPLATE!
}