Increasing Inode size continuously.

[sandeepatil]

Need your help to resolve below issue.

Nagios Core installed partition inode size insceasing continuously.

[scottwilkerson]

Can you share your nagios.cfg ?

I have a feeling you are writing performance data files somewhere and not having something reap them

[sandeepatil]

Please find below are nagios.cfg details.

Code: Select all

#admin_pager=root@localhost
translate_passive_host_checks=1
log_event_handlers=0
use_large_installation_tweaks=1
enable_environment_macros=0


# NDOUtils module
#broker_module=/opt/app/nagios/bin/ndomod.o config_file=/opt/app/nagios/etc/ndomod.cfg
broker_module=/opt/app/nagios/mk-livestatus/lib/mk-livestatus/livestatus.o /opt/app/nagios/var/rw/live
# PNP settings - bulk mode with NCPD
process_performance_data=1

host_perfdata_file=/opt/app/nagios/pnp4nagios/var/host-perfdata
service_perfdata_file=/opt/app/nagios/pnp4nagios/var/service-perfdata

host_perfdata_file_template=DATATYPE::HOSTPERFDATA\tTIMET::$TIMET$\tHOSTNAME::$HOSTNAME$\tHOSTPERFDATA::$HOSTPERFDATA$\tHOSTCHECKCOMMAND::$HOSTCHECKCOMMAND$\tHOSTSTATE::$HOSTSTATE$\tHOSTSTATETYPE::$HOSTSTATETYPE$\tHOSTOUTPUT::$HOSTOUTPUT$

service_perfdata_file_template=DATATYPE::SERVICEPERFDATA\tTIMET::$TIMET$\tHOSTNAME::$HOSTNAME$\tSERVICEDESC::$SERVICEDESC$\tSERVICEPERFDATA::$SERVICEPERFDATA$\tSERVICECHECKCOMMAND::$SERVICECHECKCOMMAND$\tHOSTSTATE::$HOSTSTATE$\tHOSTSTATETYPE::$HOSTSTATETYPE$\tSERVICESTATE::$SERVICESTATE$\tSERVICESTATETYPE::$SERVICESTATETYPE$\tSERVICEOUTPUT::$SERVICEOUTPUT$

host_perfdata_file_mode=a
service_perfdata_file_mode=a

host_perfdata_file_processing_interval=15
service_perfdata_file_processing_interval=15

host_perfdata_file_processing_command=process-host-perfdata-file
service_perfdata_file_processing_command=process-service-perfdata-file

#service_perfdata_file_processing_command=process-service-perfdata-file-bulk-npcd
#host_perfdata_file_processing_command=process-host-perfdata-file-bulk-npcd

# OBJECTS - UNMODIFIED
#cfg_file=/opt/app/nagios/etc/objects/commands.cfg
#cfg_file=/opt/app/nagios/etc/objects/contacts.cfg
#cfg_file=/opt/app/nagios/etc/objects/localhost.cfg
#cfg_file=/opt/app/nagios/etc/objects/templates.cfg
#cfg_file=/opt/app/nagios/etc/objects/timeperiods.cfg

# STATIC OBJECT DEFINITIONS (THESE DON'T GET EXPORTED/IMPORTED BY NAGIOSQL)

# OBJECTS EXPORTED FROM NAGIOSQL
cfg_file=/opt/app/nagios/etc/contactgroups.cfg
cfg_file=/opt/app/nagios/etc/contacts.cfg
cfg_file=/opt/app/nagios/etc/timeperiods.cfg
cfg_file=/opt/app/nagios/etc/commands.cfg
cfg_file=/opt/app/nagios/etc/templates.cfg

cfg_dir=/opt/app/nagios/etc/hosts
cfg_dir=/opt/app/nagios/etc/services




# UNMODIFIED
accept_passive_host_checks=1
accept_passive_service_checks=1
additional_freshness_latency=15
auto_reschedule_checks=0
auto_rescheduling_interval=30
auto_rescheduling_window=45
bare_update_check=0
cached_host_check_horizon=15
cached_service_check_horizon=15
check_external_commands=1
check_for_orphaned_hosts=1
check_for_orphaned_services=1
check_for_updates=0
check_host_freshness=1
check_result_path=/opt/app/nagios/var/spool/checkresults
check_result_reaper_frequency=10
check_service_freshness=1
#command_check_interval=1
command_file=/opt/app/nagios/var/rw/nagios.cmd
daemon_dumps_core=0
date_format=us
debug_file=/opt/app/nagios/var/nagios.debug
debug_level=0
debug_verbosity=1
#enable_embedded_perl=1
enable_event_handlers=1
enable_flap_detection=1
enable_notifications=1
enable_predictive_host_dependency_checks=1
enable_predictive_service_dependency_checks=1
event_broker_options=-1
event_handler_timeout=30
execute_host_checks=1
execute_service_checks=1
#external_command_buffer_slots=4096
high_host_flap_threshold=20.0
high_service_flap_threshold=20.0
host_check_timeout=30
host_freshness_check_interval=60
host_inter_check_delay_method=s
illegal_macro_output_chars=`~$&|'"<>
illegal_object_name_chars=`~!$%^&*|'"<>?,()=
interval_length=60
lock_file=/var/run/nagios.lock
log_archive_path=/opt/app/nagios/var/archives
log_external_commands=0
log_file=/opt/app/nagios/var/nagios.log
log_host_retries=1
log_initial_states=0
log_notifications=1
log_passive_checks=0
log_rotation_method=d
log_service_retries=1
low_host_flap_threshold=5.0
low_service_flap_threshold=5.0
max_check_result_file_age=3600
max_check_result_reaper_time=30
max_concurrent_checks=0
max_debug_file_size=1000000
max_host_check_spread=30
max_service_check_spread=30
nagios_group=nagios
nagios_user=nagios
notification_timeout=30
object_cache_file=/opt/app/nagios/var/objects.cache
obsess_over_hosts=0
obsess_over_services=0
ocsp_timeout=5
passive_host_checks_are_soft=0
perfdata_timeout=5
precached_object_file=/opt/app/nagios/var/objects.precache
resource_file=/opt/app/nagios/etc/resource.cfg
retained_contact_host_attribute_mask=0
retained_contact_service_attribute_mask=0
retained_host_attribute_mask=0
retained_process_host_attribute_mask=0
retained_process_service_attribute_mask=0
retained_service_attribute_mask=0
retain_state_information=1
retention_update_interval=60
service_check_timeout=60
service_freshness_check_interval=60
service_inter_check_delay_method=s
service_interleave_factor=s
soft_state_dependencies=0
state_retention_file=/opt/app/nagios/var/retention.dat
status_file=/opt/app/nagios/var/status.dat
status_update_interval=10
temp_file=/opt/app/nagios/var/nagios.tmp
temp_path=/tmp
use_aggressive_host_checking=0
#use_embedded_perl_implicitly=1
use_regexp_matching=0
use_retained_program_state=1
use_retained_scheduling_info=1
use_syslog=0
use_true_regexp_matching=0

[scottwilkerson]

Can you also share the command definitions for these commands

Code: Select all

process-host-perfdata-file
process-service-perfdata-file

[sandeepatil]

Please find command definition,

Code: Select all

define command{
       command_name    process-service-perfdata-file
       command_line    /bin/mv /opt/app/nagios/pnp4nagios/var/service-perfdata /opt/app/nagios/pnp4nagios/var/spool/service-perfdata.$TIMET$
}

define command{
       command_name    process-host-perfdata-file
       command_line    /bin/mv /opt/app/nagios/pnp4nagios/var/host-perfdata /opt/app/nagios/pnp4nagios/var/spool/host-perfdata.$TIMET$
}

[scottwilkerson]

Ok, so do you have something reaping the filed out of this directory? like npcd?

Code: Select all

/opt/app/nagios/pnp4nagios/var/spool

What is the output of this command

Code: Select all

ls /opt/app/nagios/pnp4nagios/var/spool|wc -l

[sandeepatil]

Now issue under control, I have stopped httpd and nagios service, delete checkresults directory and created new.

But this issue occurring repetitively. How to avoid and monitor before occurring.

Code: Select all

# ls /opt/app/nagios/pnp4nagios/var/spool|wc -l
18

[scottwilkerson]

Oh the files are in the checkresults directory?

Is the path you cleared /opt/app/nagios/var/spool/checkresults ?

[sandeepatil]

Please check below status and share solution for Inode full issue.

Code: Select all

# ll /opt/app/nagios/var/spool/checkresults | wc -l
2616
# ll /opt/app/nagios/var/spool/checkresults | wc -l
3252
# ll /opt/app/nagios/var/spool/checkresults | wc -l
633
# ll /opt/app/nagios/var/spool/checkresults | wc -l
1081
# ll /opt/app/nagios/var/spool/checkresults | wc -l
2083

[scottwilkerson]

Well this looks like it is fluctuating up and down which it should if you have a lot of passive results coming in.