Page 1 of 2
Increasing Inode size continuously.
Posted: Tue Mar 05, 2019 12:04 pm
by sandeepatil
Need your help to resolve below issue.
Nagios Core installed partition inode size insceasing continuously.
Re: Increasing Inode size continuously.
Posted: Wed Mar 06, 2019 5:16 pm
by scottwilkerson
Can you share your nagios.cfg ?
I have a feeling you are writing performance data files somewhere and not having something reap them
Re: Increasing Inode size continuously.
Posted: Sun Mar 10, 2019 1:48 pm
by sandeepatil
Please find below are nagios.cfg details.
Code: Select all
#admin_pager=root@localhost
translate_passive_host_checks=1
log_event_handlers=0
use_large_installation_tweaks=1
enable_environment_macros=0
# NDOUtils module
#broker_module=/opt/app/nagios/bin/ndomod.o config_file=/opt/app/nagios/etc/ndomod.cfg
broker_module=/opt/app/nagios/mk-livestatus/lib/mk-livestatus/livestatus.o /opt/app/nagios/var/rw/live
# PNP settings - bulk mode with NCPD
process_performance_data=1
host_perfdata_file=/opt/app/nagios/pnp4nagios/var/host-perfdata
service_perfdata_file=/opt/app/nagios/pnp4nagios/var/service-perfdata
host_perfdata_file_template=DATATYPE::HOSTPERFDATA\tTIMET::$TIMET$\tHOSTNAME::$HOSTNAME$\tHOSTPERFDATA::$HOSTPERFDATA$\tHOSTCHECKCOMMAND::$HOSTCHECKCOMMAND$\tHOSTSTATE::$HOSTSTATE$\tHOSTSTATETYPE::$HOSTSTATETYPE$\tHOSTOUTPUT::$HOSTOUTPUT$
service_perfdata_file_template=DATATYPE::SERVICEPERFDATA\tTIMET::$TIMET$\tHOSTNAME::$HOSTNAME$\tSERVICEDESC::$SERVICEDESC$\tSERVICEPERFDATA::$SERVICEPERFDATA$\tSERVICECHECKCOMMAND::$SERVICECHECKCOMMAND$\tHOSTSTATE::$HOSTSTATE$\tHOSTSTATETYPE::$HOSTSTATETYPE$\tSERVICESTATE::$SERVICESTATE$\tSERVICESTATETYPE::$SERVICESTATETYPE$\tSERVICEOUTPUT::$SERVICEOUTPUT$
host_perfdata_file_mode=a
service_perfdata_file_mode=a
host_perfdata_file_processing_interval=15
service_perfdata_file_processing_interval=15
host_perfdata_file_processing_command=process-host-perfdata-file
service_perfdata_file_processing_command=process-service-perfdata-file
#service_perfdata_file_processing_command=process-service-perfdata-file-bulk-npcd
#host_perfdata_file_processing_command=process-host-perfdata-file-bulk-npcd
# OBJECTS - UNMODIFIED
#cfg_file=/opt/app/nagios/etc/objects/commands.cfg
#cfg_file=/opt/app/nagios/etc/objects/contacts.cfg
#cfg_file=/opt/app/nagios/etc/objects/localhost.cfg
#cfg_file=/opt/app/nagios/etc/objects/templates.cfg
#cfg_file=/opt/app/nagios/etc/objects/timeperiods.cfg
# STATIC OBJECT DEFINITIONS (THESE DON'T GET EXPORTED/IMPORTED BY NAGIOSQL)
# OBJECTS EXPORTED FROM NAGIOSQL
cfg_file=/opt/app/nagios/etc/contactgroups.cfg
cfg_file=/opt/app/nagios/etc/contacts.cfg
cfg_file=/opt/app/nagios/etc/timeperiods.cfg
cfg_file=/opt/app/nagios/etc/commands.cfg
cfg_file=/opt/app/nagios/etc/templates.cfg
cfg_dir=/opt/app/nagios/etc/hosts
cfg_dir=/opt/app/nagios/etc/services
# UNMODIFIED
accept_passive_host_checks=1
accept_passive_service_checks=1
additional_freshness_latency=15
auto_reschedule_checks=0
auto_rescheduling_interval=30
auto_rescheduling_window=45
bare_update_check=0
cached_host_check_horizon=15
cached_service_check_horizon=15
check_external_commands=1
check_for_orphaned_hosts=1
check_for_orphaned_services=1
check_for_updates=0
check_host_freshness=1
check_result_path=/opt/app/nagios/var/spool/checkresults
check_result_reaper_frequency=10
check_service_freshness=1
#command_check_interval=1
command_file=/opt/app/nagios/var/rw/nagios.cmd
daemon_dumps_core=0
date_format=us
debug_file=/opt/app/nagios/var/nagios.debug
debug_level=0
debug_verbosity=1
#enable_embedded_perl=1
enable_event_handlers=1
enable_flap_detection=1
enable_notifications=1
enable_predictive_host_dependency_checks=1
enable_predictive_service_dependency_checks=1
event_broker_options=-1
event_handler_timeout=30
execute_host_checks=1
execute_service_checks=1
#external_command_buffer_slots=4096
high_host_flap_threshold=20.0
high_service_flap_threshold=20.0
host_check_timeout=30
host_freshness_check_interval=60
host_inter_check_delay_method=s
illegal_macro_output_chars=`~$&|'"<>
illegal_object_name_chars=`~!$%^&*|'"<>?,()=
interval_length=60
lock_file=/var/run/nagios.lock
log_archive_path=/opt/app/nagios/var/archives
log_external_commands=0
log_file=/opt/app/nagios/var/nagios.log
log_host_retries=1
log_initial_states=0
log_notifications=1
log_passive_checks=0
log_rotation_method=d
log_service_retries=1
low_host_flap_threshold=5.0
low_service_flap_threshold=5.0
max_check_result_file_age=3600
max_check_result_reaper_time=30
max_concurrent_checks=0
max_debug_file_size=1000000
max_host_check_spread=30
max_service_check_spread=30
nagios_group=nagios
nagios_user=nagios
notification_timeout=30
object_cache_file=/opt/app/nagios/var/objects.cache
obsess_over_hosts=0
obsess_over_services=0
ocsp_timeout=5
passive_host_checks_are_soft=0
perfdata_timeout=5
precached_object_file=/opt/app/nagios/var/objects.precache
resource_file=/opt/app/nagios/etc/resource.cfg
retained_contact_host_attribute_mask=0
retained_contact_service_attribute_mask=0
retained_host_attribute_mask=0
retained_process_host_attribute_mask=0
retained_process_service_attribute_mask=0
retained_service_attribute_mask=0
retain_state_information=1
retention_update_interval=60
service_check_timeout=60
service_freshness_check_interval=60
service_inter_check_delay_method=s
service_interleave_factor=s
soft_state_dependencies=0
state_retention_file=/opt/app/nagios/var/retention.dat
status_file=/opt/app/nagios/var/status.dat
status_update_interval=10
temp_file=/opt/app/nagios/var/nagios.tmp
temp_path=/tmp
use_aggressive_host_checking=0
#use_embedded_perl_implicitly=1
use_regexp_matching=0
use_retained_program_state=1
use_retained_scheduling_info=1
use_syslog=0
use_true_regexp_matching=0
Re: Increasing Inode size continuously.
Posted: Mon Mar 11, 2019 6:52 am
by scottwilkerson
Can you also share the command definitions for these commands
Code: Select all
process-host-perfdata-file
process-service-perfdata-file
Re: Increasing Inode size continuously.
Posted: Wed Mar 13, 2019 10:55 am
by sandeepatil
Please find command definition,
Code: Select all
define command{
command_name process-service-perfdata-file
command_line /bin/mv /opt/app/nagios/pnp4nagios/var/service-perfdata /opt/app/nagios/pnp4nagios/var/spool/service-perfdata.$TIMET$
}
define command{
command_name process-host-perfdata-file
command_line /bin/mv /opt/app/nagios/pnp4nagios/var/host-perfdata /opt/app/nagios/pnp4nagios/var/spool/host-perfdata.$TIMET$
}
Re: Increasing Inode size continuously.
Posted: Wed Mar 13, 2019 10:57 am
by scottwilkerson
Ok, so do you have something reaping the filed out of this directory? like npcd?
Code: Select all
/opt/app/nagios/pnp4nagios/var/spool
What is the output of this command
Code: Select all
ls /opt/app/nagios/pnp4nagios/var/spool|wc -l
Re: Increasing Inode size continuously.
Posted: Wed Mar 13, 2019 11:14 am
by sandeepatil
Now issue under control, I have stopped httpd and nagios service, delete checkresults directory and created new.
But this issue occurring repetitively. How to avoid and monitor before occurring.
Code: Select all
# ls /opt/app/nagios/pnp4nagios/var/spool|wc -l
18
Re: Increasing Inode size continuously.
Posted: Wed Mar 13, 2019 12:06 pm
by scottwilkerson
Oh the files are in the checkresults directory?
Is the path you cleared /opt/app/nagios/var/spool/checkresults ?
Re: Increasing Inode size continuously.
Posted: Thu Mar 14, 2019 9:39 am
by sandeepatil
Please check below status and share solution for Inode full issue.
Code: Select all
# ll /opt/app/nagios/var/spool/checkresults | wc -l
2616
# ll /opt/app/nagios/var/spool/checkresults | wc -l
3252
# ll /opt/app/nagios/var/spool/checkresults | wc -l
633
# ll /opt/app/nagios/var/spool/checkresults | wc -l
1081
# ll /opt/app/nagios/var/spool/checkresults | wc -l
2083
Re: Increasing Inode size continuously.
Posted: Thu Mar 14, 2019 11:10 am
by scottwilkerson
Well this looks like it is fluctuating up and down which it should if you have a lot of passive results coming in.