check_http returning FORBIDDEN

[niebais]

./check_http -I xxx.xxx.xxx.xxx -u 'http://www.nuskin.com'
The ip here is private, so I didn't want to list it.

And here's what I'm getting:
read(3, "# Locale name alias data base.\n#"..., 4096) = 2528
read(3, "", 4096) = 0
close(3) = 0
munmap(0xb7fa4000, 4096) = 0
open("/usr/local/nagios/share/locale/en_US.UTF-8/LC_MESSAGES/nagios-plugins.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/nagios/share/locale/en_US.utf8/LC_MESSAGES/nagios-plugins.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/nagios/share/locale/en_US/LC_MESSAGES/nagios-plugins.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/nagios/share/locale/en.UTF-8/LC_MESSAGES/nagios-plugins.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/nagios/share/locale/en.utf8/LC_MESSAGES/nagios-plugins.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/nagios/share/locale/en/LC_MESSAGES/nagios-plugins.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
fstat64(1, {st_mode=S_IFREG|0644, st_size=11212, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7fa4000
write(1, "HTTP WARNING: HTTP/1.1 403 Forbi"..., 37HTTP WARNING: HTTP/1.1 403 Forbidden
) = 37
exit_group(1)


If you want, I could send the entire stack trace. It's a bit larger then I thought.

[niebais]

Here's the full stack trace:
execve("./check_http", ["./check_http", "-I", "xxx.xxx.xxx.xxx", "-u", "http://www.nuskin.com"], [/* 25 vars */]) = 0
brk(0) = 0x8976000
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=36405, ...}) = 0
mmap2(NULL, 36405, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb7f9c000
close(3) = 0
open("/lib/libnsl.so.1", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0 !\205\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=101404, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7f9b000
mmap2(0x84f000, 92104, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x84f000
mmap2(0x862000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x12) = 0x862000
mmap2(0x864000, 6088, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x864000
close(3) = 0
open("/lib/libresolv.so.2", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\300\20\236\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=76400, ...}) = 0
mmap2(0x9df000, 75976, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x9df000
mmap2(0x9ee000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xe) = 0x9ee000
mmap2(0x9f0000, 6344, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x9f0000
close(3) = 0
open("/lib/libssl.so.6", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\20KV\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=293108, ...}) = 0
mmap2(0x559000, 294184, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x559000
mmap2(0x59d000, 16384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x43) = 0x59d000
close(3) = 0
open("/lib/libcrypto.so.6", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\200\365\214\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=1296932, ...}) = 0
mmap2(0x89c000, 1312960, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x89c000
mmap2(0x9c6000, 77824, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x129) = 0x9c6000
mmap2(0x9d9000, 14528, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x9d9000
close(3) = 0
open("/lib/libdl.so.2", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0PJx\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=16428, ...}) = 0
mmap2(0x784000, 12408, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x784000
mmap2(0x786000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1) = 0x786000
close(3) = 0
open("/lib/libc.so.6", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\340\37e\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=1611564, ...}) = 0
mmap2(0x63c000, 1332676, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x63c000
mprotect(0x77b000, 4096, PROT_NONE) = 0
mmap2(0x77c000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x13f) = 0x77c000
mmap2(0x77f000, 9668, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x77f000
close(3) = 0
open("/usr/lib/libgssapi_krb5.so.2", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\320\303\265\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=190712, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7f9a000
mmap2(0xb55000, 187804, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb55000
mmap2(0xb82000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2d) = 0xb82000
close(3) = 0
open("/usr/lib/libkrb5.so.3", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`!M\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=612148, ...}) = 0
mmap2(0x4c1000, 613224, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x4c1000
mmap2(0x554000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x92) = 0x554000
close(3) = 0
open("/lib/libcom_err.so.2", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\340G\237\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=7748, ...}) = 0
mmap2(0x9f4000, 9164, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x9f4000
mmap2(0x9f6000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1) = 0x9f6000
close(3) = 0
open("/usr/lib/libk5crypto.so.3", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220\207\301\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=157304, ...}) = 0
mmap2(0xc15000, 155040, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xc15000
mmap2(0xc3a000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x25) = 0xc3a000
close(3) = 0
open("/usr/lib/libz.so.1", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\300\345|\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=75028, ...}) = 0
mmap2(0x7cd000, 76400, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7cd000
mmap2(0x7df000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x11) = 0x7df000
close(3) = 0
open("/usr/lib/libkrb5support.so.0", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`i\270\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=33680, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7f99000
mmap2(0xb85000, 34852, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb85000
mmap2(0xb8d000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7) = 0xb8d000
close(3) = 0
open("/lib/libkeyutils.so.1", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\20\230\237\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=7880, ...}) = 0
mmap2(0x9f9000, 9208, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x9f9000
mmap2(0x9fb000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1) = 0x9fb000
close(3) = 0
open("/lib/libselinux.so.1", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0000\345\263\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=93508, ...}) = 0
mmap2(0xb3b000, 97112, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb3b000
mmap2(0xb51000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x15) = 0xb51000
close(3) = 0
open("/lib/libsepol.so.1", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0@_\257\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=245376, ...}) = 0
mmap2(0xaf3000, 285024, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xaf3000
mmap2(0xb2e000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3b) = 0xb2e000
mmap2(0xb2f000, 39264, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb2f000
close(3) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7f98000
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7f97000
set_thread_area({entry_number:-1 -> 6, base_addr:0xb7f976d0, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}) = 0
mprotect(0x77c000, 8192, PROT_READ) = 0
mprotect(0x786000, 4096, PROT_READ) = 0
mprotect(0x9ee000, 4096, PROT_READ) = 0
mprotect(0x862000, 4096, PROT_READ) = 0
mprotect(0x638000, 4096, PROT_READ) = 0
munmap(0xb7f9c000, 36405) = 0
access("/etc/selinux/", F_OK) = 0
brk(0) = 0x8976000
brk(0x8997000) = 0x8997000
open("/etc/selinux/config", O_RDONLY|O_LARGEFILE) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=511, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7fa4000
read(3, "# This file controls the state o"..., 4096) = 511
read(3, "", 4096) = 0
close(3) = 0
munmap(0xb7fa4000, 4096) = 0
open("/proc/mounts", O_RDONLY|O_LARGEFILE) = 3
fstat64(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7fa4000
read(3, "rootfs / rootfs rw 0 0\n/dev/root"..., 4096) = 548
read(3, "", 4096) = 0
close(3) = 0
munmap(0xb7fa4000, 4096) = 0
open("/usr/lib/locale/locale-archive", O_RDONLY|O_LARGEFILE) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=56423232, ...}) = 0
mmap2(NULL, 2097152, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb7d97000
close(3) = 0
rt_sigaction(SIGALRM, {0x804f0f0, [ALRM], SA_RESTART}, {SIG_DFL, [], 0}, 8) = 0
alarm(10) = 0
gettimeofday({1276620152, 799707}, NULL) = 0
socket(PF_INET, SOCK_STREAM, IPPROTO_TCP) = 3
connect(3, {sa_family=AF_INET, sin_port=htons(80), sin_addr=inet_addr("172.24.196.36")}, 16) = 0
send(3, "GET http://www.nuskin.com HTTP/1"..., 111, 0) = 111
read(3, "HTTP/1.1 403 Forbidden\r\nDate: Tu"..., 8191) = 370
read(3, "", 8191) = 0
close(3) = 0
alarm(0) = 10
open("/usr/share/locale/locale.alias", O_RDONLY) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=2528, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7fa4000
read(3, "# Locale name alias data base.\n#"..., 4096) = 2528
read(3, "", 4096) = 0
close(3) = 0
munmap(0xb7fa4000, 4096) = 0
open("/usr/local/nagios/share/locale/en_US.UTF-8/LC_MESSAGES/nagios-plugins.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/nagios/share/locale/en_US.utf8/LC_MESSAGES/nagios-plugins.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/nagios/share/locale/en_US/LC_MESSAGES/nagios-plugins.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/nagios/share/locale/en.UTF-8/LC_MESSAGES/nagios-plugins.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/nagios/share/locale/en.utf8/LC_MESSAGES/nagios-plugins.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/nagios/share/locale/en/LC_MESSAGES/nagios-plugins.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
fstat64(1, {st_mode=S_IFREG|0644, st_size=11212, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7fa4000
write(1, "HTTP WARNING: HTTP/1.1 403 Forbi"..., 37HTTP WARNING: HTTP/1.1 403 Forbidden
) = 37
exit_group(1) = ?

[mmestnik]

Now you got me thinking about possible DNS cache poisoning attack and the forbidden you see is an attempt to gather passwords.
Make sure this is not a true negative(the inverse of a false positive).

[niebais]

Good guess, but we're using our Internal DNS for nuskin.com. I would suspect that if we were using external DNS. Two of our sites so far have had this problem. All the other sites seem to have no problem.

[mmestnik]

Please make sure to remove any sensitive information from your logs.

You have your options mixed up, this would test: http://192.168.1.100http://www.nuskin.com The default option to -u is / as in http://192.168.1.100/. a URL is a full server and path identifier, but a URI only indicates the path on the server, much like a relative URI that indicates the location relative to the current document.

The options you are looking for are, leave -u at the default unless you want a specific file or folder:
-I 192.168.1.100 -H http://www.nuskin.com

[niebais]

Please make sure to remove any sensitive information from your logs.

You have your options mixed up, this would test: http://192.168.1.100http://www.nuskin.com The default option to -u is / as in http://192.168.1.100/. a URL is a full server and path identifier, but a URI only indicates the path on the server, much like a relative URI that indicates the location relative to the current document.

The options you are looking for are, leave -u at the default unless you want a specific file or folder:
-I 192.168.1.100 -H http://www.nuskin.com

Well, amazingly, changing the options works great for our main site (nuskin.com). I need to bring this up, though. We set up www.nuskin.com monitoring through the wizard. I'm ok changing the options for sites if they act strangely, but the wizard probably needs to set up the options a little differently by default.

This case can be closed now. Works great.

[lccouncil]

Try http://www.nuskin.com/nuskin.html instead of just http://www.nuskin.com

[mmestnik]

We have a new Website wizard in development and I'm pleased to say that it should be vary impossible to create this type of issue with it.

No ETA on this, it's done when we believe it's done.

[niebais]

Try http://www.nuskin.com/nuskin.html instead of just http://www.nuskin.com

Interestingly enough, we tried this URL to see if it made any difference and it did at first, but than it still had intermittent issues. Using the -I option with the right -u option made this work great.