Since today at noon, there's a number of Linux CentOS servers which are going to Halt whenever i type exit or use Control-D to close the putty window.
Did anyone encounter this weird behavior before?
I've checked the aliases list on the servers and there is no alias regarding halt command.
After the server came online i've checked the history and saw a "logout" command there but nothing which is related to Halt.
At first, i thought it happens only from my computer but later i realized that it happens to everyone which types exit, logout or control+d.
2 of these server are our main iptables firewalls and so it's super critical, your assistance is much appreciated.
It looks like that, and it only happens on servers with active IPTables:
[root@srv1 bin]# ssh srv2
root@srv2's password:
Last login: Sun Nov 11 17:19:41 2012 from 192.168.12.98
[root@srv2 ~]# vim /etc/crontab
[root@srv2 ~]# exit
logout
Broadcast message from root (pts/1) (Tue Nov 13 10:44:04 2012):
The system is going down for system halt NOW!
Connection to srv2 closed.
[root@srv1 bin]#
In my troubleshooting steps i came across the command strace, and so i've opened two bash windows to one of the problematic server and i used strace -p PID_of_bash.
When i typed in exit in the first shell it did go to halt, attached is the strace output, if you can check it out and tell me if you see anything suspicious i'd be more than thankful.
RER, 0x2b0e45a8f2f0}, 8) = 0
rt_sigaction(SIGALRM, {0x4484f0, [HUP INT ILL TRAP ABRT BUS FPE USR1 SEGV USR2 PIPE ALRM TERM XCPU XFSZ VTALRM SYS], SA_RESTORER, 0x2b0e45a8f2f0}, {0x47c450, [], SA_RESTORER, 0x2b0e45a8f2f0}, 8) = 0
rt_sigaction(SIGTSTP, {0x1, [], SA_RESTORER, 0x2b0e45a8f2f0}, {0x1, [], SA_RESTORER, 0x2b0e45a8f2f0}, 8) = 0
rt_sigaction(SIGTTOU, {0x1, [], SA_RESTORER, 0x2b0e45a8f2f0}, {0x1, [], SA_RESTORER, 0x2b0e45a8f2f0}, 8) = 0
rt_sigaction(SIGTTIN, {0x1, [], SA_RESTORER, 0x2b0e45a8f2f0}, {0x1, [], SA_RESTORER, 0x2b0e45a8f2f0}, 8) = 0
rt_sigaction(SIGWINCH, {0x448370, [], SA_RESTORER, 0x2b0e45a8f2f0}, {0x47c410, [], SA_RESTORER|SA_RESTART, 0x2b0e45a8f2f0}, 8) = 0
socket(PF_NETLINK, SOCK_RAW, 9) = 3
sendmsg(3, {msg_name(12)={sa_family=AF_NETLINK, pid=0, groups=00000000}, msg_iov(2)=[{"\25\0\0\0d\4\1\0\0\0\0\0\0\0\0\0", 16}, {"exit\0", 5}], msg_controllen=0, msg_flags=0}, 0) = 21
close(3) = 0
rt_sigaction(SIGINT, {0x448700, [], SA_RESTORER, 0x2b0e45a8f2f0}, {0x448700, [], SA_RESTORER, 0x2b0e45a8f2f0}, 8) = 0
write(2, "logout\n", 7) = 7
write(2, "There are stopped jobs.\n", 24) = 24
rt_sigprocmask(SIG_BLOCK, NULL, [], 8) = 0
rt_sigaction(SIGINT, {0x448700, [], SA_RESTORER, 0x2b0e45a8f2f0}, {0x448700, [], SA_RESTORER, 0x2b0e45a8f2f0}, 8) = 0
rt_sigprocmask(SIG_BLOCK, NULL, [], 8) = 0
rt_sigprocmask(SIG_BLOCK, [INT CHLD], [], 8) = 0
pipe([3, 4]) = 0
clone(child_stack=0, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0x2b0e45db6fe0) = 23717
setpgid(23717, 23717) = 0
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
rt_sigprocmask(SIG_BLOCK, [CHLD], [], 8) = 0
close(3) = 0
close(4) = 0
rt_sigprocmask(SIG_BLOCK, [CHLD TSTP TTIN TTOU], [CHLD], 8) = 0
ioctl(255, TIOCSPGRP, [23717]) = 0
rt_sigprocmask(SIG_SETMASK, [CHLD], NULL, 8) = 0
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
rt_sigprocmask(SIG_BLOCK, [CHLD], [], 8) = 0
wait4(-1, [{WIFEXITED(s) && WEXITSTATUS(s) == 0}], WSTOPPED|WCONTINUED, NULL) = 23717
rt_sigprocmask(SIG_BLOCK, [CHLD TSTP TTIN TTOU], [CHLD], 8) = 0
ioctl(255, TIOCSPGRP, [20458]) = 0
rt_sigprocmask(SIG_SETMASK, [CHLD], NULL, 8) = 0
ioctl(255, SNDCTL_TMR_TIMEBASE or TCGETS, {B38400 opost isig icanon echo ...}) = 0
ioctl(255, TIOCGWINSZ, {ws_row=53, ws_col=211, ws_xpixel=0, ws_ypixel=0}) = 0
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
--- SIGCHLD (Child exited) @ 0 (0) ---
wait4(-1, 0x7fff395da984, WNOHANG|WSTOPPED|WCONTINUED, NULL) = 0
rt_sigreturn(0x11) = 0
rt_sigprocmask(SIG_BLOCK, [CHLD TSTP TTIN TTOU], [], 8) = 0
ioctl(255, TIOCSPGRP, [20458]) = 0
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
rt_sigaction(SIGINT, {0x448700, [], SA_RESTORER, 0x2b0e45a8f2f0}, {0x448700, [], SA_RESTORER, 0x2b0e45a8f2f0}, 8) = 0
rt_sigprocmask(SIG_BLOCK, [INT], [], 8) = 0
ioctl(0, TIOCGWINSZ, {ws_row=53, ws_col=211, ws_xpixel=0, ws_ypixel=0}) = 0
ioctl(0, TIOCSWINSZ, {ws_row=53, ws_col=211, ws_xpixel=0, ws_ypixel=0}) = 0
ioctl(0, SNDCTL_TMR_TIMEBASE or TCGETS, {B38400 opost isig icanon echo ...}) = 0
ioctl(0, SNDCTL_TMR_STOP or TCSETSW, {B38400 opost isig -icanon -echo ...}) = 0
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
rt_sigprocmask(SIG_BLOCK, [INT QUIT ALRM TSTP TTIN TTOU], [], 8) = 0
rt_sigaction(SIGINT, {0x47c450, [], SA_RESTORER, 0x2b0e45a8f2f0}, {0x448700, [], SA_RESTORER, 0x2b0e45a8f2f0}, 8) = 0
rt_sigaction(SIGTERM, {0x47c450, [], SA_RESTORER, 0x2b0e45a8f2f0}, {0x1, [], SA_RESTORER, 0x2b0e45a8f2f0}, 8) = 0
rt_sigaction(SIGTERM, {0x1, [], SA_RESTORER, 0x2b0e45a8f2f0}, {0x47c450, [], SA_RESTORER, 0x2b0e45a8f2f0}, 8) = 0
rt_sigaction(SIGQUIT, {0x47c450, [], SA_RESTORER, 0x2b0e45a8f2f0}, {0x1, [], SA_RESTORER, 0x2b0e45a8f2f0}, 8) = 0
rt_sigaction(SIGQUIT, {0x1, [], SA_RESTORER, 0x2b0e45a8f2f0}, {0x47c450, [], SA_RESTORER, 0x2b0e45a8f2f0}, 8) = 0
rt_sigaction(SIGALRM, {0x47c450, [], SA_RESTORER, 0x2b0e45a8f2f0}, {0x4484f0, [HUP INT ILL TRAP ABRT BUS FPE USR1 SEGV USR2 PIPE ALRM TERM XCPU XFSZ VTALRM SYS], SA_RESTORER, 0x2b0e45a8f2f0}, 8) = 0
rt_sigaction(SIGTSTP, {0x47c450, [], SA_RESTORER, 0x2b0e45a8f2f0}, {0x1, [], SA_RESTORER, 0x2b0e45a8f2f0}, 8) = 0
rt_sigaction(SIGTSTP, {0x1, [], SA_RESTORER, 0x2b0e45a8f2f0}, {0x47c450, [], SA_RESTORER, 0x2b0e45a8f2f0}, 8) = 0
rt_sigaction(SIGTTOU, {0x47c450, [], SA_RESTORER, 0x2b0e45a8f2f0}, {0x1, [], SA_RESTORER, 0x2b0e45a8f2f0}, 8) = 0
rt_sigaction(SIGTTOU, {0x1, [], SA_RESTORER, 0x2b0e45a8f2f0}, {0x47c450, [], SA_RESTORER, 0x2b0e45a8f2f0}, 8) = 0
rt_sigaction(SIGTTIN, {0x47c450, [], SA_RESTORER, 0x2b0e45a8f2f0}, {0x1, [], SA_RESTORER, 0x2b0e45a8f2f0}, 8) = 0
rt_sigaction(SIGTTIN, {0x1, [], SA_RESTORER, 0x2b0e45a8f2f0}, {0x47c450, [], SA_RESTORER, 0x2b0e45a8f2f0}, 8) = 0
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
rt_sigaction(SIGWINCH, {0x47c410, [], SA_RESTORER|SA_RESTART, 0x2b0e45a8f2f0}, {0x448370, [], SA_RESTORER, 0x2b0e45a8f2f0}, 8) = 0
write(2, "[root@g2-lga ~]# ", 17) = 17
rt_sigprocmask(SIG_BLOCK, NULL, [], 8) = 0
read(0, "e", 1) = 1
write(2, "e", 1) = 1
rt_sigprocmask(SIG_BLOCK, NULL, [], 8) = 0
read(0, "x", 1) = 1
write(2, "x", 1) = 1
rt_sigprocmask(SIG_BLOCK, NULL, [], 8) = 0
read(0, "i", 1) = 1
write(2, "i", 1) = 1
rt_sigprocmask(SIG_BLOCK, NULL, [], 8) = 0
read(0, "t", 1) = 1
write(2, "t", 1) = 1
rt_sigprocmask(SIG_BLOCK, NULL, [], 8) = 0
read(0, "\r", 1) = 1
write(2, "\n", 1) = 1
rt_sigprocmask(SIG_BLOCK, [INT], [], 8) = 0
ioctl(0, SNDCTL_TMR_STOP or TCSETSW, {B38400 opost isig icanon echo ...}) = 0
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
rt_sigaction(SIGINT, {0x448700, [], SA_RESTORER, 0x2b0e45a8f2f0}, {0x47c450, [], SA_RESTORER, 0x2b0e45a8f2f0}, 8) = 0
rt_sigaction(SIGTERM, {0x1, [], SA_RESTORER, 0x2b0e45a8f2f0}, {0x1, [], SA_RESTORER, 0x2b0e45a8f2f0}, 8) = 0
rt_sigaction(SIGQUIT, {0x1, [], SA_RESTORER, 0x2b0e45a8f2f0}, {0x1, [], SA_RESTORER, 0x2b0e45a8f2f0}, 8) = 0
rt_sigaction(SIGALRM, {0x4484f0, [HUP INT ILL TRAP ABRT BUS FPE USR1 SEGV USR2 PIPE ALRM TERM XCPU XFSZ VTALRM SYS], SA_RESTORER, 0x2b0e45a8f2f0}, {0x47c450, [], SA_RESTORER, 0x2b0e45a8f2f0}, 8) = 0
rt_sigaction(SIGTSTP, {0x1, [], SA_RESTORER, 0x2b0e45a8f2f0}, {0x1, [], SA_RESTORER, 0x2b0e45a8f2f0}, 8) = 0
rt_sigaction(SIGTTOU, {0x1, [], SA_RESTORER, 0x2b0e45a8f2f0}, {0x1, [], SA_RESTORER, 0x2b0e45a8f2f0}, 8) = 0
rt_sigaction(SIGTTIN, {0x1, [], SA_RESTORER, 0x2b0e45a8f2f0}, {0x1, [], SA_RESTORER, 0x2b0e45a8f2f0}, 8) = 0
rt_sigaction(SIGWINCH, {0x448370, [], SA_RESTORER, 0x2b0e45a8f2f0}, {0x47c410, [], SA_RESTORER|SA_RESTART, 0x2b0e45a8f2f0}, 8) = 0
socket(PF_NETLINK, SOCK_RAW, 9) = 3
sendmsg(3, {msg_name(12)={sa_family=AF_NETLINK, pid=0, groups=00000000}, msg_iov(2)=[{"\25\0\0\0d\4\1\0\0\0\0\0\0\0\0\0", 16}, {"exit\0", 5}], msg_controllen=0, msg_flags=0}, 0) = 21
close(3) = 0
rt_sigaction(SIGINT, {0x448700, [], SA_RESTORER, 0x2b0e45a8f2f0}, {0x448700, [], SA_RESTORER, 0x2b0e45a8f2f0}, 8) = 0
write(2, "logout\n", 7) = 7
open("/root/.bash_logout", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=24, ...}) = 0
read(3, "# ~/.bash_logout\n\nclear\n", 24) = 24
close(3) = 0
rt_sigprocmask(SIG_BLOCK, NULL, [], 8) = 0
rt_sigprocmask(SIG_BLOCK, NULL, [], 8) = 0
rt_sigprocmask(SIG_BLOCK, NULL, [], 8) = 0
rt_sigprocmask(SIG_BLOCK, NULL, [], 8) = 0
stat(".", {st_mode=S_IFDIR|0750, st_size=12288, ...}) = 0
stat("/usr/kerberos/sbin/clear", 0x7fff395da960) = -1 ENOENT (No such file or directory)
stat("/usr/kerberos/bin/clear", 0x7fff395da960) = -1 ENOENT (No such file or directory)
stat("/usr/local/sbin/clear", 0x7fff395da960) = -1 ENOENT (No such file or directory)
stat("/usr/local/bin/clear", 0x7fff395da960) = -1 ENOENT (No such file or directory)
stat("/sbin/clear", 0x7fff395da960) = -1 ENOENT (No such file or directory)
stat("/bin/clear", 0x7fff395da960) = -1 ENOENT (No such file or directory)
stat("/usr/sbin/clear", 0x7fff395da960) = -1 ENOENT (No such file or directory)
stat("/usr/bin/clear", {st_mode=S_IFREG|0755, st_size=12712, ...}) = 0
access("/usr/bin/clear", X_OK) = 0
access("/usr/bin/clear", R_OK) = 0
stat("/usr/bin/clear", {st_mode=S_IFREG|0755, st_size=12712, ...}) = 0
access("/usr/bin/clear", X_OK) = 0
access("/usr/bin/clear", R_OK) = 0
rt_sigprocmask(SIG_BLOCK, [INT CHLD], [], 8) = 0
pipe([3, 4]) = 0
clone(child_stack=0, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0x2b0e45db6fe0) = 23726
setpgid(23726, 23726) = 0
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
rt_sigprocmask(SIG_BLOCK, [CHLD], [], 8) = 0
close(3) = 0
close(4) = 0
rt_sigprocmask(SIG_BLOCK, [CHLD TSTP TTIN TTOU], [CHLD], 8) = 0
ioctl(255, TIOCSPGRP, [23726]) = 0
rt_sigprocmask(SIG_SETMASK, [CHLD], NULL, 8) = 0
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
rt_sigprocmask(SIG_BLOCK, [CHLD], [], 8) = 0
wait4(-1,
Broadcast message from root (pts/0) (Wed Nov 14 12:41:44 2012):
The system is going down for system halt NOW!
[{WIFEXITED(s) && WEXITSTATUS(s) == 0}], WSTOPPED|WCONTINUED, NULL) = 23726
rt_sigprocmask(SIG_BLOCK, [CHLD TSTP TTIN TTOU], [CHLD], 8) = 0
ioctl(255, TIOCSPGRP, [20458]) = 0
rt_sigprocmask(SIG_SETMASK, [CHLD], NULL, 8) = 0
ioctl(255, SNDCTL_TMR_TIMEBASE or TCGETS, {B38400 opost isig icanon echo ...}) = 0
ioctl(255, TIOCGWINSZ, {ws_row=53, ws_col=211, ws_xpixel=0, ws_ypixel=0}) = 0
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
--- SIGCHLD (Child exited) @ 0 (0) ---
wait4(-1, 0x7fff395da634, WNOHANG|WSTOPPED|WCONTINUED, NULL) = 0
rt_sigreturn(0x11) = 0
open("/etc/bash.bash_logout", O_RDONLY) = -1 ENOENT (No such file or directory)
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
rt_sigaction(SIGINT, {0x448700, [], SA_RESTORER, 0x2b0e45a8f2f0}, {0x448700, [], SA_RESTORER, 0x2b0e45a8f2f0}, 8) = 0
stat("/root/.bash_history", {st_mode=S_IFREG|0600, st_size=28900, ...}) = 0
open("/root/.bash_history", O_WRONLY|O_APPEND) = 3
write(3, "cd /etc/profile.d/\nls\nls -alrt\ng"..., 1120) = 1120
close(3) = 0
open("/root/.bash_history", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0600, st_size=30020, ...}) = 0
read(3, "history \nping g1-lga\nping g1-lga"..., 30020) = 30020
close(3) = 0
open("/root/.bash_history", O_WRONLY|O_TRUNC) = 3
write(3, "grep \"216.18\" *\nhistory \nexit\nvi"..., 27609) = 27609
close(3) = 0
kill(4294965658, SIGTERM) = 0
kill(4294965658, SIGCONT) = 0
--- SIGCHLD (Child exited) @ 0 (0) ---
wait4(-1, [{WIFSIGNALED(s) && WTERMSIG(s) == SIGTERM}], WNOHANG|WSTOPPED|WCONTINUED, NULL) = 1638
wait4(-1, 0x7fff395dac34, WNOHANG|WSTOPPED|WCONTINUED, NULL) = -1 ECHILD (No child processes)
rt_sigreturn(0x11) = 0
--- SIGCHLD (Child exited) @ 0 (0) ---
wait4(-1, 0x7fff395dac34, WNOHANG|WSTOPPED|WCONTINUED, NULL) = -1 ECHILD (No child processes)
rt_sigreturn(0xffffffffffffffff) = 0
rt_sigprocmask(SIG_BLOCK, [CHLD TSTP TTIN TTOU], [], 8) = 0
ioctl(255, TIOCSPGRP, [20458]) = 0
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
setpgid(0, 20458) = -1 EPERM (Operation not permitted)
exit_group(1) = ?
Process 20458 detached
[root@g2-lga ~]#
