Add display of number of alerts

This commit is contained in:
root 2016-09-21 14:38:15 +02:00
parent 8807bc02e9
commit 544b4f7986
1 changed files with 171 additions and 161 deletions

View File

@ -60,7 +60,7 @@ while (!$conn) {
Nick => 'vivivi',
Ircname => 'April Nagios alerts',
Username => 'vivivi')
or print "Redialing...\n";
or print "Redialing...\n";
sleep 1;
}
@ -80,7 +80,7 @@ sub on_connect {
my $self = shift;
$identified_to_nickserv = 1;
print "Joining #april-monitoring...\n";
print "Joining #april-admin...\n";
# FIXME: this is broken right now. when this is re-added, it has to happen
# before we try to join channels.
@ -88,7 +88,7 @@ sub on_connect {
#$self->privmsg('nickserv',"identify xxx");
# CONFIG: channels you want us to announce to ...
$self->join("#april-monitoring");
$self->join("#april-admin");
#$self->join("#status");
}
@ -129,8 +129,8 @@ sub on_msg {
if ($arg =~ /Go away/i) { # Tell him to leave, and he does.
$self->quit("Yow!!");
exit 0;
# CONFIG: if you don't change this, anybody can restart your IRC bot at will
# if they've seen this ...
# CONFIG: if you don't change this, anybody can restart your IRC bot at will
# if they've seen this ...
} elsif ($arg =~ /restart ya dope/) {
$self->quit("Regrouping");
exec $0;
@ -155,94 +155,95 @@ sub on_public {
if ($arg =~ /^$mynick[,: ]/i || $arg =~ /^!/ ) {
if ($arg =~ /^(?:$mynick[,: ]|!)\s*ignore (.+)\s*$/i) {
$ignore{$1} = 1;
$self->privmsg([ @to ], "$nick: ok, ignoring notifications about $1");
$ignore{$1} = 1;
$self->privmsg([ @to ], "$nick: ok, ignoring notifications about $1");
} elsif ($arg =~ /^(?:$mynick[,: ]|!)\s*ignore$/i) {
if (keys %ignore) {
foreach my $i (keys %ignore) {
$self->privmsg([ @to ], "$nick: ignoring $i");
}
$self->privmsg([ @to ], "$nick: end of list");
} else {
$self->privmsg([ @to ], "$nick: I'm not currently ignoring any notifications");
}
if (keys %ignore) {
foreach my $i (keys %ignore) {
$self->privmsg([ @to ], "$nick: ignoring $i");
}
$self->privmsg([ @to ], "$nick: end of list");
} else {
$self->privmsg([ @to ], "$nick: I'm not currently ignoring any notifications");
}
} elsif ($arg =~ /^(?:$mynick[,: ]|!)\s*unignore (.+)\s*$/i) {
if (!exists($ignore{$1})) {
$self->privmsg([ @to ], "$nick: notifications about $1 weren't being ignored anyway.");
} else {
delete $ignore{$1};
$self->privmsg([ @to ], "$nick: ok, notifications about $1 no longer being ignored.");
}
if (!exists($ignore{$1})) {
$self->privmsg([ @to ], "$nick: notifications about $1 weren't being ignored anyway.");
} else {
delete $ignore{$1};
$self->privmsg([ @to ], "$nick: ok, notifications about $1 no longer being ignored.");
}
} elsif ($arg =~ /^(?:$mynick[,: ]|!)+(?:\s*ack\s*)?(\d+)(?:\s*ack\s*)?[:\s]+([^:]+)\s*$/) {
open CMDPIPE,">",$nagioscmd;
my ( $host, $svc ) = @{ $ACKS[$1] || [ undef, undef ] };
if ( defined $host && defined $svc ) {
printf CMDPIPE "[%lu] ACKNOWLEDGE_SVC_PROBLEM;%s;%s;1;1;1;%s;%s\n",time(),$host,$svc,$nick,$2;
} elsif ( defined $host ) {
printf CMDPIPE "[%lu] ACKNOWLEDGE_HOST_PROBLEM;%s;1;1;1;%s;%s\n",time(),$host,$nick,$2;
}
close CMDPIPE;
open CMDPIPE,">",$nagioscmd;
my ( $host, $svc ) = @{ $ACKS[$1] || [ undef, undef ] };
if ( defined $host && defined $svc ) {
printf CMDPIPE "[%lu] ACKNOWLEDGE_SVC_PROBLEM;%s;%s;1;1;1;%s;%s\n",time(),$host,$svc,$nick,$2;
} elsif ( defined $host ) {
printf CMDPIPE "[%lu] ACKNOWLEDGE_HOST_PROBLEM;%s;1;1;1;%s;%s\n",time(),$host,$nick,$2;
}
close CMDPIPE;
} elsif ($arg =~ /^(?:$mynick[,: ]|!)\s*ack ([^:]+):([^:]+)\s*$/) {
open CMDPIPE,">",$nagioscmd;
printf CMDPIPE "[%lu] ACKNOWLEDGE_HOST_PROBLEM;%s;1;1;1;%s;%s\n",time(),$1,$nick,$2;
close CMDPIPE;
open CMDPIPE,">",$nagioscmd;
printf CMDPIPE "[%lu] ACKNOWLEDGE_HOST_PROBLEM;%s;1;1;1;%s;%s\n",time(),$1,$nick,$2;
close CMDPIPE;
} elsif ($arg =~ /^(?:$mynick[,: ]|!)\s*ack ([^:]+):([^:]+):([^:]+)\s*$/) {
open CMDPIPE,">",$nagioscmd;
printf CMDPIPE "[%lu] ACKNOWLEDGE_SVC_PROBLEM;%s;%s;1;1;1;%s;%s\n",time(),$1,$2,$nick,$3;
close CMDPIPE;
open CMDPIPE,">",$nagioscmd;
printf CMDPIPE "[%lu] ACKNOWLEDGE_SVC_PROBLEM;%s;%s;1;1;1;%s;%s\n",time(),$1,$2,$nick,$3;
close CMDPIPE;
} elsif ($arg =~ /^(?:$mynick[,: ]|!)\s*unack (\d+)$/) {
open CMDPIPE,">",$nagioscmd;
my ( $host, $svc ) = @{ $ACKS[$1] || [ undef, undef ] };
if ( defined $host && defined $svc ) {
printf CMDPIPE "[%lu] REMOVE_SVC_ACKNOWLEDGEMENT;%s;%s\n",time(),$host,$svc;
$self->privmsg([ @to ], "$nick: ok, acknowledgment (if any) for $host:$svc has been removed.");
} elsif ( defined $host ) {
printf CMDPIPE "[%lu] REMOVE_HOST_ACKNOWLEDGEMENT;%s\n",time(),$host;
$self->privmsg([ @to ], "$nick: ok, acknowledgment (if any) for $host has been removed.");
}
close CMDPIPE;
open CMDPIPE,">",$nagioscmd;
my ( $host, $svc ) = @{ $ACKS[$1] || [ undef, undef ] };
if ( defined $host && defined $svc ) {
printf CMDPIPE "[%lu] REMOVE_SVC_ACKNOWLEDGEMENT;%s;%s\n",time(),$host,$svc;
$self->privmsg([ @to ], "$nick: ok, acknowledgment (if any) for $host:$svc has been removed.");
} elsif ( defined $host ) {
printf CMDPIPE "[%lu] REMOVE_HOST_ACKNOWLEDGEMENT;%s\n",time(),$host;
$self->privmsg([ @to ], "$nick: ok, acknowledgment (if any) for $host has been removed.");
}
close CMDPIPE;
} elsif ($arg =~ /^(?:$mynick[,: ]|!)\s*unack ([^:]+)\s*$/) {
open CMDPIPE,">",$nagioscmd;
printf CMDPIPE "[%lu] REMOVE_HOST_ACKNOWLEDGEMENT;%s\n",time(),$1;
close CMDPIPE;
$self->privmsg([ @to ], "$nick: ok, acknowledgment (if any) for $1 has been removed.");
open CMDPIPE,">",$nagioscmd;
printf CMDPIPE "[%lu] REMOVE_HOST_ACKNOWLEDGEMENT;%s\n",time(),$1;
close CMDPIPE;
$self->privmsg([ @to ], "$nick: ok, acknowledgment (if any) for $1 has been removed.");
} elsif ($arg =~ /^(?:$mynick[,: ]|!)\s*unack ([^:]+):([^:]+)\s*$/) {
open CMDPIPE,">",$nagioscmd;
printf CMDPIPE "[%lu] REMOVE_SVC_ACKNOWLEDGEMENT;%s;%s\n",time(),$1,$2;
close CMDPIPE;
$self->privmsg([ @to ], "$nick: ok, acknowledgment (if any) for $1:$2 has been removed.");
open CMDPIPE,">",$nagioscmd;
printf CMDPIPE "[%lu] REMOVE_SVC_ACKNOWLEDGEMENT;%s;%s\n",time(),$1,$2;
close CMDPIPE;
$self->privmsg([ @to ], "$nick: ok, acknowledgment (if any) for $1:$2 has been removed.");
} elsif ($arg =~ /^(?:$mynick[,: ]|!)\s*status\s*$/) {
my @lines = ();
open CMDPIPE,"-|","/usr/sbin/icingastats -m -d NUMSVCWARN,NUMSVCCRIT,NUMSVCUNKN";
while (<CMDPIPE>) {
chomp;
push @lines, $_;
}
close CMDPIPE;
my @prefix = ( 'Warn', 'Critical', 'Unknown' );
my $i = 0;
foreach my $line (@lines) {
$self->privmsg([ @to ], "$nick: ".$prefix[$i++].": $line");
}
my @lines = ();
open CMDPIPE,"-|","/usr/sbin/icingastats -m -d NUMSVCWARN,NUMSVCCRIT,NUMSVCUNKN";
while (<CMDPIPE>) {
chomp;
push @lines, $_;
}
close CMDPIPE;
my @prefix = ( 'Warn', 'Critical', 'Unknown' );
my $i = 0;
foreach my $line (@lines) {
$self->privmsg([ @to ], "$nick: ".$prefix[$i++].": $line");
}
} elsif ($arg =~ /^(?:$mynick[,: ]|!)\s*help/i) {
$self->privmsg([ @to ], "$nick: You're right, I need help!");
$self->privmsg([ @to ], "$nick: supported commands: ignore, unignore, ack, unack, help");
$self->privmsg([ @to ], "$nick: You're right, I need help!");
$self->privmsg([ @to ], "$nick: supported commands: ignore, unignore, ack, unack, ta gueule, help");
} elsif ($arg =~ /(?:hey|hi|hello|yo)(?:\.|,|\!|\s_)/i) {
$self->privmsg([ @to ], "Yo, $nick!");
} elsif ($arg =~ /(?:$mynick[,: ]|!)\s*ta\s*g(:?ueu|o)le/i) {
$self->privmsg([ @to ], "Yo, $nick!");
} elsif ($arg =~ /(?:$mynick[,: ]|!)\s*ta\s*g(:?ueu|o)le/i ||
$arg =~ /(?:$mynick[,: ]|!)\s*chut/i) {
if ( $::tagueule == 0 )
{
$self->privmsg([ @to ], "$nick: ok, I'll shut up!");
$self->privmsg([ @to ], "$nick: d'accord je la ferme :-(");
$::tagueule = 1;
$self->nick('vivivi[zzz]');
}
else {
$self->privmsg([ @to ], "$nick: ok, I'm back!");
$self->privmsg([ @to ], "$nick: je reviens et je ne suis pas content !");
$::tagueule = 0;
$self->nick('vivivi');
&adjust_nick ( $conn );
}
} else {
# $self->privmsg([ @to ], "Yo!");
# $self->privmsg([ @to ], "Yo!");
}
}
}
@ -317,7 +318,7 @@ sub on_disconnect {
my ($self, $event) = @_;
print "Disconnected from ", $event->from(), " (",
($event->args())[0], "). Attempting to reconnect...\n";
($event->args())[0], "). Attempting to reconnect...\n";
$identified_to_nickserv = 0;
while (!$self->connected) {
$self->connect();
@ -336,8 +337,8 @@ sub on_notice {
# FIXME: the nickserv path is broken ...
if ((lc($event->nick) eq "nickserv") && ($args[0] =~ "nickname is registered")) {
# print "Identifying to NickServ...\n";
# $self->privmsg('nickserv',"identify xxx");
# print "Identifying to NickServ...\n";
# $self->privmsg('nickserv',"identify xxx");
}
# FIXME: the nickserv path is broken ...
@ -347,7 +348,7 @@ sub on_notice {
print "Joining channels...\n";
# CONFIG: channels that should be joined go here.
$self->join("#april-monitoring");
$self->join("#april-admin");
}
}
@ -393,7 +394,7 @@ my %C = (
C => "\x0311",
Y => "\x038",
N => "\x0315",
);
);
my $state_to_color = {
OK => $C{G},
@ -409,94 +410,103 @@ while (1) {
$irc->do_one_loop();
if ($identified_to_nickserv) {
# CONFIG: change where we announce stuff here
my @channels = ('#april-monitoring');
while (defined (my $line = <NAGIOS>)) {
print $line;
chomp($line);
next if $::tagueule;
if ($line =~ /^\[\d+\] (HOST|SERVICE) NOTIFICATION: (.+)$/) {
my ($type, $msg) = ($1, $2);
my ($who, $host, $service, $state, $how, $output);
if ($type eq "HOST") {
($who, $host, $state, $how, $output) = split(";",$msg,5);
next if exists $renot{$host} && $renot{$host} >= time() - 5;
$renot{$host} = time();
# CONFIG: change where we announce stuff here
my @channels = ('#april-admin');
while (defined (my $line = <NAGIOS>)) {
print $line;
chomp($line);
next if $::tagueule;
if ($line =~ /^\[\d+\] (HOST|SERVICE) NOTIFICATION: (.+)$/) {
my ($type, $msg) = ($1, $2);
my ($who, $host, $service, $state, $how, $output);
if ($type eq "HOST") {
($who, $host, $state, $how, $output) = split(";",$msg,5);
next if exists $renot{$host} && $renot{$host} >= time() - 5;
$renot{$host} = time();
my $id = ackable($host,undef,$state,$output);
$msg = $state_to_color->{$state} . "$id$host is $state: $output";
}
else {
($who, $host, $service, $state, $how, $output) = split(";",$msg,6);
next if exists $renot{"$host:$service"} && $renot{"$host:$service"} >= time() - 5;
$renot{"$host:$service"} = time();
my $id = ackable($host,undef,$state,$output);
$msg = $state_to_color->{$state} . "$id$host is $state: $output";
&adjust_nick ( $conn );
}
else {
($who, $host, $service, $state, $how, $output) = split(";",$msg,6);
next if exists $renot{"$host:$service"} && $renot{"$host:$service"} >= time() - 5;
$renot{"$host:$service"} = time();
my $id = ackable($host,$service,$state,$output);
$output =~ s/^\s*CRITICAL\s*:\s//gi;
$output =~ s/^\s*WARNING\s*:\s//gi;
$output =~ s/^\s*OK\s*:\s//gi;
$msg = $state_to_color->{$state} . "$id$host:$service is $state: $output";
}
$service ||= "";
my $id = ackable($host,$service,$state,$output);
$output =~ s/^\s*CRITICAL\s*:\s//gi;
$output =~ s/^\s*WARNING\s*:\s//gi;
$output =~ s/^\s*OK\s*:\s//gi;
$msg = $state_to_color->{$state} . "$id$host:$service is $state: $output";
&adjust_nick ( $conn );
}
$service ||= "";
# alerts can be separated by who they're for (which group is getting it)
#if ($who eq "mark") { push @channels, "#dw_ops", "#dw_work" }
# alerts can be separated by who they're for (which group is getting it)
#if ($who eq "mark") { push @channels, "#dw_ops", "#dw_work" }
if ((!grep { $_ eq "$host:$service" } keys %ignore) &&
(!grep { $_ eq "$host" } keys %ignore)) {
$conn->privmsg($_,"$msg") foreach @channels;
}
#[1310493492] EXTERNAL COMMAND: SCHEDULE_HOST_DOWNTIME;gearmanworker-gearmanworker3;1310492897;1310500097;1;0;28800;stumble.barr;rebooting
} elsif ($line =~ /EXTERNAL COMMAND: SCHEDULE_HOST_DOWNTIME;(.+?);(\d+);(\d+);\d+;\d+;\d+;(.+?);(.+)$/) {
my $ns = $3 - $2;
$conn->privmsg($_, "Downtime (${ns}s) scheduled for host $1 by $4: $5")
foreach @channels;
} elsif ($line =~ /EXTERNAL COMMAND: SCHEDULE_SVC_DOWNTIME;(.+?);(.+?);(\d+);(\d+);\d+;\d+;\d+;(.+?);(.+)$/) {
my $ns = $4 - $3;
$conn->privmsg($_, "Downtime (${ns}s) scheduled for service $1: $2 by $5: $6")
foreach @channels;
} elsif ($line =~ /EXTERNAL COMMAND: (EN|DIS)ABLE_HOST_(SVC_)?NOTIFICATIONS;(.+)$/) {
my $type = $2 ? 'All service' : 'Host';
my $verb = $1 eq 'EN' ? "$C{G}enabled$C{N}" : "$C{R}disabled$C{N}";
$conn->privmsg($_, "$type notifications $verb: $C{O}$3")
foreach @channels;
} elsif ($line =~ /EXTERNAL COMMAND: (EN|DIS)ABLE_(SVC_)?NOTIFICATIONS;(.+?);(.+)$/) {
my $verb = $1 eq 'EN' ? "$C{G}enabled$C{N}" : "$C{R}disabled$C{N}";
$conn->privmsg($_, "Service notifications $verb on $C{O}$3$C{N}: $C{O}$4")
foreach @channels;
} elsif ($line =~ /HOST DOWNTIME ALERT: (.+?);(.+?);/) {
my $vb = { CANCELLED => 'been manually removed from', STOPPED => 'exited from', STARTED => 'entered' }->{$2};
$conn->privmsg($_, "Host $C{O}$1$C{N} has $vb scheduled downtime.")
foreach @channels;
} elsif ($line =~ /SERVICE DOWNTIME ALERT: (.+?);(.+?);(.+?);/) {
my $vb = { CANCELLED => 'been manually removed from', STOPPED => 'exited from', STARTED => 'entered' }->{$3};
$conn->privmsg($_, "Service $C{O}$1$C{N}: $C{O}$2$C{N} has $vb scheduled downtime.")
foreach @channels;
}
}
if ((time - $laststat) > 30) {
# we don't want to constantly stat the file or we'll hammer the disk.
# keep it to once every 30 seconds. Here we compare the inode of the
# file we have open and the file at the path we're expecting and see
# if they match. If they don't, then the log has been rotated, so we
# close and reopen it to pick up the new logfile. We also only do it
# if we don't read a line from the file, so that way we make sure
# actually gotten to the end of the current file before we check for
# a rotation.
$laststat = time;
if ((stat NAGIOS)[1] != (stat $nagioslog)[1]) {
close NAGIOS;
open NAGIOS, "<$nagioslog";
}
open CMDPIPE,"-|","/usr/sbin/icingastats -m -d NUMSVCWARN,NUMSVCCRIT,NUMSVCUNKN";
my $count = 0;
while (<CMDPIPE>) {
$count += $_;
if ((!grep { $_ eq "$host:$service" } keys %ignore) &&
(!grep { $_ eq "$host" } keys %ignore)) {
$conn->privmsg($_,"$msg") foreach @channels;
}
#[1310493492] EXTERNAL COMMAND: SCHEDULE_HOST_DOWNTIME;gearmanworker-gearmanworker3;1310492897;1310500097;1;0;28800;stumble.barr;rebooting
} elsif ($line =~ /EXTERNAL COMMAND: SCHEDULE_HOST_DOWNTIME;(.+?);(\d+);(\d+);\d+;\d+;\d+;(.+?);(.+)$/) {
my $ns = $3 - $2;
$conn->privmsg($_, "Downtime (${ns}s) scheduled for host $1 by $4: $5")
foreach @channels;
} elsif ($line =~ /EXTERNAL COMMAND: SCHEDULE_SVC_DOWNTIME;(.+?);(.+?);(\d+);(\d+);\d+;\d+;\d+;(.+?);(.+)$/) {
my $ns = $4 - $3;
$conn->privmsg($_, "Downtime (${ns}s) scheduled for service $1: $2 by $5: $6")
foreach @channels;
} elsif ($line =~ /EXTERNAL COMMAND: (EN|DIS)ABLE_HOST_(SVC_)?NOTIFICATIONS;(.+)$/) {
my $type = $2 ? 'All service' : 'Host';
my $verb = $1 eq 'EN' ? "$C{G}enabled$C{N}" : "$C{R}disabled$C{N}";
$conn->privmsg($_, "$type notifications $verb: $C{O}$3")
foreach @channels;
} elsif ($line =~ /EXTERNAL COMMAND: (EN|DIS)ABLE_(SVC_)?NOTIFICATIONS;(.+?);(.+)$/) {
my $verb = $1 eq 'EN' ? "$C{G}enabled$C{N}" : "$C{R}disabled$C{N}";
$conn->privmsg($_, "Service notifications $verb on $C{O}$3$C{N}: $C{O}$4")
foreach @channels;
} elsif ($line =~ /HOST DOWNTIME ALERT: (.+?);(.+?);/) {
my $vb = { CANCELLED => 'been manually removed from', STOPPED => 'exited from', STARTED => 'entered' }->{$2};
$conn->privmsg($_, "Host $C{O}$1$C{N} has $vb scheduled downtime.")
foreach @channels;
} elsif ($line =~ /SERVICE DOWNTIME ALERT: (.+?);(.+?);(.+?);/) {
my $vb = { CANCELLED => 'been manually removed from', STOPPED => 'exited from', STARTED => 'entered' }->{$3};
$conn->privmsg($_, "Service $C{O}$1$C{N}: $C{O}$2$C{N} has $vb scheduled downtime.")
foreach @channels;
}
}
if ((time - $laststat) > 30) {
# we don't want to constantly stat the file or we'll hammer the disk.
# keep it to once every 30 seconds. Here we compare the inode of the
# file we have open and the file at the path we're expecting and see
# if they match. If they don't, then the log has been rotated, so we
# close and reopen it to pick up the new logfile. We also only do it
# if we don't read a line from the file, so that way we make sure
# actually gotten to the end of the current file before we check for
# a rotation.
$laststat = time;
if ((stat NAGIOS)[1] != (stat $nagioslog)[1]) {
close NAGIOS;
open NAGIOS, "<$nagioslog";
}
&adjust_nick ( $conn );
}
close CMDPIPE;
$conn->nick(sprintf('vivivi[%s]', $count)) if ( $count > 0 );
}
}
}
sub adjust_nick
{
my ( $conn ) = @_;
open CMDPIPE,"-|","/usr/sbin/icingastats -m -d NUMSVCWARN,NUMSVCCRIT,NUMSVCUNKN";
my $count = 0;
while (<CMDPIPE>) {
$count += $_;
}
close CMDPIPE;
$conn->nick(sprintf('vivivi[%s]', $count)) if ( $count > 0 );
}