This script is a Big Brother monitoring script for an application server running an application that provides provisioning for a Lucent switch network. There were several requirements. The users needed to be notified if the provisioning operation failed so that it could be restarted and if it succeeded so that the next provisioning operation could be started. Also the users wanted to be notified if any of the critical app server processes died unexpectedly. The following script was written to address the user's requests. 



#!/usr/bin/perl
#
#
# Date 6/10/2004
# Author - Vic Engle
# 
#
use Sys::Hostname;
$LocalHost=hostname();
#
#
	

# Set BigBro vars
$BBHOME = '/opt/bb/bb';
$BBTMP = '/opt/bb/bb/tmp';
$BB = "$BBHOME/bin/bb";
$BBDISP = 'jumpstart.mydomain.local';
$test = 'ssi';
$test2 = 'provisioning';
$host = GetBBHostName();
$errors = "$BBHOME/tmp/errors.dat";
$serverlog = "/var/opt/SUNWappserver7/domains/ursdomain/ursserver/logs/server.log";

$PIDDIR='/var/opt/SSIengctr/PIDs';
$SSIPROCS{SSIOTR} = `cat $PIDDIR/SSIOTR.pid`;
$SSIPROCS{SSIOAM} = `cat $PIDDIR/SSIOAM.pid`;
$SSIPROCS{SSILR} = `cat $PIDDIR/SSILR.pid`;

$OTR_RESTART_TIME = GetOTR();
%ProcessList = GetProcList();


$TIME = localtime(time);

if (! -e "$errors") {
	system("/usr/bin/touch $errors");
}

open(ERR,"<$errors");
$errorlist = ;
close(ERR);
chomp($errorlist);

$color = 'green';



$Message = "SSI server.log\n\n";

open(LOG,"<$serverlog") or Fail("$!");

while ( $line = ) {
	if ($line =~/provision failed/i){
		$curr = substr($line,1,20);
		$curr =~ s/\///g;
		$curr =~ s/://g;
		next if ("$errorlist" =~ /$curr/);
		$errorlist .= "$curr";
		$color = "red";
                $Message .= "&red Provisioning Failed!\n\n$line\n";
        }  
        if ($line =~/provision succeeded/i){
                $curr = substr($line,1,20);
                $curr =~ s/\///g;
                $curr =~ s/://g;
                next if ("$errorlist" =~ /$curr/);
                $errorlist .= "$curr";
                $color = "red";
                $Message .= "&red Provisioning Succeeded!\n\n$line\n";
        }

}
close(LOG);


$proc_color = 'green';
$ProcMessage = "SSI Process Monitor\n\n";
foreach $ssiproc (  keys %SSIPROCS  ) {
	chomp($SSIPROCS{$ssiproc});
	if (exists ($ProcessList{$SSIPROCS{$ssiproc}}) ) {
		$ProcMessage .= "&green $ssiproc PID $SSIPROCS{$ssiproc} OK\n";
	} else {
		$proc_color = 'red';
		if ($SSIPROC eq "SSIOTR" ) {
			$ProcMessage .= "&red $ssiproc PID $SSIPROCS{$ssiproc} NOT Running. Attempting to restart.\n";
			#$OTR_Status = RestartOTR();
			#$ProcMessage .= " - $OTR_Status\n";
		} else {
			$ProcMessage .= "&red $ssiproc PID $SSIPROCS{$ssiproc} NOT Running\n";
		}
	}
}

open(ERR,">$errors");
print ERR "$errorlist";
close(ERR);



$Message .= "&green Server.log (No Current Errors)\n\n" if ($color eq 'green'); 

#print "$BB $BBDISP \"status $host.$test $color $TIME server.log messages $LocalHost.\n\n$Message\"";
system("$BB $BBDISP \"status $host.$test2 $color $TIME server.log messages $LocalHost.\n\n$Message\"");
system("$BB $BBDISP \"status $host.$test $proc_color $TIME Process List on $LocalHost.\n\n$ProcMessage\"");

#
#
sub Fail {
        $Message = "Failed opening $serverlog - ($_[0]).\n\n";
        system("$BB $BBDISP \"status $host.$test yellow $TIME server.log $LocalHost.\n\n$Message\"");
        exit;
}
#
sub GetBBHostName {
        open(BBHOSTS,"<$BBHOME/etc/bb-hosts") or die "Failed opening $BBHOME/etc/bb-hosts. ($!)";
        @Hosts = ;
        @Host = grep(/$LocalHost/,@Hosts);
        @Host = split(/\s+/,$Host[0]);
	$Host[1] =~ s/\./,/g;
        return "$Host[1]";
}

#Remove leading and trailing space from a string
sub CleanString {
                my $Var = $_[0];
                chomp($Var);
                $Var =~ s/\s+$//;
                $Var =~ s/^\s+//;
                return $Var;
}

sub GetProcList {
	my(@ps,$proc,@procs,$procs);
	@PS = `/usr/bin/ps -eo pid,args`;
	foreach $proc (@PS) {
		chomp($proc);
		$proc = CleanString("$proc");
		@procs = split(/\s+/,$proc);
		$procs{$procs[0]} = $procs[1];
	}
	return %procs;
}

sub GetOTR {
	if ( -e "$BBTMP/OTR.dat" ) {
		open(OTR,"<$BBTMP/OTR.dat");
		my $OTR_TIME = ;
		close(OTR);
		return $OTR_TIME;
	} else {
		open(OTR,">$BBTMP/OTR.dat");
		print OTR "0";
		close(OTR);
		return 0;
	}
}


You are visitor number 1734