#!/bin/bash
#########################################################################
#                                                                       #
# Script to generate Backup Status Reports                              #
# Modification Date:      01/13/2010                                    #
#                                                                       #
# Author: Don Wilder                                                    #
#                                                                       #
# History:                                                              #
#   Ver 1.0 - Initial coding based on a report script provided by       #
#             Veritas with the ETB 5.1 software                         #
#   Ver 1.1 - Added checks for no backups registered in over 1 week     #
#   Ver 1.2 - Added code to ignore client if it had a good backup the   #
#             current day.                                              #
#                                                                       #
# Purpose:                                                              #
#       This script generates a daily status report all for NBU client  #
#       backups and mails alert reports to someone_who_cares            #
#       This script supposes that an error code 0 or 1 are not hard     #
#       errors and thus they will not be added to the "bad" COUNT       #
#                                                                       #
#########################################################################

Version='1.2'
appName=${0##*/}

#               Who gets default email
someone_who_cares="someone@somewhere.com,someone2@somewhere.else.com"

#### Variables ####
#
today=`date +%m-%d-%y`
day_of_week=`date +%w`
yesterday=$(( ${day_of_week} - 1 ))
if [ ${yesterday} = -1 ]
then
    yesterday=6
fi

yesterday2=$(( ${day_of_week} - 2 ))
if [ ${yesterday2} = -1 ]
then
    yesterday2=6
else
    if [ ${yesterday2} = -2 ]
    then
        yesterday2=5
    fi
fi

# Customize this to where you want to keep the history files
HISTORY_DIR=/usr/openv/nbu_history/Client_Risk_Reports

OUTPUT=/tmp/nbu_client_risk
ARCHIVE1=${HISTORY_DIR}/nbu_client_risk.${yesterday}
ARCHIVE2=${HISTORY_DIR}/nbu_client_risk.${yesterday2}
REPORT=${HISTORY_DIR}/nbu_client_risk-rpt.${today}

#echo "History Dir = ${HISTORY_DIR}"
#echo "Output File Current = ${OUTPUT}"
#echo "Output File Previous = ${ARCHIVE1}"
#echo "Report File = ${REPORT}"

#       Change this value to search further
#       back in the error reports
HOURSAGO=24
COUNT=0
TOTAL=0
#
#               What to exclude from REPORT
EXCLUDE_S="XXXXXXXXXXXXXXXXXX"
#
#               Makes Report more selective
SEARCH_S="house"
#
#               Makes Report on policy
POLICY_S=0
#
#               Creates Summary Report
# RSUMFLAG = 0 for detail only report"
# RSUMFLAG = 1 for summary only report"
# RSUMFLAG = 2 for both or complete report (Default option)"
# RSUMFLAG = 3 for errors only report"
RSUMFLAG=1
#

ADMCMD=/usr/openv/netbackup/bin/admincmd
STATUS_CODES=/usr3/openv/nbu_scripts/NBU_Status_Codes.txt
CLIST=${OUTPUT}.clients.$$
${ADMCMD}/bpplclients | (line>/dev/null ;line>/dev/null;tee) | cut -c35-75 > ${CLIST}

### Report Header information
#
FORM4="==============================================================================="
FORM6="_______________________________________________________________________________"

Client_List=$( ${ADMCMD}/bpplclients | (line>/dev/null ;line>/dev/null;tee) | cut -c35-75 )

#
#      Get rid of any old output files if they are there. 
cat /dev/null > ${OUTPUT}.${day_of_week}
cat /dev/null > ${OUTPUT}.data.${day_of_week}
cat /dev/null > ${OUTPUT}.detail.${day_of_week}
cat /dev/null > ${OUTPUT}.header.${day_of_week}
cat /dev/null > ${OUTPUT}.summary.${day_of_week}
cat /dev/null > ${REPORT}.data-196
cat /dev/null > ${REPORT}.data-2d
cat /dev/null > ${REPORT}.data-3d
cat /dev/null > ${REPORT}.data-7d
cat /dev/null > ${REPORT}.data-${HOURSAGO}
cat /dev/null > ${REPORT}.data

#       Format Report with Title, etc.
echo "Reporting on the last "${HOURSAGO}" hours for ${SEARCH_S} backups.  "`date +%m/%d/%Y`"  ("`date +%H:%M`")" > ${OUTPUT}.header.${day_of_week
}

echo >> ${OUTPUT}.header.${day_of_week}
echo "${FORM4}" >> ${OUTPUT}.header.${day_of_week}
echo >> ${OUTPUT}.header.${day_of_week}


#       Begin Main Script

#echo "Client List --> ${Client_List}"
###########################################################################
#       Main Client data gathering Loop
#
for i in ${Client_List}
do

############################################################################
#       Look for servers with no backup history for the past 7 days
#
#    echo "Client = ${i}"
     ${ADMCMD}/bperror -U -backstat -hoursago 180 -client ${i}  > /dev/null 2>&1
     if [ $? != 0 ]
     then
            echo "Check Client ${i}. It is at risk because it has no backup history for the past 7 Days." >> ${REPORT}.data-7d
     fi

############################################################################
#       Look for servers with no backup history for the past ${HOURSAGO}
#
#    echo "Client = ${i}"
     ${ADMCMD}/bperror -U -backstat -hoursago ${HOURSAGO} -client ${i}  > /dev/null 2>&1
     if [ $? != 0 ]
     then
            echo "Check Client ${i}. It is at risk because it has no backup history for the past ${HOURSAGO} hours." >> ${REPORT}.data-${HOURSAGO
}
     fi

############################################################################
#       Look for Backups with Status code 0 because those are
#       successful and a backup image was created for them on tape
#
#    echo "Client = ${i}"
#		Swap these next 2 lines if you consider a return status code 1 a successful backup.
#    ${ADMCMD}/bperror -U -backstat -hoursago ${HOURSAGO} -client ${i} | egrep '  (0|1)  ' >> ${OUTPUT}.data.${day_of_week}
    ${ADMCMD}/bperror -U -backstat -hoursago ${HOURSAGO} -client ${i} | egrep '  0  ' >> ${OUTPUT}.data.${day_of_week}

############################################################################
#       Now that the successful backups were reported,
#       its time to log the error details
#
#		Swap these next 2 lines if you consider a return status code 1 a successful backup.
#     ${ADMCMD}/bperror -U -backstat -hoursago ${HOURSAGO} -client ${i} | egrep -v STATUS | egrep -v '  (0|1)  ' > /dev/null 2>&1
     ${ADMCMD}/bperror -U -backstat -hoursago ${HOURSAGO} -client ${i} | grep -v STATUS | grep -v '  0  ' > /dev/null 2>&1
     if [ $? = 0 ]
     then
#       echo "Checking if Client ${i} had good backup today."
        grep ${i} ${OUTPUT}.data.${day_of_week} > /dev/null 2>&1
        if [ $? != 0 ]
        then
#            echo "Client = ${i}"
             echo >> ${OUTPUT}.detail.${day_of_week}
             echo >> ${OUTPUT}.detail.${day_of_week}
             ${ADMCMD}/bperror -U -backstat -hoursago ${HOURSAGO} -client ${i} | grep -v STATUS | grep -v '  0  ' >> ${OUTPUT}.detail.${day_of_we
ek}
             echo "${FORM4}" >> ${OUTPUT}.detail.${day_of_week}
             echo >> ${OUTPUT}.detail.${day_of_week}
        fi
     fi
done

#
# Compute the job count details
# Setup the count array
COUNT_STATUS[700]=0

cp /dev/null ${OUTPUT}.0
cp /dev/null ${OUTPUT}.1
cp /dev/null ${OUTPUT}.2
cp /dev/null ${OUTPUT}.3
cp /dev/null ${OUTPUT}.4
cp /dev/null ${OUTPUT}.5

cp ${OUTPUT}.data.${day_of_week} ${OUTPUT}.9
cat ${OUTPUT}.detail.${day_of_week} >> ${OUTPUT}.9
sed 's/ ..\/.*//' ${OUTPUT}.9 > ${OUTPUT}.0;

for ((r = 0; r < 10; r += 1)) 
do 
{
#            echo "Number = ${r}"; 
    egrep "^  ${r}  " ${OUTPUT}.0 >> ${OUTPUT}.1; 
    COUNT_STATUS[r]=`egrep "^  ${r}  " ${OUTPUT}.0|wc -l`;
    TOTAL=`echo "${TOTAL}+${COUNT_STATUS[r]}" | bc`;
    }
done
sed 's/^  //' ${OUTPUT}.1 >> ${OUTPUT}.4;

for ((r = 10; r < 100; r += 1)) 
do 
{
#            echo "Number = ${r}"; 
    egrep ^" ${r}  " ${OUTPUT}.0 >> ${OUTPUT}.2; 
    COUNT_STATUS[r]=`egrep "^ ${r}  " ${OUTPUT}.0|wc -l`;
    TOTAL=`echo "${TOTAL}+${COUNT_STATUS[r]}" | bc`;
    }
done
sed 's/^ //' ${OUTPUT}.2 >> ${OUTPUT}.4;

for ((r = 100; r < 700; r += 1)) 
do 
{
#            echo "Number = ${r}"; 
    egrep ^"${r}  " ${OUTPUT}.0 >> ${OUTPUT}.3; 
    COUNT_STATUS[r]=`egrep "^${r}  " ${OUTPUT}.0|wc -l`;
    TOTAL=`echo "${TOTAL}+${COUNT_STATUS[r]}" | bc`;
    }
done
cat ${OUTPUT}.3 >> ${OUTPUT}.4;
sort -u ${OUTPUT}.4 > ${OUTPUT}.5

SUCESS=`echo "${COUNT_STATUS[0]}+${COUNT_STATUS[1]}" | bc`
echo "Successful Backup Count: ${SUCESS}" >> ${OUTPUT}.header.${day_of_week}
echo >> ${OUTPUT}.header.${day_of_week}
echo >> ${OUTPUT}.header.${day_of_week}

for ((r = 1; r < 700; r += 1))
do
{
    if [ ${COUNT_STATUS[r]} != 0 ]
    then
        echo "${FORM4}" >> ${OUTPUT}.summary.${day_of_week}
        echo >> ${OUTPUT}.summary.${day_of_week}
        echo "Status Code ${r} occurred ${COUNT_STATUS[r]} times." >> ${OUTPUT}.summary.${day_of_week};
        grep "^${r} " ${STATUS_CODES} >> ${OUTPUT}.summary.${day_of_week}
        echo "${FORM6}" >> ${OUTPUT}.summary.${day_of_week}
        echo >> ${OUTPUT}.summary.${day_of_week}
        egrep "^${r}  " ${OUTPUT}.5 >> ${OUTPUT}.summary.${day_of_week}
        echo >> ${OUTPUT}.summary.${day_of_week}
    fi
    }
done
echo "${FORM4}" >> ${OUTPUT}.summary.${day_of_week}

#
# Process and Assemble the Report
# 
cat ${OUTPUT}.header.${day_of_week} > ${OUTPUT}.${day_of_week}
echo >> ${OUTPUT}.${day_of_week}
cat ${OUTPUT}.summary.${day_of_week} >> ${OUTPUT}.${day_of_week}

Client_List=$( ${ADMCMD}/bpplclients | (line>/dev/null ;line>/dev/null;tee) | cut -c35-47 )
for i in ${Client_List}
do
#    echo "Client = ${i}"
     grep ${i} ${OUTPUT}.${day_of_week} > /dev/null 2>&1
     if [ $? = 0 ]
     then
#       echo "Checking if Client ${i} had good backup today. Process if not."
        grep ${i} ${OUTPUT}.data.${day_of_week} > /dev/null 2>&1
        if [ $? != 0 ]
        then
#          echo "1. Client = ${i}"
           z="196    ${i}"
#          echo "Search Arg = .${z}."
           grep "${z}" ${OUTPUT}.summary.${day_of_week} > /dev/null 2>&1
           if [ $? = 0 ]
           then
#             echo "2. 196 Client = ${i}"
              echo "Check Client ${i}. It is at risk because it missed it's time window today." >> ${REPORT}.data-196
           fi
#          echo "1. Client = ${i}"
           grep ${i} ${ARCHIVE1} > /dev/null 2>&1
           if [ $? = 0 ]
           then
#             echo "2. Client = ${i}"
              echo "Check Client ${i}. It is at risk because it has had problems 2 days in a row." >> ${REPORT}.data-2d
              grep ${i} ${ARCHIVE2} > /dev/null 2>&1
              if [ $? = 0 ]
              then
#                 echo "3. Client = ${i}"
                  echo "*** Check Client ${i}. It is at risk because it has had problems *3* days in a row." >> ${REPORT}.data-3d
              fi
           fi
        fi
     fi
done

#
# Assemble the report
## 7 Day Report
if [ -s ${REPORT}.data-7d ]
then
    echo >> ${REPORT}.data
    echo "Clients that have no backup history for the past 7 Days" >> ${REPORT}.data
    echo "${FORM6}" >> ${REPORT}.data
    cat ${REPORT}.data-7d >> ${REPORT}.data
    echo >> ${REPORT}.data
    echo "${FORM4}" >> ${REPORT}.data
    echo >> ${REPORT}.data
fi

## 3 bad Day Report
if [ -s ${REPORT}.data-3d ]
then
    echo >> ${REPORT}.data
    echo "Clients that have had problems for the past 3 consecutive Days" >> ${REPORT}.data
    echo "${FORM6}" >> ${REPORT}.data
    cat ${REPORT}.data-3d >> ${REPORT}.data
    echo >> ${REPORT}.data
    echo "${FORM4}" >> ${REPORT}.data
    echo >> ${REPORT}.data
fi

## 2 bad Day Report
if [ -s ${REPORT}.data-2d ]
then
    echo >> ${REPORT}.data
    echo "Clients that have had problems for the past 2 consecutive Days" >> ${REPORT}.data
    echo "${FORM6}" >> ${REPORT}.data
    cat ${REPORT}.data-2d >> ${REPORT}.data
    echo >> ${REPORT}.data
    echo "${FORM4}" >> ${REPORT}.data
    echo >> ${REPORT}.data
fi

## Exceeded Time Window Reporting
if [ -s ${REPORT}.data-196 ]
then
    echo >> ${REPORT}.data
    echo "****** Clients that have missed their time window and did not get backed up.******" >> ${REPORT}.data
    echo "${FORM6}" >> ${REPORT}.data
    cat ${REPORT}.data-196 >> ${REPORT}.data
    echo >> ${REPORT}.data
    echo "${FORM4}" >> ${REPORT}.data
    echo >> ${REPORT}.data
fi

## ${HOURSAGO} Reporting
if [ -s ${REPORT}.data-${HOURSAGO} ]
then
    echo >> ${REPORT}.data
    echo "Clients that have no backup history for the past ${HOURSAGO} Hours" >> ${REPORT}.data
    echo "${FORM6}" >> ${REPORT}.data
    cat ${REPORT}.data-${HOURSAGO} >> ${REPORT}.data
    echo >> ${REPORT}.data
    echo "${FORM4}" >> ${REPORT}.data
    echo >> ${REPORT}.data
fi

#
# mail the report to someone_who_cares
if [ -s ${REPORT}.data ]
then
    cat ${OUTPUT}.header.${day_of_week} > ${REPORT}
    cat ${REPORT}.data >> ${REPORT}
## Clients included in the Report
    echo >> ${REPORT}
    echo "${FORM4}" >> ${REPORT}
    echo "Clients Covered in this report:" >> ${REPORT}
    echo "${FORM6}" >> ${REPORT}
    cat ${CLIST} >> ${REPORT}
    mailx -s "Enterprise Clients at Risk Report for ${today}. " ${someone_who_cares} < ${REPORT}
else
## Clients included in the Report
    cat ${OUTPUT}.header.${day_of_week} > ${REPORT}
    echo >> ${REPORT}
    echo "All Enterprise Clients have had no problems for the past 7 days."
    echo >> ${REPORT}
    echo "${FORM4}" >> ${REPORT}
    echo "Clients Covered in this report:" >> ${REPORT}
    echo "${FORM6}" >> ${REPORT}
    cat ${CLIST} >> ${REPORT}
    mailx -s "Enterprise Clients at Risk Report for ${today}. " ${someone_who_cares} < ${REPORT}
fi

# Clean up the /tmp files
mv ${OUTPUT}.${day_of_week} ${HISTORY_DIR}
rm -f ${OUTPUT}* ${REPORT}.data*
#
#EOF