#!/bin/ksh # # SCRIPT: BacklogCheck.ksh # VERSION: 1.8.6.B # AUTHOR: Omar A. Villa # DATE: 01/27/2012 # # PLATFORM: Solaris, Linux # # PURPOSE: Calculate Daily Backups and Backlog, present Available space and duplication status # # REV LIST: # DATE: 01/27/2012 BY: Omar A Villa # MODIFICATION: Script Created # DATE: 01/30/2012 BY: Omar A Villa # MODIFICATION: Added GetDailyDups & Print Results Functions (ver 1.1.0.B) # DATE: 02/02/2012 BY: Omar A Villa # MODIFICATION: Added _GetNBUVersion (ver 1.2.0.B) # DATE: 02/23/2012 BY: Omar A Villa # MODIFICATION: Added Arguments Validation (Ver 1.2.1.B) # DATE: 02/27/2012 BY: Omar A Villa # MODIFICATION: Added _CleanLogs Function (Ver 1.3.1.B) # DATE: 02/29/2012 BY: Omar A Villa # MODIFICATION: Documented SendMail and PrintResults Functions (Ver 1.3.2.B) # DATE: 02/29/2012 BY: Omar A Villa # MODIFICATION: Added Send Mail Argument Option (Ver 1.3.3.B) # DATE: 03/02/2012 BY: Omar A Villa # MODIFICATION: Added GetImagesState Function (Ver 1.4.3.B) # DATE: 03/02/2012 BY: Omar A Villa # MODIFICATION: Extend Script Argument Options (Ver 1.4.4.B) # DATE: 03/07/2012 BY: Omar A Villa # MODIFICATION: Fixed Bug in GetImagesState Function (Ver 1.4.5.B) # DATE: 03/23/2012 BY: Omar A Villa # MODIFICATION: NBU Command Full Paths are saved in to a variable to help cron find the command (Ver 1.4.6.B) # DATE: 03/23/2012 BY: Omar A Villa # MODIFICATION: Replaced GetAdvDiskFreeSpace with GetStsDiskFreeSpace to capture all Storage Types Free Space (Ver 1.5.7.B) # DATE: 03/26/2012 BY: Omar A Villa # MODIFICATION: Introduce in to CleanLogs function the avility to delete logs based on user Argument (Ver 1.5.8.B) # DATE: 03/28/2012 BY: Omar A Villa # MODIFICATION: Fixed Mail Header in SendMail Function (Ver 1.5.9.B) # DATE: 03/29/2012 BY: Omar A Villa # MODIFICATION: Added GetClientsBacklog Function (Ver 1.6.9.B) # DATE: 04/04/2012 BY: Omar A Villa # MODIFICATION: Added GetImagesSizes Function (Ver 1.7.9.B) # DATE: 06/05/2012 BY: Omar A Villa # MODIFICATION: Introduced GetMediaServerDups Function (Ver 1.8.1.B) # DATE: 07/15/2013 BY: Omar A Villa # MODIFICATION: Added GetLibraryBacklog Function (Ver 1.8.2.B) # DATE: 11/14/2013 BY: Omar A Villa # MODIFICATION: Fixed bug under ValidateFilesAndFolders function when script is run by first time (Ver 1.8.3.B) # DATE: 11/14/2013 BY: Omar A Villa # MODIFICATION: Improved SendMail function to identify mail or mailx commands (Ver 1.8.4.B) # DATE: 02/01/2014 BY: Kevin Good # MODIFICATION: Improved GetSLPsBklog algorithm. Print only SLP's with backlog (Ver 1.8.5.B) # DATE: 02/01/2014 BY: Omar A Villa # MODIFICATION: Added -k parameter in to main to only list SLP's with backlog (Ver 1.8.6.B) # # set -n # Uncomment to check your syntax, without execution. # # NOTE: Do not forget to put the comment back in or # # the shell script will not execute! # set -x # Uncomment to debug this shell script (Korn shell only) # ########################################################## ########### FILES AND VARIABLES ############## ########################################################## SCRIPT_NAME=BacklogCheck LOG_DATE=$(date '+%m%d%y') LOG_DIR=/var/log/$SCRIPT_NAME/logs LOG_FILE=$LOG_DIR/$SCRIPT_NAME-Error.$LOG_DATE.log LOG_LAST_RUN=$LOG_DIR/$SCRIPT_NAME-LastRun.$LOG_DATE.log SLP_DUMP=$LOG_DIR/$SCRIPT_NAME.DUMP REPORT=$LOG_DIR/$SCRIPT_NAME.REPORT.$LOG_DATE.OUT HOSTNAME=$(hostname) ############### FUNCTIONS PROTOTYPE ###################### # void Usage (void) # Print Usage, Syntax and Help information. # void ValidateFilesAndFolders (void) # Check critical folders and files are in place and exist. # void DataDumps (void) # Collect data and dump it in to a files. # void GetTotalBklog (void) # Sum all pending images to be duplicated # void GetMediaServerBklog (void) # Sum pending images to be duplicated by media server # void GetSLPsBklog (void) # Sum pending images to be duplicated by SLP # void GetAdvDiskFreeSpace (void) # Deprecated by GetStsDiskFreeSpace # void GetStsDiskFreeSpace (void) # Capture All Volumes Free Space by Media Server and Storage Type plus Total Available Space. # void GetDate (void) # Capture actual date and print it in report head. # void GetSLPLast24hrBkp (void) # Pull last 24 hours images and sum them by SLP. # int _GetNBUVersion (void) # Collects NBU Version in order to sort the right column from bpimagelist command when we use GetSLPLast24hrBkp # void CleanLogs (int) # Delete log files older than N days based on User ARGV # void SendMail (string) # Sends Report to specified eMail under a string parameter. # void PrintResults (void) # Prints Report on Screen # void GetImagesState (void) # Print images count and sum by state # void GetClientsBacklog (int) # Print top X clients with largest backlog with -a option is Top 10 # void GetImagesSizes (void) # Print images count by size range to help tune nbstserv batching # void GetDailyDups (void) # Print Duplications completed by date # void GetMediaServerDups (void) # Print Duplications completed by media server # void GetLibraryBacklog (void) # Sum Backlog own by source libraries spliting the libraries based on tape barcodes function Usage { printf "\nIncorrect Syntax or missing parameter.\n\n \ SYNTAX: BacklogCheck.ksh -a | -sSkBbDdFMplh [-m ] [-C ] | [-c ] \n \ -a:\tPrint Full Report\n \ -s:\tPrint Short Report (NO SLP's)\n \ -S:\tPrint, Count and Sum SLP Images State\n \ -k:\tList SLP's with Backlog\n \ -B:\tPrint Total Backlog in TB \n \ -b:\tPrint last 24hr Backup Info splited by SLP\n \ -c:\tDelete log files older than N days based on User Argunment\n \ -C:\tGet Top X Clients backlog were X is the desired top clients list\n \ -D:\tPrint Sum of Daily Duplications\n \ -d:\tPrint Sum of Media Server Duplications\n \ -i:\tPrint images count by size range\n \ -F:\tPrint DSU's Free Space \n \ -M:\tPrint Backlog hold by Media Server \n \ -m:\tSend Report to a Specified eMail\n \ -l:\tPrint Backlog hold by Library\n \ -h:\tPrint this help.\n \ Sample:\t./BacklogCheck.ksh -a -m darth.vader@thedarkside.com \n" } function ValidateFilesAndFolders { if [[ -d $LOG_DIR ]] # Validate Log directory exists then CleanFiles echo "$(date "+%b %d %I:%M:%S") $HOSTNAME $SCRIPT_NAME.ksh: INF: Log directory exists" 2>$LOG_FILE | tee -a $LOG_LAST_RUN >/dev/null else mkdir -p $LOG_DIR echo "$(date "+%b %d %I:%M:%S") $HOSTNAME $SCRIPT_NAME.ksh: INF: Log directory didnt exist, folder created" 2>$LOG_FILE | tee -a $LOG_LAST_RUN >/dev/null fi echo "$(date "+%b %d %I:%M:%S") $HOSTNAME $SCRIPT_NAME.ksh: INF: Files and Folders Validated, Starting Script for host $HOSTNAME" 2>$LOG_FILE | tee -a $LOG_LAST_RUN >/dev/null } function CleanFiles { >$LOG_LAST_RUN # Wipeout BuildSLPs-LastRun.log file >$REPORT # Wipeout Last Report >$SLP_DUMP # Wipeout Last SLP Dump } function CleanLogs { { ((DAYSTOKEEP=$1+1)) ls -lat $LOG_DIR | egrep -vi "^d" | grep -i LastRun | tail +$DAYSTOKEEP | awk '{print $NF}' | while read LOG do rm $LOG_DIR/$LOG echo "$(date "+%b %d %I:%M:%S") $HOSTNAME $SCRIPT_NAME.ksh: INF: Log $LOG file Deleted" done ls -lat $LOG_DIR | egrep -vi "^d" | grep -i REPORT | tail +$DAYSTOKEEP | awk '{print $NF}' | while read LOG do rm $LOG_DIR/$LOG echo "$(date "+%b %d %I:%M:%S") $HOSTNAME $SCRIPT_NAME.ksh: INF: Log $LOG file Deleted" done ls -lat $LOG_DIR | egrep -vi "^d" | grep -i ERROR | tail +$DAYSTOKEEP | awk '{print $NF}' | while read LOG do rm $LOG_DIR/$LOG echo "$(date "+%b %d %I:%M:%S") $HOSTNAME $SCRIPT_NAME.ksh: INF: Log $LOG file Deleted" done } 2>$LOG_FILE | tee -a $LOG_LAST_RUN >/dev/null } function DataDumps { NBSTLUTIL=/usr/openv/netbackup/bin/admincmd/nbstlutil $NBSTLUTIL list -l -image_incomplete > $SLP_DUMP } function GetMediaServerBklog { { printf "%-30s%s\n" "Media Server DSU" "Backlog Size" awk '{if ($2=="F") {print $9,$14} }' $SLP_DUMP | sort | \ awk ' { MED_LIST[$1]+=$2 } END { for(MED in MED_LIST) printf ("%-30s%.2f TB\n", MED, MED_LIST[MED]/1024/1024/1024/1024) |"sort" }' printf "\n\n" } 2>$LOG_FILE | tee -a $REPORT >/dev/null } function GetTotalBklog { { awk ' $2=="F" {SUM+=$14} END { printf ("%-30s%.2f TB\n\n", "Total Backlog ", SUM/1024/1024/1024/1024 ) }' $SLP_DUMP } 2>$LOG_FILE | tee -a $REPORT >/dev/null } function GetSLPsBklog { { awk ' { if($2=="I") {SLPNAME=$10} else if($2=="F") {SUM[SLPNAME]+=$(NF-2)} } END { for (FoundSLP in SUM) printf ("%50s%20.2f TB\n", FoundSLP, SUM[FoundSLP]/1024/1024/1024/1024) }' $SLP_DUMP } 2>$LOG_FILE | tee -a $REPORT >/dev/null } function GetStsDiskFreeSpace { { >$LOG_DIR/total.log # Wipeout total.log file used to Summarize Free Space. NBDEVQUERY=/usr/openv/netbackup/bin/admincmd/nbdevquery printf "%-30s%-20s%s\n" "Media Server" "Storage Type" "Free Space" $NBDEVQUERY -liststs -l | awk '{print $3}' | sort -u | while read STYPE do $NBDEVQUERY -listdv -stype $STYPE -l | awk ' { SER[$2]+=$7 TOTAL+=$7 } END { print TOTAL >> "'$LOG_DIR'/total.log" for (INC in SER) {printf ("%-30s%-20s%.2f TB\n",INC, awkSTYPE, SER[INC]/1024)} }' awkSTYPE=$STYPE done awk ' { TOTAL+=$1 } END { printf ("\n%-50s%.2f TB\n\n", "Total Free Space", TOTAL/1024) }' $LOG_DIR/total.log } 2>$LOG_FILE | tee -a $REPORT >/dev/null } function _GetNBUVersion { VER=$(cat /usr/openv/netbackup/version | grep -i version | awk '{print $NF}' | awk -F. '{print $1}') print $VER } function GetSLPLast24hrBkp { { BPIMAGELIST=/usr/openv/netbackup/bin/admincmd/bpimagelist NBUVER=$(_GetNBUVersion) if (( NBUVER == 7 )) then IMG_SLP_COL=6 else IMG_SLP_COL=3 fi $BPIMAGELIST -l -hoursago 24 | grep -i ^IMAGE | awk ' { SLP_LIST[$(NF-SLPCOL)]+=$19 TOTAL+=$19 } END { printf ("%-30s%.2f TB\n\n", "Total 24hr Backup", TOTAL/1024/1024/1024) printf ("%50s%27s\n", "Policy Name", "Backup Size") for (SLP in SLP_LIST) {printf ("%50s%20.2f TB\n", SLP,SLP_LIST[SLP]/1024/1024/1024)} }' SLPCOL=$IMG_SLP_COL printf "\n\n" } 2>$LOG_FILE | tee -a $REPORT >/dev/null } function GetDailyDups { { BPDBJOBS=/usr/openv/netbackup/bin/admincmd/bpdbjobs BPDBM=/usr/openv/netbackup/bin/bpdbm printf "%-30s%s\n" "Date" "Duplicated Data" $BPDBJOBS -report | grep -i duplica | grep -i done | awk '{print $1}' | tr '\n' ',' | read JOBSLIST echo $JOBSLIST | wc -m | read JOBSCHARS ((JOBSNUM=$JOBSCHARS-2)) echo $JOBSLIST | cut -c1-$JOBSNUM | read FINALJOBSLIST $BPDBJOBS -jobid $FINALJOBSLIST -most_columns | awk -F, '{print $8,$9,$15}' | while read UDATE SIZEKB do RES=$($BPDBM -ctime $UDATE | awk '{print $4"-"$5"-"$NF}') echo $RES $SIZEKB done | awk ' { DAYLIST[$1]+=$2 } END { for (DAYDUP in DAYLIST) printf ("%-30s%.2f%s\n",DAYDUP, DAYLIST[DAYDUP]/1024/1024/1024, "TB" ) }' | sort -n printf "\n\n" } 2>$LOG_FILE | tee -a $REPORT >/dev/null } function GetMediaServerDups { { BPDBJOBS=/usr/openv/netbackup/bin/admincmd/bpdbjobs BPDBM=/usr/openv/netbackup/bin/bpdbm printf "%-30s%-30s%s\n" "Media Server" "Duplicated/5days)" "Average/Day" $BPDBJOBS -report | grep -i duplica | grep -i done | awk '{print $1}' | tr '\n' ',' | read JOBSLIST echo $JOBSLIST | wc -m | read JOBSCHARS ((JOBSNUM=$JOBSCHARS-2)) echo $JOBSLIST | cut -c1-$JOBSNUM | read FINALJOBSLIST $BPDBJOBS -jobid $FINALJOBSLIST -most_columns | \ awk -F, ' { MEDIASERVER=$8 SIZEKB=$15 MEDLIST[MEDIASERVER]+=SIZEKB } END { for (MEDDUP in MEDLIST) printf ("%-30s%-30.2f%.2f%s\n",MEDDUP, MEDLIST[MEDDUP]/1024/1024/1024, MEDLIST[MEDDUP]/1024/1024/1024/5, " TB" ) }' | sort -n printf "\n\n" } 2>$LOG_FILE | tee -a $REPORT >/dev/null } function GetImagesState { { printf "%-30s%-15s%s\n" "IMAGE STATUS" "IMAGES COUNT" "SIZE" awk ' $2=="I" { IMAGE=$4 STATE_COL=$11 if (STATE_COL == 0) STATE = "NOT_MANAGED" else if (STATE_COL == 1) STATE = "NOT_STARTED" else if (STATE_COL == 2) STATE = "IN_PROCESS" else if (STATE_COL == 3) STATE = "COMPLETE" else if (STATE_COL == 9) STATE = "NOT_STARTED INACTIVE" else if (STATE_COL == 10) STATE = "IN_PROCESS INACTIVE" else STATE = "OTHER" IMG_STATE_LIST[STATE]+=1 } $2=="F" && $4==IMAGE { IMG_SUM[STATE]+=$14 } END { for (STATE_ELM in IMG_STATE_LIST) printf ("%-30s%-15d%.2f TB\n", STATE_ELM, IMG_STATE_LIST[STATE_ELM], IMG_SUM[STATE_ELM]/1024/1024/1024/1024) }' $SLP_DUMP | sort printf "\n\n" } 2>$LOG_FILE | tee -a $REPORT >/dev/null } function GetClientsBacklog { { TOP=$1 printf "%-30s%s\n" "Client Name" "Backlog Size" awk '$2=="F" {print $4,$14} ' $SLP_DUMP | tr '_' ' ' | awk ' { CLIENT_LIST[$1]+=$3 } END { for (CLIENT in CLIENT_LIST) printf ("%-30s%.2f GB\n", CLIENT, CLIENT_LIST[CLIENT]/1024/1024/1024) }' | sort -nk 2,2 | tail -$TOP print "\n\n" } 2>$LOG_FILE | tee -a $REPORT >/dev/null } function GetImagesSizes { { awk ' $2=="I" { IMAGE=$4 } $2=="F" && $4==IMAGE { IMGSUM[IMAGE]+=$14 } END { S100MB=104857600 S500MB=524288000 S1GB=1073741824 S5GB=5368709120 S10GB=10737418240 S50GB=53687091200 S100GB=107374182400 S250GB=268435456000 S500GB=536870912000 S1TB=1073741824000 for (IMGSIZE in IMGSUM) { if (IMGSUM[IMGSIZE] <= S100MB) S100MB_COUNT+=1 else if (IMGSUM[IMGSIZE] > S100MB && IMGSUM[IMGSIZE] <= S500MB) S500MB_COUNT+=1 else if (IMGSUM[IMGSIZE] > S500MB && IMGSUM[IMGSIZE] <= S1GB) S1GB_COUNT+=1 else if (IMGSUM[IMGSIZE] > S1GB && IMGSUM[IMGSIZE] <= S5GB) S5GB_COUNT+=1 else if (IMGSUM[IMGSIZE] > S5GB && IMGSUM[IMGSIZE] <= S10GB) S10GB_COUNT+=1 else if (IMGSUM[IMGSIZE] > S10GB && IMGSUM[IMGSIZE] <= S50GB) S50GB_COUNT+=1 else if (IMGSUM[IMGSIZE] > S50GB && IMGSUM[IMGSIZE] <= S100GB) S100GB_COUNT+=1 else if (IMGSUM[IMGSIZE] > S100GB && IMGSUM[IMGSIZE] <= S250GB) S250GB_COUNT+=1 else if (IMGSUM[IMGSIZE] > S250GB && IMGSUM[IMGSIZE] <= S500GB) S500GB_COUNT+=1 else if (IMGSUM[IMGSIZE] > S500GB && IMGSUM[IMGSIZE] <= S1TB) S1TB_COUNT+=1 else SM1TB_COUNT+=1 } printf ("Images Size Range Image Count\n") printf ("< 100MB %d\n", S100MB_COUNT) printf ("> 100MB < 500MB %d\n", S500MB_COUNT) printf ("> 500MB < 1GB %d\n", S1GB_COUNT) printf ("> 1GB < 5GB %d\n", S5GB_COUNT) printf ("> 5GB < 10GB %d\n", S10GB_COUNT) printf ("> 10GB < 50GB %d\n", S50GB_COUNT) printf ("> 50GB < 100GB %d\n", S100GB_COUNT) printf ("> 100GB < 250GB %d\n", S250GB_COUNT) printf ("> 250GB < 500GB %d\n", S500GB_COUNT) printf ("> 500GB < 1TB %d\n", S1TB_COUNT) printf ("> 1TB %d\n", SM1TB_COUNT) }' $SLP_DUMP printf "\n\n" } 2>$LOG_FILE | tee -a $REPORT >/dev/null } function GetLibraryBacklog { { #set -x DESCRIPTION=$1 awk '{if ($2=="F") {print substr($8,0,2),$9,$14} }' $SLP_DUMP | awk ' { ARRAY[$1]+=$3 } END { for (BAR in ARRAY) printf ("%-5s%20s\n", BAR, ARRAY[BAR]/1024/1024/1024/1024) }' > $LOG_DIR/GetLibraryBacklog.out if ( $DESCRIPTION == 1 ) then vmquery -b -a > $LOG_DIR/vmquery.dump while read BARCODE SIZE do TAPE=$(grep -i $BARCODE $LOG_DIR/vmquery.dump | tail -1 | awk '{print $1}') LIBRARY=$(vmquery -m $TAPE | grep -i description | awk '{print $3}') printf "%-5s%20s%.2f\n", $BARCODE, $LIBRARY, $SIZE done < $LOG_DIR/GetLibraryBacklog.out else cat $LOG_DIR/GetLibraryBacklog.out fi print "\n\n" } 2>$LOG_FILE | tee -a $REPORT >/dev/null } function GetDate { { date print "\n" } 2>$LOG_FILE | tee -a $REPORT >/dev/null } function PrintResults { cat $REPORT } function SendMail { eMAIL=$1 #grep -i total $REPORT | awk '{print $(NF-1)}' | tr '\n' ' ' which mailx 1>/dev/null 2>&1 if [ "$?" -eq "0" ]; then mailx -s "$HOSTNAME Swing SLPs REPORT" $eMAIL < $REPORT else mail -s "$HOSTNAME Swing SLPs REPORT" $eMAIL < $REPORT fi } ### MAIN if [[ $# -eq 0 ]] then print "Error: Missing Argument" Usage exit 0 else ValidateFilesAndFolders GetDate while getopts "asSkc:C:BbDdiFMm:l:h" opt; do case $opt in a ) DataDumps GetTotalBklog GetStsDiskFreeSpace GetImagesState GetMediaServerBklog GetDailyDups GetMediaServerDups GetLibraryBacklog 1 GetClientsBacklog 10 GetImagesSizes GetSLPLast24hrBkp GetSLPsBklog ;; s ) DataDumps GetTotalBklog GetStsDiskFreeSpace GetMediaServerBklog GetSLPsBklog ;; S ) DataDumps GetImagesState ;; k ) DataDumps GetSLPsBklog ;; B ) DataDumps GetTotalBklog ;; b ) GetSLPLast24hrBkp ;; c ) CleanLogs $OPTARG ;; C ) DataDumps GetClientsBacklog $OPTARG ;; D ) GetDailyDups ;; d ) GetMediaServerDups ;; i ) DataDumps GetImagesSizes ;; F ) #GetAdvDiskFreeSpace # Decom function GetStsDiskFreeSpace ;; M ) DataDumps GetMediaServerBklog ;; m ) SendMail $OPTARG ;; l ) DataDumps GetLibraryBacklog $OPTARG ;; h|* ) Usage exit 0 ;; esac done shift $(($OPTIND - 1)) PrintResults fi