Database rman backup failed with file does not exist

I have a database of 20TB which is backed up using RMAN. The backup runs but most times failed after the channels have been downed due to the error below

released channel: ch05
released channel: ch06
RMAN-00571: ===========================================================
RMAN-00569: =============== ERROR MESSAGE STACK FOLLOWS ===============
RMAN-00571: ===========================================================
RMAN-03009: failure of backup command on ch00 channel at 02/17/2013 10:05:04
ORA-01157: cannot identify/lock data file 84 - see DBWR trace file
ORA-01110: data file 84: 'No file with this number, file does not exist'
 

I have increased the number of channels, but most times the backup failed when all the channels are downed. The error also comes as shown below

RMAN-03009: failure of backup command on ch01 channel at 01/05/2013 00:18:57

ORA-19505: failed to identify file "+DATA/xman/datafile/xman_eri_sgsn_f_20121005.338.795654005"

ORA-17503: ksfdopn:2 Failed to open file +DATA/xman/datafile/xman_eri_sgsn_f_20121005.338.795654005

ORA-15012: ASM file '+DATA/xman/datafile/xman_eri_sgsn_f_20121005.338.795654005' does not exist

 

Below is the RMAN script

root@kingfisher # cat RmanDatabase-xman.sh
#!/bin/sh
#
#
ORACLE_SID=XMAN
ORACLE_BASE=/oracle/app
ORACLE_HOME=$ORACLE_BASE/product/11.2.0/db1
ORACLE_USER=oracle
#
#
#CONNECT_STR='target  / nocatalog'
CONNECT_STR='target  / catalog rman/rman_user@prcat'
RMAN_LOG_RETENTION=+14
#
RMAN=rman
RMAN_LOG_DIR=/oracle/admin/$ORACLE_SID/rman.dated
#
#
RMAN_LOG_DIR_TODAY=$RMAN_LOG_DIR/`date +%d%h%Y`
RMAN_PID_DIR=$RMAN_LOG_DIR/pid
#
/usr/bin/test -d $RMAN_PID_DIR || mkdir -p $RMAN_PID_DIR
/usr/bin/test -d $RMAN_LOG_DIR_TODAY || mkdir -p $RMAN_LOG_DIR_TODAY
#
if [ $RMAN_LOG_DIR ] ;then
 find $RMAN_LOG_DIR -mtime $RMAN_LOG_RETENTION -exec rm -r {} \;
fi
#
#
echo $$ > $RMAN_PID_DIR/Rman-database-$ORACLE_SID-$$.pid

if [ "$NB_ORA_FULL" = "1" ]
then
        RMAN_LOG_FILE=$RMAN_LOG_DIR_TODAY/Full-$ORACLE_SID-Database.`date +%d%h%Y:%H:%M:%S`
        echo ==== started on `date` ==== >> $RMAN_LOG_FILE
        echo "Full backup requested" >> $RMAN_LOG_FILE
        BACKUP_TYPE="INCREMENTAL LEVEL=0"

elif [ "$NB_ORA_INCR" = "1" ]
then
        RMAN_LOG_FILE=$RMAN_LOG_DIR_TODAY/Full-$ORACLE_SID-Database.`date +%d%h%Y:%H:%M:%S`
        echo ==== started on `date` ==== >> $RMAN_LOG_FILE
        echo "Differential incremental backup requested" >> $RMAN_LOG_FILE
        BACKUP_TYPE="INCREMENTAL LEVEL=1"

elif [ "$NB_ORA_CINC" = "1" ]
then
        RMAN_LOG_FILE=$RMAN_LOG_DIR_TODAY/Full-$ORACLE_SID-Database.`date +%d%h%Y:%H:%M:%S`
        echo ==== started on `date` ==== >> $RMAN_LOG_FILE
        echo "Cumulative incremental backup requested" >> $RMAN_LOG_FILE
        BACKUP_TYPE="INCREMENTAL LEVEL=1 CUMULATIVE"

elif [ "$BACKUP_TYPE" = "" ]
then
        RMAN_LOG_FILE=$RMAN_LOG_DIR_TODAY/Full-$ORACLE_SID-Database.`date +%d%h%Y:%H:%M:%S`
        echo ==== started on `date` ==== >> $RMAN_LOG_FILE
        echo "Default - Full backup requested" >> $RMAN_LOG_FILE
        BACKUP_TYPE="INCREMENTAL LEVEL=0"
fi

# Determine the user which is executing this script.
# ---------------------------------------------------------------------------
#
CUSER=`id |cut -d"(" -f2 | cut -d ")" -f1`
#
/usr/bin/chmod 666 $RMAN_LOG_FILE
#
# ---------------------------------------------------------------------------
# Print out the value of the variables set by this script.
# ---------------------------------------------------------------------------
#
echo >> $RMAN_LOG_FILE
echo   "RMAN: $RMAN" >> $RMAN_LOG_FILE
echo   "ORACLE_USER: $ORACLE_USER" >> $RMAN_LOG_FILE
echo   "ORACLE_SID: $ORACLE_SID" >> $RMAN_LOG_FILE
echo   "ORACLE_HOME: $ORACLE_HOME" >> $RMAN_LOG_FILE
#
# ---------------------------------------------------------------------------
# Print out the value of the variables set by bphdb.
# ---------------------------------------------------------------------------
#
echo  >> $RMAN_LOG_FILE
echo   "NB_ORA_FULL: $NB_ORA_FULL" >> $RMAN_LOG_FILE
echo   "NB_ORA_INCR: $NB_ORA_INCR" >> $RMAN_LOG_FILE
echo   "NB_ORA_CINC: $NB_ORA_CINC" >> $RMAN_LOG_FILE
echo   "NB_ORA_SERV: $NB_ORA_SERV" >> $RMAN_LOG_FILE
echo   "NB_ORA_POLICY: $NB_ORA_POLICY" >> $RMAN_LOG_FILE
echo >> $RMAN_LOG_FILE
#
CMD_STR="
ORACLE_HOME=$ORACLE_BASE/product/11.2.0/db1
export ORACLE_HOME
ORACLE_SID=$ORACLE_SID
export ORACLE_SID
ORAENV_ASK=NO
export ORAENV_ASK
. oraenv
$ORACLE_BASE/OracleRmanScripts/netbackup_exclude
$RMAN $CONNECT_STR msglog $RMAN_LOG_FILE append << EOF
RUN {
ALLOCATE CHANNEL ch00 DEVICE TYPE 'SBT_TAPE' parms='ENV=(NB_ORA_POLICY=$NB_ORA_POLICY,NB_ORA_SERV=$NB_ORA_SERV)';
ALLOCATE CHANNEL ch01 DEVICE TYPE 'SBT_TAPE' parms='ENV=(NB_ORA_POLICY=$NB_ORA_POLICY,NB_ORA_SERV=$NB_ORA_SERV)';
ALLOCATE CHANNEL ch02 DEVICE TYPE 'SBT_TAPE' parms='ENV=(NB_ORA_POLICY=$NB_ORA_POLICY,NB_ORA_SERV=$NB_ORA_SERV)';
ALLOCATE CHANNEL ch03 DEVICE TYPE 'SBT_TAPE' parms='ENV=(NB_ORA_POLICY=$NB_ORA_POLICY,NB_ORA_SERV=$NB_ORA_SERV)';
ALLOCATE CHANNEL ch04 DEVICE TYPE 'SBT_TAPE' parms='ENV=(NB_ORA_POLICY=$NB_ORA_POLICY,NB_ORA_SERV=$NB_ORA_SERV)';
ALLOCATE CHANNEL ch05 DEVICE TYPE 'SBT_TAPE' parms='ENV=(NB_ORA_POLICY=$NB_ORA_POLICY,NB_ORA_SERV=$NB_ORA_SERV)';
ALLOCATE CHANNEL ch06 DEVICE TYPE 'SBT_TAPE' parms='ENV=(NB_ORA_POLICY=$NB_ORA_POLICY,NB_ORA_SERV=$NB_ORA_SERV)';
#BACKUP not backed up since time 'sysdate -1'
backup
INCREMENTAL LEVEL 0 keep until time 'sysdate+31' logs
TAG hot_db_bk_level
FILESPERSET 20
FORMAT 'bk_%s_%p_%t'
SKIP INACCESSIBLE
(database include current controlfile);
sql 'alter system archive log current';
Crosscheck Archivelog all;
backup
format '%s_%t_%r.arc'
(archivelog all);
#(archivelog all delete  input);
RELEASE CHANNEL ch00;
RELEASE CHANNEL ch01;
RELEASE CHANNEL ch02;
RELEASE CHANNEL ch03;
RELEASE CHANNEL ch04;
RELEASE CHANNEL ch05;
RELEASE CHANNEL ch06;
}
EOF
"
#
# Initiate the command string

echo    su - $ORACLE_USER -c "$CMD_STR" >> $RMAN_LOG_FILE
if [ "$CUSER" = "root" ]
then
    su - $ORACLE_USER -c "$CMD_STR" >> $RMAN_LOG_FILE
    RSTAT=$?
else
    /usr/bin/sh -c "$CMD_STR" >> $RMAN_LOG_FILE
    RSTAT=$?
fi

# ---------------------------------------------------------------------------
# Log the completion of this script.
# ---------------------------------------------------------------------------

if [ "$RSTAT" = "0" ]
then
    LOGMSG="ended successfully"
else
    LOGMSG="ended in error"
fi

# ----------------------------------------------------------------------------
# Change the ownership of Directory / Files.
# ----------------------------------------------------------------------------
chown -R oracle:dba $RMAN_LOG_DIR_TODAY
chmod -R o-w $RMAN_LOG_DIR_TODAY
#
echo >> $RMAN_LOG_FILE
echo Script $0 >> $RMAN_LOG_FILE
echo ==== $LOGMSG on `date` ==== >> $RMAN_LOG_FILE
echo >> $RMAN_LOG_FILE